diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..0c21d9c68e477542c6b5448452c645f150a56407 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +# ChromaDB +rag/chroma_db/ +memory/mem0_store/chroma_db/ + +# SQLite +*.sqlite3 +*.bin + +# Python +__pycache__/ +.venv/ +.env \ No newline at end of file diff --git a/agents/__init__.py b/agents/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/agents/comparison.py b/agents/comparison.py new file mode 100644 index 0000000000000000000000000000000000000000..17e90af52ec08c068743f30ffa30b674a23e605c --- /dev/null +++ b/agents/comparison.py @@ -0,0 +1,47 @@ +""" +Comparison agent. + +Handles queries like "difference between BFS and DFS" — retrieves chunks +per-topic (rather than pooled) so the prompt builder can present each +topic's relevant sections side by side. +""" + +from logs.logger import get_logger + +logger = get_logger(__name__) + + +def gather_comparison_context(topics: list, retriever, top_k_per_topic: int = None) -> dict: + """ + Args: + topics: canonical topic names, e.g. ["Breadth-First Search", "Depth-First Search"]. + retriever: a rag.retriever.Retriever instance. + top_k_per_topic: chunks to retrieve per topic (defaults to retriever/config default). + + Returns: + { + "Breadth-First Search": [chunk, chunk, ...], + "Depth-First Search": [chunk, chunk, ...], + } + A topic with no chunks clearing the similarity threshold maps to []. + """ + context = retriever.retrieve_topics(topics, top_k_per_topic=top_k_per_topic) + + for topic, chunks in context.items(): + if not chunks: + logger.warning( + "Comparison agent: no chunks passed the similarity threshold for topic %r", + topic, + ) + + return context + + +def has_sufficient_context(comparison_context: dict) -> bool: + """ + True only if every topic in the comparison has at least one retrieved + chunk. If any topic comes back empty, the caller should fall back to + the "insufficient information" response rather than comparing a topic + it knows nothing about against one it does. + """ + return all(len(chunks) > 0 for chunks in comparison_context.values()) diff --git a/agents/followup.py b/agents/followup.py new file mode 100644 index 0000000000000000000000000000000000000000..49c82f35d36b0234de682a2f1357e5d683b5dce6 --- /dev/null +++ b/agents/followup.py @@ -0,0 +1,68 @@ +""" +Follow-up agent. + +Handles queries like "can you explain that again?" or "what about the +space complexity?" that only make sense in light of recent conversation. + +Strategy: reuse recent conversation history as context by default, and +only fall back to a fresh embedding-based retrieval if the follow-up +mentions a topic that wasn't part of the recent conversation (e.g. user +pivots to a new topic with a pronoun-only phrasing). +""" + +from agents.intent_classifier import extract_topics,topics_in_recent_history +from logs.logger import get_logger + +logger = get_logger(__name__) + +def gather_followup_context( + query: str, + recent_messages: list, + retriever, + top_k: int = None, +) -> dict: + """ + Args: + query: the current follow-up question. + recent_messages: recent conversation turns, oldest -> newest + (see history/service.py for how these are fetched; + bounded by config.MAX_RECENT_HISTORY_MESSAGES). + retriever: a rag.retriever.Retriever instance. + top_k: chunks to retrieve if a fresh lookup is needed. + + Returns: + { + "reused_topics": [...], # topics inferred from history + "additional_chunks": [...], # only populated if a new topic + # was mentioned in `query` itself + "recent_messages": recent_messages, + } + """ + reused_topics = topics_in_recent_history(recent_messages) + + # Does the follow-up query itself introduce a topic not seen recently? + query_topics = extract_topics(query) + new_topics = [t for t in query_topics if t not in reused_topics] + + additional_chunks = [] + if new_topics: + logger.info( + "Follow-up query introduces new topic(s) not in recent history: %s", + new_topics, + ) + for topic in new_topics: + additional_chunks.extend( + retriever.retrieve(query=query, top_k=top_k, topic_filter=[topic]) + ) + else: + logger.info( + "Follow-up query reuses existing conversation context (topics=%s); " + "skipping fresh retrieval.", + reused_topics, + ) + + return { + "reused_topics": reused_topics, + "additional_chunks": additional_chunks, + "recent_messages": recent_messages, + } diff --git a/agents/intent_classifier.py b/agents/intent_classifier.py new file mode 100644 index 0000000000000000000000000000000000000000..69cdd5ea45d301838dc633954fabf95823969059 --- /dev/null +++ b/agents/intent_classifier.py @@ -0,0 +1,208 @@ +""" +Intent classification helpers used by the agent router. + +This module is intentionally rule-based (regex + fuzzy topic-name matching) +rather than LLM-based, since the LLM provider abstraction (llm/) hasn't +been wired in yet. It's isolated behind plain functions so it can be +swapped for an LLM-based classifier later without changing router.py's +interface — router.py only calls extract_topics(), is_comparison_query(), +and is_followup_query(). + +No keyword-based *retrieval* happens here — this only decides ROUTING. +Actual knowledge retrieval is always embedding-based (see rag/retriever.py). +""" + +import difflib +import json +import re + +import config + +# --------------------------------------------------------------------------- +# Topic vocabulary (loaded once from topic_metadata.json) +# --------------------------------------------------------------------------- + +_FUZZY_MATCH_CUTOFF = 0.85 + + +def _load_known_topics() -> list: + """Returns the list of canonical topic names, e.g. ['Binary Search', ...].""" + with open(config.TOPIC_METADATA_PATH, "r", encoding="utf-8") as f: + metadata = json.load(f) + return sorted({entry["topic"] for entry in metadata.values() if entry.get("topic")}) + + +_KNOWN_TOPICS = None + + +def get_known_topics() -> list: + global _KNOWN_TOPICS + if _KNOWN_TOPICS is None: + _KNOWN_TOPICS = _load_known_topics() + return _KNOWN_TOPICS + + +def refresh_known_topics() -> None: + """Call after topic_metadata.json changes (e.g. after a re-index) to + force reloading the topic vocabulary on next use.""" + global _KNOWN_TOPICS + _KNOWN_TOPICS = None + + +# --------------------------------------------------------------------------- +# Comparison detection +# --------------------------------------------------------------------------- + +_COMPARISON_PATTERNS = [ + re.compile(r"\bdifference between\b", re.IGNORECASE), + re.compile(r"\bvs\.?\b", re.IGNORECASE), + re.compile(r"\bversus\b", re.IGNORECASE), + re.compile(r"\bcompare(d|s)?\b", re.IGNORECASE), + re.compile(r"\bwhich (one )?is (better|faster|more efficient)\b", re.IGNORECASE), + re.compile(r"\bpros and cons of\b", re.IGNORECASE), +] + + +def is_comparison_query(query: str) -> bool: + return any(p.search(query) for p in _COMPARISON_PATTERNS) + + +# --------------------------------------------------------------------------- +# Follow-up detection +# --------------------------------------------------------------------------- + +_FOLLOWUP_PATTERNS = [ + re.compile(r"^\s*(explain|say|show)?\s*(that|this|it)\b.*again\b", re.IGNORECASE), + re.compile(r"^\s*what (do you mean|about (that|it|this))\b", re.IGNORECASE), + re.compile(r"^\s*(can you|could you)\s+(clarify|elaborate|expand)\b", re.IGNORECASE), + re.compile(r"^\s*(and|so)\b", re.IGNORECASE), + re.compile(r"^\s*(why|how) (is that|does that work)\b", re.IGNORECASE), + re.compile(r"\b(more|another) example\b", re.IGNORECASE), + re.compile(r"^\s*(simpler|simplify|dumb it down|eli5)\b", re.IGNORECASE), +] + +# Bare pronoun references with no topic name mentioned, e.g. "explain it", +# "why is that", "tell me more" +_PRONOUN_ONLY_PATTERN = re.compile( + r"\b(it|that|this|those|these)\b", re.IGNORECASE +) + +_WORKED_EXAMPLE_PATTERN = re.compile( + r"\d+(?:\s*,\s*\d+){1,}.*\b(this|that|same)\b.*\b(search|sort|algorithm|method|approach|traversal)\b", + re.IGNORECASE, +) + +def is_followup_query(query: str, has_conversation_history: bool) -> bool: + """ + A query is treated as a follow-up when: + 1. There IS prior conversation to follow up on, AND + 2. Either it matches a known follow-up phrasing pattern, OR + it's short, contains a bare pronoun reference, and mentions no + known topic by name (i.e. it can't stand on its own). + """ + if not has_conversation_history: + return False + + if any(p.search(query) for p in _FOLLOWUP_PATTERNS): + return True + + if _WORKED_EXAMPLE_PATTERN.search(query): + return True + + word_count = len(query.strip().split()) + has_pronoun = bool(_PRONOUN_ONLY_PATTERN.search(query)) + mentions_topic = bool(extract_topics(query)) + + if has_pronoun and not mentions_topic and word_count <= 8: + return True + + return False + +_GENERIC_COMPARISON_PATTERN = re.compile( + r"\b(compare|comparison|vs\.?|versus|difference|different from|how does .* (differ|compare))\b.*" + r"\b(other|similar|related|different|alternative)\b", + re.IGNORECASE, +) + + +def is_generic_comparison_followup(query: str) -> bool: + """ + Matches comparison phrasing that references *other* algorithms without + naming a second one explicitly, e.g. "compare with other graph + algorithms", "how does this differ from similar approaches". + """ + return bool(_GENERIC_COMPARISON_PATTERN.search(query)) + +def topics_in_recent_history(recent_messages: list) -> list: + """ + Scans recent conversation turns (both user and assistant messages) for + known topic mentions. Order preserved, most-recently-mentioned last. + """ + topics = [] + for message in recent_messages: + for topic in extract_topics(message.get("content", "")): + if topic not in topics: + topics.append(topic) + else: + topics.remove(topic) + topics.append(topic) # bump to most-recent position + return topics + + +# --------------------------------------------------------------------------- +# Topic extraction (fuzzy match against known topic names) +# --------------------------------------------------------------------------- + +def extract_topics(query: str) -> list: + """ + Returns the list of known topic names mentioned in the query, matched + via substring + fuzzy matching (handles typos/casing/partial names like + "binary search" or "BFS traversal"). + + This is NOT used for retrieval (retrieval stays purely embedding-based) + — it's only used for ROUTING decisions: deciding single-topic vs. + comparison, and picking which topics a comparison should retrieve. + """ + query_lower = query.lower() + found = [] + + for topic in get_known_topics(): + topic_lower = topic.lower() + # Strip a trailing parenthetical, e.g. "Kruskal's Algorithm (MST)" + # -> "kruskal's algorithm", since people ask using the plain name, + # not the parenthetical qualifier. + topic_core = re.sub(r"\s*\([^)]*\)\s*$", "", topic_lower).strip() + + # Direct substring match (full name or core name without qualifier) + if topic_lower in query_lower or (topic_core and topic_core in query_lower): + found.append(topic) + continue + + # Acronym match, e.g. "Depth-First Search" -> "DFS", "Breadth-First + # Search" -> "BFS". Common in how people actually ask DSA questions. + acronym = "".join(w[0] for w in re.split(r"[\s\-]+", topic) if w).lower() + if len(acronym) >= 3 and re.search(rf"\b{re.escape(acronym)}\b", query_lower): + found.append(topic) + continue + + # Fuzzy match against each sliding n-gram window of the query, + # sized to the topic's word count, to catch typos/near-matches. + topic_words = topic_lower.split() + query_words = query_lower.split() + window = len(topic_words) + for i in range(len(query_words) - window + 1): + candidate = " ".join(query_words[i : i + window]) + ratio = difflib.SequenceMatcher(None, candidate, topic_lower).ratio() + if ratio >= _FUZZY_MATCH_CUTOFF: + found.append(topic) + break + + # De-duplicate while preserving order + seen = set() + deduped = [] + for t in found: + if t not in seen: + seen.add(t) + deduped.append(t) + return deduped + diff --git a/agents/router.py b/agents/router.py new file mode 100644 index 0000000000000000000000000000000000000000..15cf85d72d53f61ebde878a63f129b6158fba5b3 --- /dev/null +++ b/agents/router.py @@ -0,0 +1,173 @@ +""" +Agent router. + +Decides how each incoming query should be processed and gathers the +context needed for the corresponding prompt builder (llm/prompts.py, +built in a later layer). This module does NOT call the LLM — it only +classifies + retrieves. + +Routes: + SINGLE_TOPIC - e.g. "Explain Merge Sort" -> retrieve one topic + COMPARISON - e.g. "Difference between BFS/DFS" -> retrieve multiple topics + FOLLOWUP - e.g. "Can you explain that again?" -> reuse conversation context + OUT_OF_SCOPE - e.g. "Who won the World Cup?" -> reject politely +""" + +from dataclasses import dataclass, field +from enum import Enum + +from agents.comparison import gather_comparison_context, has_sufficient_context +from agents.followup import gather_followup_context +from agents.intent_classifier import ( + extract_topics, + is_comparison_query, + is_followup_query, + is_generic_comparison_followup, + topics_in_recent_history, +) +from agents.topic_categories import get_sibling_topics +from logs.logger import get_logger + +logger = get_logger(__name__) + + +class RouteType(str, Enum): + SINGLE_TOPIC = "single_topic" + COMPARISON = "comparison" + FOLLOWUP = "followup" + OUT_OF_SCOPE = "out_of_scope" + + +@dataclass +class RouteDecision: + route_type: RouteType + query: str + topics: list = field(default_factory=list) + # Populated by gather_context() below, per route type: + single_chunks: list = field(default_factory=list) + comparison_context: dict = field(default_factory=dict) + followup_context: dict = field(default_factory=dict) + + def to_log_dict(self) -> dict: + return { + "route_type": self.route_type.value, + "query": self.query, + "topics": self.topics, + } + + +def classify(query: str, has_conversation_history: bool = False, recent_topics: list = None) -> RouteDecision: + """ + Pure classification step (no retrieval yet). Order matters: + 1. Follow-up check first — a short pronoun-based query with existing + history should be treated as a follow-up even if it superficially + contains comparison words. + 2. Comparison check — requires comparison phrasing AND 2+ topics. + 3. Single topic — exactly one recognized topic mentioned. + 4. Otherwise — out of scope. + """ + recent_topics = recent_topics or [] + + if is_followup_query(query, has_conversation_history): + decision = RouteDecision(route_type=RouteType.FOLLOWUP, query=query) + logger.info("Routed to FOLLOWUP: %r", query) + return decision + + topics = extract_topics(query) + + if is_comparison_query(query) and len(topics) >= 2: + decision = RouteDecision(route_type=RouteType.COMPARISON, query=query, topics=topics) + logger.info("Routed to COMPARISON: %r | topics=%s", query, topics) + return decision + + # NEW: comparison referencing the previous topic implicitly, e.g. + # "compare with other graph algorithms" after discussing BFS. + if is_generic_comparison_followup(query) and recent_topics: + anchor_topic = recent_topics[-1] + siblings = get_sibling_topics(anchor_topic, exclude=topics, limit=2) + combined = list(dict.fromkeys(topics + [anchor_topic] + siblings)) + if len(combined) >= 2: + decision = RouteDecision(route_type=RouteType.COMPARISON, query=query, topics=combined) + logger.info( + "Routed to COMPARISON (category follow-up): %r | anchor=%s | topics=%s", + query, anchor_topic, combined, + ) + return decision + + if len(topics) >= 1: + decision = RouteDecision(route_type=RouteType.SINGLE_TOPIC, query=query, topics=topics[:1]) + logger.info("Routed to SINGLE_TOPIC: %r | topic=%s", query, topics[0]) + return decision + + decision = RouteDecision(route_type=RouteType.OUT_OF_SCOPE, query=query) + logger.info("Routed to OUT_OF_SCOPE: %r", query) + return decision + + +def gather_context( + decision: RouteDecision, + retriever, + recent_messages: list = None, +) -> RouteDecision: + """ + Populates the appropriate context field on `decision` based on its + route_type. Mutates and returns the same decision object. + + Args: + decision: output of classify(). + retriever: a rag.retriever.Retriever instance. + recent_messages: required for FOLLOWUP routes — recent conversation + turns, oldest -> newest. + """ + recent_messages = recent_messages or [] + + if decision.route_type == RouteType.SINGLE_TOPIC: + topic = decision.topics[0] if decision.topics else None + decision.single_chunks = retriever.retrieve( + query=decision.query, + topic_filter=[topic] if topic else None, + bypass_threshold_if_single_topic=True, + ) + if not decision.single_chunks: + logger.warning( + "SINGLE_TOPIC query had no chunks pass the similarity " + "threshold: %r", + decision.query, + ) + + elif decision.route_type == RouteType.COMPARISON: + decision.comparison_context = gather_comparison_context(decision.topics, retriever) + if not has_sufficient_context(decision.comparison_context): + logger.warning( + "COMPARISON query missing sufficient context for one or " + "more topics: %s", + decision.topics, + ) + + elif decision.route_type == RouteType.FOLLOWUP: + decision.followup_context = gather_followup_context( + query=decision.query, + recent_messages=recent_messages, + retriever=retriever, + ) + + # OUT_OF_SCOPE needs no retrieval. + + return decision + +def route(query: str, retriever, recent_messages: list = None) -> RouteDecision: + """ + Convenience entrypoint combining classify() + gather_context(). + + recent_messages: recent conversation turns (oldest -> newest). Presence + of any messages here signals "has_conversation_history=True" for + follow-up detection. + """ + recent_messages = recent_messages or [] + recent_topics = topics_in_recent_history(recent_messages) + decision = classify( + query, + has_conversation_history=len(recent_messages) > 0, + recent_topics=recent_topics, + ) + return gather_context(decision, retriever, recent_messages=recent_messages) \ No newline at end of file diff --git a/agents/topic_categories.py b/agents/topic_categories.py new file mode 100644 index 0000000000000000000000000000000000000000..1a788a42c4ac984cc40884c7c9fb6c048389fceb --- /dev/null +++ b/agents/topic_categories.py @@ -0,0 +1,56 @@ +""" +Category lookups for topic_metadata.json. + +Used by the router to resolve "compare this with other X" style queries +that reference the previous topic implicitly, without naming a second +topic explicitly (e.g. "compare BFS with other graph algorithms"). +""" + +import json +import os + +import config +from logs.logger import get_logger + +logger = get_logger(__name__) + +_metadata_cache = None + + +def _load_metadata() -> dict: + global _metadata_cache + if _metadata_cache is None: + if not os.path.exists(config.TOPIC_METADATA_PATH): + logger.warning("topic_metadata.json not found; category lookups disabled.") + _metadata_cache = {} + else: + with open(config.TOPIC_METADATA_PATH, "r", encoding="utf-8") as f: + _metadata_cache = json.load(f) + return _metadata_cache + + +def get_category(topic_name: str) -> str | None: + """Returns the category string for a canonical topic name, or None.""" + for entry in _load_metadata().values(): + if entry.get("topic") == topic_name: + return entry.get("category") + return None + + +def get_sibling_topics(topic_name: str, exclude: list = None, limit: int = 2) -> list: + """ + Returns up to `limit` other topic names sharing the same category as + `topic_name` (e.g. siblings of "Breadth-First Search" in category + "Graph Traversal" -> ["Depth-First Search", "Dijkstra's Algorithm"]). + """ + exclude = set(exclude or []) | {topic_name} + category = get_category(topic_name) + if not category: + return [] + + siblings = [ + entry["topic"] + for entry in _load_metadata().values() + if entry.get("category") == category and entry.get("topic") not in exclude + ] + return siblings[:limit] \ No newline at end of file diff --git a/app.py b/app.py new file mode 100644 index 0000000000000000000000000000000000000000..d1b5be912e9c83bedbdafb5351ad38c984df7ae7 --- /dev/null +++ b/app.py @@ -0,0 +1,94 @@ +""" +Flask application entrypoint. + +At this stage of the build (Layer 1), only the database and authentication +are wired up so this layer can be tested in isolation. Chat/RAG/agent +routes will be added in later layers. +""" +# http://127.0.0.1:7860/ + + +import os +import threading + +# Some corporate proxies / VPNs set HTTP_PROXY / HTTPS_PROXY, which makes +# Gradio's internal "is localhost reachable" check fail even for 127.0.0.1. +os.environ["NO_PROXY"] = "127.0.0.1,localhost," + os.environ.get("NO_PROXY", "") +os.environ["no_proxy"] = os.environ["NO_PROXY"] + +# --- Workaround for a gradio_client bug (gradio 4.44.1 + newer pydantic) --- +# When a schema (or a nested value like `additionalProperties`) is a plain +# bool (True/False) rather than a dict, gradio_client's schema walker either +# crashes with "TypeError: argument of type 'bool' is not iterable" or raises +# "APIInfoParseError: Cannot parse schema True". This happens while Gradio +# builds the page's embedded API info, unconditionally, on every request to +# "/" - so it can't be avoided via show_api=False. Patch both entry points. +import gradio_client.utils as _gc_utils + +_orig_get_type = _gc_utils.get_type +_orig_json_schema_to_python_type = _gc_utils._json_schema_to_python_type + + +def _safe_get_type(schema): + if isinstance(schema, bool): + return "Any" + return _orig_get_type(schema) + + +def _safe_json_schema_to_python_type(schema, defs=None): + if isinstance(schema, bool): + return "Any" + return _orig_json_schema_to_python_type(schema, defs) + + +_gc_utils.get_type = _safe_get_type +_gc_utils._json_schema_to_python_type = _safe_json_schema_to_python_type +# --- end workaround --- + +from flask import Flask + +import config +from auth.db import init_db +from auth.routes import auth_bp +from chat_routes import chat_bp +from ui.layout import build_ui + + +def create_app(): + app = Flask(__name__) + + app.config["SECRET_KEY"] = config.SECRET_KEY + app.config["SQLALCHEMY_DATABASE_URI"] = config.SQLALCHEMY_DATABASE_URI + app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = config.SQLALCHEMY_TRACK_MODIFICATIONS + + init_db(app) + + app.register_blueprint(auth_bp) + app.register_blueprint(chat_bp) + + @app.route("/health") + def health(): + return {"status": "ok"} + + return app + + +if __name__ == "__main__": + flask_app = create_app() + + # Run the Flask REST API (auth + chat endpoints) on a background thread. + api_thread = threading.Thread( + target=lambda: flask_app.run(port=5000, debug=False, use_reloader=False), + daemon=True, + ) + api_thread.start() + + # Gradio UI is the main process. Its callbacks call into Flask + # in-process via app_context() (see ui/callbacks.py) rather than HTTP. + demo = build_ui(flask_app) + demo.queue().launch( + server_name="0.0.0.0", + server_port=7860, + show_api=False, + inbrowser=False, + ) \ No newline at end of file diff --git a/auth/__init__.py b/auth/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/auth/db.py b/auth/db.py new file mode 100644 index 0000000000000000000000000000000000000000..67019bdb2702b02b3fd238efb9c89cf12caf9603 --- /dev/null +++ b/auth/db.py @@ -0,0 +1,22 @@ +""" +Shared SQLAlchemy instance. + +Kept in its own module (rather than inside models.py or app.py) so that +every part of the application — auth, history, etc. — can import the same +`db` object without circular imports. +""" + +from flask_sqlalchemy import SQLAlchemy + +db = SQLAlchemy() + + +def init_db(app): + """Bind SQLAlchemy to the Flask app and create tables if they don't exist.""" + db.init_app(app) + with app.app_context(): + # Import models here so they're registered on db.metadata before create_all() + from auth.models import User # noqa: F401 + from history.models import ChatSession, ChatMessage # noqa: F401 + + db.create_all() diff --git a/auth/models.py b/auth/models.py new file mode 100644 index 0000000000000000000000000000000000000000..ab986d7aa50f00c4b992f807c43abab5a029216f --- /dev/null +++ b/auth/models.py @@ -0,0 +1,41 @@ +""" +User model for email/password authentication. + +No OAuth. No plaintext passwords — only Werkzeug password hashes are stored. +""" + +from datetime import datetime + +from werkzeug.security import generate_password_hash, check_password_hash + +from auth.db import db + + +class User(db.Model): + __tablename__ = "users" + + id = db.Column(db.Integer, primary_key=True) + email = db.Column(db.String(255), unique=True, nullable=False, index=True) + password_hash = db.Column(db.String(255), nullable=False) + created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + + # Relationship to chat sessions (defined in history/models.py) + chat_sessions = db.relationship( + "ChatSession", backref="user", lazy=True, cascade="all, delete-orphan" + ) + + def set_password(self, raw_password: str) -> None: + self.password_hash = generate_password_hash(raw_password) + + def check_password(self, raw_password: str) -> bool: + return check_password_hash(self.password_hash, raw_password) + + def to_dict(self) -> dict: + return { + "id": self.id, + "email": self.email, + "created_at": self.created_at.isoformat(), + } + + def __repr__(self) -> str: + return f"" diff --git a/auth/routes.py b/auth/routes.py new file mode 100644 index 0000000000000000000000000000000000000000..d1d2adc33a4dab5a9feeda50cdda425e7fe054f3 --- /dev/null +++ b/auth/routes.py @@ -0,0 +1,87 @@ +""" +Authentication REST endpoints. + +Routes: + POST /signup -> create a new user account + POST /login -> start a session-cookie-based login + POST /logout -> clear the session + GET /me -> return the currently authenticated user, if any + +Session-based auth only: Flask's signed session cookie stores the user id. +No tokens, no OAuth. +""" + +import re + +from flask import Blueprint, request, jsonify, session + +from auth.db import db +from auth.models import User + +auth_bp = Blueprint("auth", __name__) + +EMAIL_REGEX = re.compile(r"^[^@\s]+@[^@\s]+\.[^@\s]+$") +MIN_PASSWORD_LENGTH = 8 + + +def _current_user(): + user_id = session.get("user_id") + if user_id is None: + return None + return User.query.get(user_id) + + +@auth_bp.route("/signup", methods=["POST"]) +def signup(): + data = request.get_json(silent=True) or {} + email = (data.get("email") or "").strip().lower() + password = data.get("password") or "" + + if not email or not EMAIL_REGEX.match(email): + return jsonify({"error": "A valid email is required."}), 400 + + if len(password) < MIN_PASSWORD_LENGTH: + return jsonify({ + "error": f"Password must be at least {MIN_PASSWORD_LENGTH} characters." + }), 400 + + if User.query.filter_by(email=email).first() is not None: + return jsonify({"error": "An account with this email already exists."}), 409 + + user = User(email=email) + user.set_password(password) + db.session.add(user) + db.session.commit() + + session["user_id"] = user.id + + return jsonify({"message": "Account created.", "user": user.to_dict()}), 201 + + +@auth_bp.route("/login", methods=["POST"]) +def login(): + data = request.get_json(silent=True) or {} + email = (data.get("email") or "").strip().lower() + password = data.get("password") or "" + + user = User.query.filter_by(email=email).first() + if user is None or not user.check_password(password): + return jsonify({"error": "Invalid email or password."}), 401 + + session["user_id"] = user.id + + return jsonify({"message": "Logged in.", "user": user.to_dict()}), 200 + + +@auth_bp.route("/logout", methods=["POST"]) +def logout(): + session.pop("user_id", None) + return jsonify({"message": "Logged out."}), 200 + + +@auth_bp.route("/me", methods=["GET"]) +def me(): + user = _current_user() + if user is None: + return jsonify({"user": None}), 200 + return jsonify({"user": user.to_dict()}), 200 diff --git a/chat_routes.py b/chat_routes.py new file mode 100644 index 0000000000000000000000000000000000000000..eb3d6a1a53d71684de0fe6a85e2a34d3e5b2841c --- /dev/null +++ b/chat_routes.py @@ -0,0 +1,82 @@ +""" +Chat + session-history REST endpoints. + +Routes: + POST /chat -> send a message, get a response + GET /sessions -> list the logged-in user's chat sessions + GET /sessions/ -> full transcript of one session + DELETE /sessions/ -> delete a session + +Anonymous users can POST /chat (no persistence, no session panel per spec) +but cannot access /sessions endpoints. +""" + +from flask import Blueprint, jsonify, request, session + +from auth.models import User +from chat_service import handle_chat_message +from history import service as history_service + +chat_bp = Blueprint("chat", __name__) + + +def _current_user(): + user_id = session.get("user_id") + if user_id is None: + return None + return User.query.get(user_id) + + +@chat_bp.route("/chat", methods=["POST"]) +def chat(): + data = request.get_json(silent=True) or {} + query = (data.get("query") or "").strip() + + if not query: + return jsonify({"error": "query is required."}), 400 + + user = _current_user() + session_id = data.get("session_id") + anonymous_history = data.get("history") if user is None else None + + result = handle_chat_message( + query=query, + user=user, + session_id=session_id, + anonymous_history=anonymous_history, + ) + return jsonify(result), 200 + + +@chat_bp.route("/sessions", methods=["GET"]) +def list_sessions(): + user = _current_user() + if user is None: + return jsonify({"error": "Login required."}), 401 + return jsonify({"sessions": history_service.list_sessions(user.id)}), 200 + + +@chat_bp.route("/sessions/", methods=["GET"]) +def get_session_route(session_id): + user = _current_user() + if user is None: + return jsonify({"error": "Login required."}), 401 + + full_session = history_service.get_full_session(session_id, user.id) + if full_session is None: + return jsonify({"error": "Session not found."}), 404 + + return jsonify({"session": full_session}), 200 + + +@chat_bp.route("/sessions/", methods=["DELETE"]) +def delete_session_route(session_id): + user = _current_user() + if user is None: + return jsonify({"error": "Login required."}), 401 + + deleted = history_service.delete_session(session_id, user.id) + if not deleted: + return jsonify({"error": "Session not found."}), 404 + + return jsonify({"message": "Session deleted."}), 200 diff --git a/chat_service.py b/chat_service.py new file mode 100644 index 0000000000000000000000000000000000000000..a2e658adcd9fbc3e682b9f35011b8e5f9ad44dee --- /dev/null +++ b/chat_service.py @@ -0,0 +1,126 @@ +""" +Chat orchestration service. + +This is the single place that wires together everything built in the +previous layers for one chat turn: + + agents.router -> classify + retrieve context + llm.prompts -> build the right prompt for the route + llm.generate -> call the configured LLM provider + memory.mem0_client -> pull/store long-term preference memory (logged-in only) + history.service -> persist the transcript (logged-in only) + +Kept separate from app.py so Flask routes stay thin, and so the same logic +can be reused by the Gradio UI layer later without going through HTTP. +""" + +import config +from agents.router import RouteType, route +from history import service as history_service +from history.session_manager import resolve_session +from llm.generate import generate +from llm.prompts import ( + build_comparison_prompt, + build_followup_prompt, + build_reject_prompt, + build_single_prompt, +) +from logs.logger import get_logger +from memory.mem0_client import get_mem0_client +from rag.retriever import get_retriever + +logger = get_logger(__name__) + + +def _build_prompt(decision, mem0_context, recent_messages): + if decision.route_type == RouteType.SINGLE_TOPIC: + return build_single_prompt( + decision.single_chunks, mem0_context, recent_messages, decision.query + ) + if decision.route_type == RouteType.COMPARISON: + return build_comparison_prompt( + decision.comparison_context, mem0_context, recent_messages, decision.query + ) + if decision.route_type == RouteType.FOLLOWUP: + return build_followup_prompt( + decision.followup_context, mem0_context, recent_messages, decision.query + ) + # OUT_OF_SCOPE + return build_reject_prompt(decision.query) + + +def handle_chat_message( + query: str, + user=None, + session_id: int = None, + anonymous_history: list = None, +) -> dict: + """ + Args: + query: the user's message. + user: the authenticated auth.models.User, or None if anonymous. + session_id: existing chat session id (logged-in users only, optional). + anonymous_history: client-managed recent history for anonymous users + (no server-side persistence — see spec: "when + not logged in, chat works normally, no + persistence, no history panel"). Ignored for + logged-in users, whose history is read from + SQLite instead. + + Returns: + { + "response": "...", + "route_type": "single_topic" | "comparison" | "followup" | "out_of_scope", + "topics": [...], + "session_id": int | None, # None for anonymous users + } + """ + is_logged_in = user is not None + session_obj = None + + if is_logged_in: + session_obj = resolve_session(user.id, session_id, first_message=query) + recent_messages = history_service.get_recent_messages(session_obj.id) + else: + recent_messages = (anonymous_history or [])[-config.MAX_RECENT_HISTORY_MESSAGES :] + + retriever = get_retriever() + decision = route(query, retriever, recent_messages=recent_messages) + + mem0_context = [] + if is_logged_in: + mem0_context = get_mem0_client().get_relevant_context(str(user.id), query) + + prompt = _build_prompt(decision, mem0_context, recent_messages) + response_text = generate(prompt) + + if is_logged_in: + history_service.add_message( + session_obj.id, + role="user", + content=query, + route_type=decision.route_type.value, + retrieved_topics=decision.topics, + ) + history_service.add_message( + session_obj.id, + role="assistant", + content=response_text, + route_type=decision.route_type.value, + ) + get_mem0_client().add_interaction(str(user.id), query, response_text) + + logger.info( + "Chat turn complete | user_id=%s | route=%s | topics=%s | session_id=%s", + user.id if is_logged_in else None, + decision.route_type.value, + decision.topics, + session_obj.id if session_obj else None, + ) + + return { + "response": response_text, + "route_type": decision.route_type.value, + "topics": decision.topics, + "session_id": session_obj.id if session_obj else None, + } diff --git a/config.py b/config.py new file mode 100644 index 0000000000000000000000000000000000000000..9b1045f8b3b4d410555fe656861b75b557f85692 --- /dev/null +++ b/config.py @@ -0,0 +1,90 @@ +""" +Central configuration for the DSA RAG Chatbot. +No magic numbers should be hardcoded anywhere else in the codebase — import from here. +""" + +import os +from dotenv import load_dotenv + +load_dotenv() + +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) + +# --------------------------------------------------------------------------- +# Flask / App +# --------------------------------------------------------------------------- +SECRET_KEY = os.getenv("SECRET_KEY", "dev-secret-key-change-me") +SESSION_COOKIE_NAME = "dsa_chatbot_session" +SESSION_LIFETIME_MINUTES = int(os.getenv("SESSION_LIFETIME_MINUTES", 60 * 24 * 7)) # 7 days + +# --------------------------------------------------------------------------- +# Database (SQLite via SQLAlchemy) +# --------------------------------------------------------------------------- +DATABASE_PATH = os.path.join(BASE_DIR, "database", "app.db") +SQLALCHEMY_DATABASE_URI = f"sqlite:///{DATABASE_PATH}" +SQLALCHEMY_TRACK_MODIFICATIONS = False + +# --------------------------------------------------------------------------- +# RAG / Knowledge base +# --------------------------------------------------------------------------- +KNOWLEDGE_BASE_DIR = os.path.join(BASE_DIR, "rag", "knowledge_base") +TOPIC_METADATA_PATH = os.path.join(BASE_DIR, "rag", "knowledge_base", "topic_metadata.json") +CHROMA_DB_DIR = os.path.join(BASE_DIR, "rag", "chroma_db") +CHROMA_COLLECTION_NAME = "dsa_knowledge_base" + +EMBEDDING_MODEL_NAME = "all-MiniLM-L6-v2" + +# Sections expected inside each knowledge-base .txt file. Used by the chunker +# to split on structure rather than arbitrary fixed-size windows. +KB_SECTION_HEADERS = [ + "DEFINITION", + "TIME_COMPLEXITY", + "SPACE_COMPLEXITY", + "USE_WHEN", + "EXAMPLE", +] + +# --------------------------------------------------------------------------- +# Retrieval pipeline +# --------------------------------------------------------------------------- +INITIAL_RETRIEVAL_K = int(os.getenv("INITIAL_RETRIEVAL_K", 15)) +TOP_K = int(os.getenv("TOP_K", 5)) +SIMILARITY_THRESHOLD = float(os.getenv("SIMILARITY_THRESHOLD", 0.45)) + +# --------------------------------------------------------------------------- +# Memory (mem0) +# --------------------------------------------------------------------------- +USE_MEM0 = os.getenv("USE_MEM0", "True") == "True" +MEM0_LOCAL_STORAGE_DIR = os.path.join(BASE_DIR, "memory", "mem0_store") + +# --------------------------------------------------------------------------- +# History / conversation +# --------------------------------------------------------------------------- +MAX_RECENT_HISTORY_MESSAGES = int(os.getenv("MAX_RECENT_HISTORY_MESSAGES", 6)) + +# --------------------------------------------------------------------------- +# LLM Provider +# --------------------------------------------------------------------------- +# "local" -> Mistral-7B-Instruct-v0.2 via transformers/bitsandbytes +# "groq" -> future Groq-hosted provider +# "openai"-> future OpenAI provider +# LLM_PROVIDER = os.getenv("LLM_PROVIDER", "local") +LLM_PROVIDER = os.getenv("LLM_PROVIDER", "groq") + +LOCAL_MODEL_NAME = os.getenv("LOCAL_MODEL_NAME", "mistralai/Mistral-7B-Instruct-v0.2") +LOCAL_MODEL_MAX_NEW_TOKENS = int(os.getenv("LOCAL_MODEL_MAX_NEW_TOKENS", 512)) +LOCAL_MODEL_TEMPERATURE = float(os.getenv("LOCAL_MODEL_TEMPERATURE", 0.3)) +LOCAL_MODEL_LOAD_IN_4BIT = os.getenv("LOCAL_MODEL_LOAD_IN_4BIT", "True") == "True" + +GROQ_API_KEY = os.getenv("GROQ_API_KEY") +GROQ_MODEL_NAME = os.getenv("GROQ_MODEL_NAME", "llama-3.3-70b-versatile") + +OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "") +OPENAI_MODEL_NAME = os.getenv("OPENAI_MODEL_NAME", "") + +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- +LOG_DIR = os.path.join(BASE_DIR, "logs") +LOG_FILE_PATH = os.path.join(LOG_DIR, "app.log") +LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO") diff --git a/database/__init__.py b/database/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/database/app.db b/database/app.db new file mode 100644 index 0000000000000000000000000000000000000000..1ebcc98a7675b82109296e750fa5cd65e42b3d19 Binary files /dev/null and b/database/app.db differ diff --git a/evaluation/__init__.py b/evaluation/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/evaluation/evaluate.py b/evaluation/evaluate.py new file mode 100644 index 0000000000000000000000000000000000000000..c38011c83c971ec7ea013ab0900167921f57770a --- /dev/null +++ b/evaluation/evaluate.py @@ -0,0 +1,222 @@ +""" +Evaluation harness for the DSA RAG chatbot. + +Runs the predefined question set (evaluation/questions.json) through the +agent router + retriever (and optionally the full LLM pipeline) and reports: + + - routing accuracy (predicted route_type vs expected_route) + - retrieval topic recall (expected topics found among retrieved chunks) + - retrieval latency (avg / p95, ms) + - generation latency (avg / p95, ms) -- only with --with-llm + +Usage: + python -m evaluation.evaluate + python -m evaluation.evaluate --with-llm + python -m evaluation.evaluate --questions evaluation/questions.json + +Run from the project root (dsa-rag-chatbot/) so `config` and the app +packages resolve correctly. +""" + +import argparse +import json +import os +import statistics +import time + +import config +from agents.followup import gather_followup_context +from agents.router import RouteType, classify, gather_context +from logs.logger import get_logger + +logger = get_logger(__name__) + +DEFAULT_QUESTIONS_PATH = os.path.join(os.path.dirname(__file__), "questions.json") + +# A tiny bit of fake prior conversation so the follow-up route can actually +# be exercised (is_followup_query requires has_conversation_history=True). +_FAKE_HISTORY_FOR_FOLLOWUPS = [ + {"role": "user", "content": "Explain merge sort"}, + {"role": "assistant", "content": "Merge sort is a divide-and-conquer sorting algorithm..."}, +] + + +def _load_questions(path: str) -> list: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def _percentile(values: list, pct: float) -> float: + if not values: + return 0.0 + values = sorted(values) + k = (len(values) - 1) * (pct / 100) + f, c = int(k), min(int(k) + 1, len(values) - 1) + if f == c: + return values[f] + return values[f] + (k - f) * (values[c] - values[f]) + + +def _retrieved_topics(chunks: list) -> set: + return {c.get("metadata", {}).get("topic") for c in chunks if c.get("metadata", {}).get("topic")} + + +def _topic_recall(expected_topics: list, retrieved: set) -> float: + if not expected_topics: + return 1.0 # nothing expected (e.g. out_of_scope) => trivially satisfied + hits = sum(1 for t in expected_topics if t in retrieved) + return hits / len(expected_topics) + + +def evaluate_one(item: dict, retriever, with_llm: bool) -> dict: + query = item["query"] + expected_route = item["expected_route"] + expected_topics = item.get("expected_topics", []) + recent_messages = _FAKE_HISTORY_FOR_FOLLOWUPS if item.get("requires_history") else [] + + t0 = time.perf_counter() + decision = classify(query, has_conversation_history=len(recent_messages) > 0) + decision = gather_context(decision, retriever, recent_messages=recent_messages) + retrieval_latency_ms = (time.perf_counter() - t0) * 1000 + + if decision.route_type == RouteType.SINGLE_TOPIC: + retrieved = _retrieved_topics(decision.single_chunks) + elif decision.route_type == RouteType.COMPARISON: + retrieved = set() + for chunks in decision.comparison_context.values(): + retrieved |= _retrieved_topics(chunks) + elif decision.route_type == RouteType.FOLLOWUP: + retrieved = _retrieved_topics(decision.followup_context.get("additional_chunks", [])) + else: + retrieved = set() + + route_correct = decision.route_type.value == expected_route + topic_recall = _topic_recall(expected_topics, retrieved) + + result = { + "id": item.get("id"), + "query": query, + "expected_route": expected_route, + "predicted_route": decision.route_type.value, + "route_correct": route_correct, + "expected_topics": expected_topics, + "retrieved_topics": sorted(t for t in retrieved if t), + "topic_recall": round(topic_recall, 3), + "retrieval_latency_ms": round(retrieval_latency_ms, 2), + } + + if with_llm: + from llm.generate import generate + from llm.prompts import ( + build_comparison_prompt, + build_followup_prompt, + build_reject_prompt, + build_single_prompt, + ) + + if decision.route_type == RouteType.SINGLE_TOPIC: + prompt = build_single_prompt(decision.single_chunks, [], recent_messages, query) + elif decision.route_type == RouteType.COMPARISON: + prompt = build_comparison_prompt(decision.comparison_context, [], recent_messages, query) + elif decision.route_type == RouteType.FOLLOWUP: + prompt = build_followup_prompt(decision.followup_context, [], recent_messages, query) + else: + prompt = build_reject_prompt(query) + + t1 = time.perf_counter() + try: + response_text = generate(prompt) + gen_error = None + except Exception as exc: # keep the run going even if the LLM call fails + response_text = None + gen_error = str(exc) + generation_latency_ms = (time.perf_counter() - t1) * 1000 + + result["generation_latency_ms"] = round(generation_latency_ms, 2) + result["response_preview"] = (response_text or "")[:200] + if gen_error: + result["generation_error"] = gen_error + + return result + + +def run(questions_path: str = DEFAULT_QUESTIONS_PATH, with_llm: bool = False) -> dict: + from rag.retriever import get_retriever + + questions = _load_questions(questions_path) + retriever = get_retriever() + + results = [evaluate_one(item, retriever, with_llm) for item in questions] + + route_accuracy = sum(r["route_correct"] for r in results) / len(results) + avg_topic_recall = statistics.mean(r["topic_recall"] for r in results) + retrieval_latencies = [r["retrieval_latency_ms"] for r in results] + + summary = { + "total_questions": len(results), + "route_accuracy": round(route_accuracy, 3), + "avg_topic_recall": round(avg_topic_recall, 3), + "retrieval_latency_ms_avg": round(statistics.mean(retrieval_latencies), 2), + "retrieval_latency_ms_p95": round(_percentile(retrieval_latencies, 95), 2), + } + + if with_llm: + gen_latencies = [r["generation_latency_ms"] for r in results if "generation_latency_ms" in r] + if gen_latencies: + summary["generation_latency_ms_avg"] = round(statistics.mean(gen_latencies), 2) + summary["generation_latency_ms_p95"] = round(_percentile(gen_latencies, 95), 2) + summary["generation_errors"] = sum(1 for r in results if r.get("generation_error")) + + report = {"summary": summary, "results": results} + + logger.info("Evaluation complete: %s", summary) + return report + + +def _print_report(report: dict) -> None: + summary = report["summary"] + print("\n=== DSA RAG Chatbot — Evaluation Summary ===") + for key, value in summary.items(): + print(f" {key}: {value}") + + print("\n=== Per-question results ===") + header = f"{'id':<5} {'route (exp->got)':<28} {'recall':<8} {'ret_ms':<8} query" + print(header) + print("-" * len(header)) + for r in report["results"]: + route_str = f"{r['expected_route']} -> {r['predicted_route']}" + mark = "OK" if r["route_correct"] else "MISS" + print( + f"{r['id']:<5} {route_str:<28} {r['topic_recall']:<8} " + f"{r['retrieval_latency_ms']:<8} [{mark}] {r['query']}" + ) + + +def main(): + parser = argparse.ArgumentParser(description="Evaluate DSA RAG chatbot retrieval + routing.") + parser.add_argument( + "--questions", default=DEFAULT_QUESTIONS_PATH, help="Path to questions.json" + ) + parser.add_argument( + "--with-llm", + action="store_true", + help="Also call the configured LLM provider end-to-end (uses API quota).", + ) + parser.add_argument( + "--out", + default=None, + help="Path to write the full JSON report (default: evaluation/last_report.json)", + ) + args = parser.parse_args() + + report = run(questions_path=args.questions, with_llm=args.with_llm) + _print_report(report) + + out_path = args.out or os.path.join(os.path.dirname(__file__), "last_report.json") + with open(out_path, "w", encoding="utf-8") as f: + json.dump(report, f, indent=2) + print(f"\nFull report written to: {out_path}") + + +if __name__ == "__main__": + main() diff --git a/evaluation/last_report.json b/evaluation/last_report.json new file mode 100644 index 0000000000000000000000000000000000000000..064949309a2f5ee3ed14a76e5125e3c0cf2c06f1 --- /dev/null +++ b/evaluation/last_report.json @@ -0,0 +1,302 @@ +{ + "summary": { + "total_questions": 20, + "route_accuracy": 1.0, + "avg_topic_recall": 1.0, + "retrieval_latency_ms_avg": 46.26, + "retrieval_latency_ms_p95": 101.1 + }, + "results": [ + { + "id": "q01", + "query": "Explain how binary search works", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Binary Search" + ], + "retrieved_topics": [ + "Binary Search" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 144.79 + }, + { + "id": "q02", + "query": "What is the time complexity of merge sort?", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Merge Sort" + ], + "retrieved_topics": [ + "Merge Sort" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 30.63 + }, + { + "id": "q03", + "query": "Explain quick sort with an example", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Quick Sort" + ], + "retrieved_topics": [ + "Quick Sort" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 33.24 + }, + { + "id": "q04", + "query": "What is a Trie used for?", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Trie" + ], + "retrieved_topics": [ + "Trie" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 31.99 + }, + { + "id": "q05", + "query": "What is a doubly linked list?", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Doubly Linked List" + ], + "retrieved_topics": [ + "Doubly Linked List" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 34.44 + }, + { + "id": "q06", + "query": "Explain how a stack works", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Stack" + ], + "retrieved_topics": [ + "Stack" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 28.98 + }, + { + "id": "q07", + "query": "What is topological sort used for?", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Topological Sort" + ], + "retrieved_topics": [ + "Topological Sort" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 33.71 + }, + { + "id": "q08", + "query": "Explain Dijkstra's Algorithm", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Dijkstra's Algorithm" + ], + "retrieved_topics": [ + "Dijkstra's Algorithm" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 27.78 + }, + { + "id": "q09", + "query": "What is a segment tree?", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Segment Tree" + ], + "retrieved_topics": [ + "Segment Tree" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 35.57 + }, + { + "id": "q10", + "query": "Explain the sliding window technique", + "expected_route": "single_topic", + "predicted_route": "single_topic", + "route_correct": true, + "expected_topics": [ + "Sliding Window Technique" + ], + "retrieved_topics": [ + "Sliding Window Technique" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 34.67 + }, + { + "id": "q11", + "query": "What is the difference between BFS and DFS?", + "expected_route": "comparison", + "predicted_route": "comparison", + "route_correct": true, + "expected_topics": [ + "Breadth-First Search", + "Depth-First Search" + ], + "retrieved_topics": [ + "Breadth-First Search", + "Depth-First Search" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 51.73 + }, + { + "id": "q12", + "query": "Compare merge sort vs quick sort", + "expected_route": "comparison", + "predicted_route": "comparison", + "route_correct": true, + "expected_topics": [ + "Merge Sort", + "Quick Sort" + ], + "retrieved_topics": [ + "Merge Sort", + "Quick Sort" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 63.2 + }, + { + "id": "q13", + "query": "Difference between a stack and a queue", + "expected_route": "comparison", + "predicted_route": "comparison", + "route_correct": true, + "expected_topics": [ + "Stack", + "Queue" + ], + "retrieved_topics": [ + "Queue", + "Stack" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 66.79 + }, + { + "id": "q14", + "query": "Which one is better, Prim's Algorithm or Kruskal's Algorithm, for MST?", + "expected_route": "comparison", + "predicted_route": "comparison", + "route_correct": true, + "expected_topics": [ + "Prim's Algorithm (MST)", + "Kruskal's Algorithm (MST)" + ], + "retrieved_topics": [ + "Kruskal's Algorithm (MST)", + "Prim's Algorithm (MST)" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 98.8 + }, + { + "id": "q15", + "query": "Compare AVL Tree and Binary Search Tree", + "expected_route": "comparison", + "predicted_route": "comparison", + "route_correct": true, + "expected_topics": [ + "AVL Tree", + "Binary Search Tree" + ], + "retrieved_topics": [ + "AVL Tree", + "Binary Search", + "Binary Search Tree" + ], + "topic_recall": 1.0, + "retrieval_latency_ms": 84.79 + }, + { + "id": "q16", + "query": "Who won the football World Cup in 2022?", + "expected_route": "out_of_scope", + "predicted_route": "out_of_scope", + "route_correct": true, + "expected_topics": [], + "retrieved_topics": [], + "topic_recall": 1.0, + "retrieval_latency_ms": 23.85 + }, + { + "id": "q17", + "query": "What's a good recipe for chocolate cake?", + "expected_route": "out_of_scope", + "predicted_route": "out_of_scope", + "route_correct": true, + "expected_topics": [], + "retrieved_topics": [], + "topic_recall": 1.0, + "retrieval_latency_ms": 16.91 + }, + { + "id": "q18", + "query": "What is the capital of France?", + "expected_route": "out_of_scope", + "predicted_route": "out_of_scope", + "route_correct": true, + "expected_topics": [], + "retrieved_topics": [], + "topic_recall": 1.0, + "retrieval_latency_ms": 14.83 + }, + { + "id": "q19", + "query": "Can you explain that again?", + "expected_route": "followup", + "predicted_route": "followup", + "route_correct": true, + "expected_topics": [], + "retrieved_topics": [], + "topic_recall": 1.0, + "retrieval_latency_ms": 44.24 + }, + { + "id": "q20", + "query": "Give me another example", + "expected_route": "followup", + "predicted_route": "followup", + "route_correct": true, + "expected_topics": [], + "retrieved_topics": [], + "topic_recall": 1.0, + "retrieval_latency_ms": 24.32 + } + ] +} \ No newline at end of file diff --git a/evaluation/questions.json b/evaluation/questions.json new file mode 100644 index 0000000000000000000000000000000000000000..0cdf5512e318a9c514f3dfd3ca12ee47b49df5b4 --- /dev/null +++ b/evaluation/questions.json @@ -0,0 +1,124 @@ +[ + { + "id": "q01", + "query": "Explain how binary search works", + "expected_route": "single_topic", + "expected_topics": ["Binary Search"] + }, + { + "id": "q02", + "query": "What is the time complexity of merge sort?", + "expected_route": "single_topic", + "expected_topics": ["Merge Sort"] + }, + { + "id": "q03", + "query": "Explain quick sort with an example", + "expected_route": "single_topic", + "expected_topics": ["Quick Sort"] + }, + { + "id": "q04", + "query": "What is a Trie used for?", + "expected_route": "single_topic", + "expected_topics": ["Trie"] + }, + { + "id": "q05", + "query": "What is a doubly linked list?", + "expected_route": "single_topic", + "expected_topics": ["Doubly Linked List"] + }, + { + "id": "q06", + "query": "Explain how a stack works", + "expected_route": "single_topic", + "expected_topics": ["Stack"] + }, + { + "id": "q07", + "query": "What is topological sort used for?", + "expected_route": "single_topic", + "expected_topics": ["Topological Sort"] + }, + { + "id": "q08", + "query": "Explain Dijkstra's Algorithm", + "expected_route": "single_topic", + "expected_topics": ["Dijkstra's Algorithm"] + }, + { + "id": "q09", + "query": "What is a segment tree?", + "expected_route": "single_topic", + "expected_topics": ["Segment Tree"] + }, + { + "id": "q10", + "query": "Explain the sliding window technique", + "expected_route": "single_topic", + "expected_topics": ["Sliding Window Technique"] + }, + { + "id": "q11", + "query": "What is the difference between BFS and DFS?", + "expected_route": "comparison", + "expected_topics": ["Breadth-First Search", "Depth-First Search"] + }, + { + "id": "q12", + "query": "Compare merge sort vs quick sort", + "expected_route": "comparison", + "expected_topics": ["Merge Sort", "Quick Sort"] + }, + { + "id": "q13", + "query": "Difference between a stack and a queue", + "expected_route": "comparison", + "expected_topics": ["Stack", "Queue"] + }, + { + "id": "q14", + "query": "Which one is better, Prim's Algorithm or Kruskal's Algorithm, for MST?", + "expected_route": "comparison", + "expected_topics": ["Prim's Algorithm (MST)", "Kruskal's Algorithm (MST)"] + }, + { + "id": "q15", + "query": "Compare AVL Tree and Binary Search Tree", + "expected_route": "comparison", + "expected_topics": ["AVL Tree", "Binary Search Tree"] + }, + { + "id": "q16", + "query": "Who won the football World Cup in 2022?", + "expected_route": "out_of_scope", + "expected_topics": [] + }, + { + "id": "q17", + "query": "What's a good recipe for chocolate cake?", + "expected_route": "out_of_scope", + "expected_topics": [] + }, + { + "id": "q18", + "query": "What is the capital of France?", + "expected_route": "out_of_scope", + "expected_topics": [] + }, + { + "id": "q19", + "query": "Can you explain that again?", + "expected_route": "followup", + "expected_topics": [], + "requires_history": true + }, + { + "id": "q20", + "query": "Give me another example", + "expected_route": "followup", + "expected_topics": [], + "requires_history": true + } +] diff --git a/history/__init__.py b/history/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/history/models.py b/history/models.py new file mode 100644 index 0000000000000000000000000000000000000000..8291560e40431bbb014baadca65e79a917846896 --- /dev/null +++ b/history/models.py @@ -0,0 +1,71 @@ +""" +Chat history models. + +SQLite here is used ONLY for exact transcript storage (raw messages, +timestamps, sessions). It must never store embeddings (that's ChromaDB) +or user preference/long-term memory (that's mem0). +""" + +from datetime import datetime + +from auth.db import db + + +class ChatSession(db.Model): + __tablename__ = "chat_sessions" + + id = db.Column(db.Integer, primary_key=True) + user_id = db.Column(db.Integer, db.ForeignKey("users.id"), nullable=False, index=True) + title = db.Column(db.String(255), nullable=True) # e.g. first user message, truncated + created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + updated_at = db.Column( + db.DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False + ) + + messages = db.relationship( + "ChatMessage", + backref="chat_session", + lazy=True, + order_by="ChatMessage.created_at", + cascade="all, delete-orphan", + ) + + def to_dict(self, include_messages: bool = False) -> dict: + result = { + "id": self.id, + "user_id": self.user_id, + "title": self.title, + "created_at": self.created_at.isoformat(), + "updated_at": self.updated_at.isoformat(), + } + if include_messages: + result["messages"] = [m.to_dict() for m in self.messages] + return result + + +class ChatMessage(db.Model): + __tablename__ = "chat_messages" + + id = db.Column(db.Integer, primary_key=True) + session_id = db.Column( + db.Integer, db.ForeignKey("chat_sessions.id"), nullable=False, index=True + ) + role = db.Column(db.String(20), nullable=False) # "user" or "assistant" + content = db.Column(db.Text, nullable=False) + + # Optional debugging/eval metadata about how this response was produced. + route_type = db.Column(db.String(50), nullable=True) # single/comparison/followup/reject + retrieved_topics = db.Column(db.Text, nullable=True) # JSON-encoded list + + created_at = db.Column(db.DateTime, default=datetime.utcnow, nullable=False) + + def to_dict(self) -> dict: + return { + "id": self.id, + "session_id": self.session_id, + "role": self.role, + "content": self.content, + "route_type": self.route_type, + "retrieved_topics": self.retrieved_topics, + "created_at": self.created_at.isoformat(), + } diff --git a/history/service.py b/history/service.py new file mode 100644 index 0000000000000000000000000000000000000000..2c1c8c0c14ff2d611a9242fa8efed31f99cfe820 --- /dev/null +++ b/history/service.py @@ -0,0 +1,91 @@ +""" +Chat history service. + +The only module that reads/writes history.models (ChatSession, ChatMessage). +Callers (chat_service.py, chat_routes.py, the Gradio UI layer) go through +these functions instead of touching the ORM directly, so persistence +behavior stays in one place. + +Reminder of the scope boundary (see history/models.py): SQLite here is for +exact transcript storage only — no embeddings, no long-term preference +memory (that's ChromaDB / mem0 respectively). +""" + +import json + +from auth.db import db +from history.models import ChatMessage, ChatSession +from logs.logger import get_logger + +logger = get_logger(__name__) + + +def add_message( + session_id: int, + role: str, + content: str, + route_type: str = None, + retrieved_topics: list = None, +) -> ChatMessage: + """Append one message (user or assistant) to a session's transcript.""" + message = ChatMessage( + session_id=session_id, + role=role, + content=content, + route_type=route_type, + retrieved_topics=json.dumps(retrieved_topics) if retrieved_topics else None, + ) + db.session.add(message) + db.session.commit() + return message + + +def get_recent_messages(session_id: int, limit: int = None) -> list: + """ + Returns the most recent messages for a session, oldest -> newest, as + plain dicts ({"role": ..., "content": ...}) — the shape llm/prompts.py + and the follow-up agent expect. + """ + import config + + limit = limit or config.MAX_RECENT_HISTORY_MESSAGES + + messages = ( + ChatMessage.query.filter_by(session_id=session_id) + .order_by(ChatMessage.created_at.desc()) + .limit(limit) + .all() + ) + messages.reverse() + return [{"role": m.role, "content": m.content} for m in messages] + + +def list_sessions(user_id: int) -> list: + """Returns a summary (no messages) of every chat session for a user, + most recently updated first — used for the history panel.""" + sessions = ( + ChatSession.query.filter_by(user_id=user_id) + .order_by(ChatSession.updated_at.desc()) + .all() + ) + return [s.to_dict(include_messages=False) for s in sessions] + + +def get_full_session(session_id: int, user_id: int) -> dict: + """Returns the full transcript of one session, or None if it doesn't + exist or doesn't belong to this user.""" + session_obj = ChatSession.query.filter_by(id=session_id, user_id=user_id).first() + if session_obj is None: + return None + return session_obj.to_dict(include_messages=True) + + +def delete_session(session_id: int, user_id: int) -> bool: + """Deletes a session (and its messages, via cascade) if it belongs to + this user. Returns True if something was deleted.""" + session_obj = ChatSession.query.filter_by(id=session_id, user_id=user_id).first() + if session_obj is None: + return False + db.session.delete(session_obj) + db.session.commit() + return True diff --git a/history/session_manager.py b/history/session_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..5a799b53f39cd687b6cef445110fea0f63522ec2 --- /dev/null +++ b/history/session_manager.py @@ -0,0 +1,46 @@ +""" +Session resolution helper. + +Given a logged-in user and an optional client-supplied session_id, decides +whether to reuse an existing ChatSession or create a new one. Kept separate +from history/service.py so chat_service.py has one obvious entrypoint for +"which session am I writing to?" without duplicating that logic per caller. +""" + +from auth.db import db +from history.models import ChatSession + +TITLE_MAX_LENGTH = 60 + + +def _make_title(first_message: str) -> str: + first_message = (first_message or "").strip() + if not first_message: + return "New chat" + if len(first_message) <= TITLE_MAX_LENGTH: + return first_message + return first_message[: TITLE_MAX_LENGTH - 1].rstrip() + "…" + + +def resolve_session(user_id: int, session_id: int = None, first_message: str = "") -> ChatSession: + """ + Args: + user_id: the authenticated user's id. + session_id: an existing session id the client wants to continue. + Ignored (falls back to creating a new session) if it + doesn't exist or doesn't belong to this user. + first_message: used as the auto-generated title when a new session + is created. + + Returns: + The ChatSession to append messages to (already committed, has .id). + """ + if session_id is not None: + existing = ChatSession.query.filter_by(id=session_id, user_id=user_id).first() + if existing is not None: + return existing + + session_obj = ChatSession(user_id=user_id, title=_make_title(first_message)) + db.session.add(session_obj) + db.session.commit() + return session_obj diff --git a/llm/__init__.py b/llm/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/llm/generate.py b/llm/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..b3863d769d7f995c03a837b2ec6b09534adb08e9 --- /dev/null +++ b/llm/generate.py @@ -0,0 +1,64 @@ +""" +Provider factory. + +This is the ONLY module the rest of the application should import to talk +to an LLM. It picks the concrete provider based on config.LLM_PROVIDER and +exposes a single generate(prompt) function. Swapping providers is a +config change only. +""" + +import config +from logs.logger import get_logger + +logger = get_logger(__name__) + +_PROVIDER_SINGLETON = None + + +def _build_provider(): + provider_name = config.LLM_PROVIDER.lower() + + if provider_name == "local": + from llm.local_provider import LocalMistralProvider + + return LocalMistralProvider() + + if provider_name == "groq": + from llm.groq_provider import GroqProvider + + return GroqProvider() + + if provider_name == "openai": + from llm.openai_provider import OpenAIProvider + + return OpenAIProvider() + + raise ValueError( + f"Unknown LLM_PROVIDER={provider_name!r} in config.py. " + "Expected one of: 'local', 'groq', 'openai'." + ) + + +def get_provider(): + """Module-level singleton so the local model (if used) loads only once.""" + global _PROVIDER_SINGLETON + if _PROVIDER_SINGLETON is None: + _PROVIDER_SINGLETON = _build_provider() + logger.info("LLM provider initialized: %s", _PROVIDER_SINGLETON.name()) + return _PROVIDER_SINGLETON + + +def generate(prompt: str, stream: bool = False): + """ + The single entrypoint the rest of the app (Flask routes, agents, + evaluation) should call. Never import a provider class directly outside + of this module. + """ + provider = get_provider() + return provider.generate(prompt, stream=stream) + + +def reset_provider(): + """Mainly for tests: force re-reading config.LLM_PROVIDER on next call.""" + global _PROVIDER_SINGLETON + _PROVIDER_SINGLETON = None diff --git a/llm/groq_provider.py b/llm/groq_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..d7a1578bf09540b9358060c9150eaf526f119bb5 --- /dev/null +++ b/llm/groq_provider.py @@ -0,0 +1,82 @@ +""" +Groq-hosted LLM provider. + +Implements the same generate(prompt, stream=False) interface as +LocalMistralProvider, using Groq's OpenAI-compatible chat completions API. +Nothing outside llm/generate.py should import this class directly — always +go through llm.generate.generate(). +""" + +import config +from llm.provider import LLMProvider +from logs.logger import get_logger + +logger = get_logger(__name__) + + +class GroqProvider(LLMProvider): + def __init__(self): + if not config.GROQ_API_KEY: + raise ValueError( + "GROQ_API_KEY is not set in config/.env. Set it before using " + "LLM_PROVIDER=groq." + ) + + from groq import Groq + + self._client = Groq(api_key=config.GROQ_API_KEY) + self._model = config.GROQ_MODEL_NAME + + def generate(self, prompt: str, stream: bool = False): + """ + Args: + prompt: the fully-constructed prompt string from llm/prompts.py. + stream: if True, returns a generator yielding text chunks as + Groq streams them back. If False, returns the full + response as a single string. + """ + messages = [{"role": "user", "content": prompt}] + + if stream: + return self._generate_stream(messages) + return self._generate_sync(messages) + + def _generate_sync(self, messages) -> str: + try: + response = self._client.chat.completions.create( + model=self._model, + messages=messages, + temperature=config.LOCAL_MODEL_TEMPERATURE, + max_tokens=config.LOCAL_MODEL_MAX_NEW_TOKENS, + stream=False, + ) + except Exception: + logger.exception("Groq API call failed (model=%s)", self._model) + return ( + "Sorry, I couldn't reach the language model service right now. " + "Please try again in a moment." + ) + + return response.choices[0].message.content.strip() + + def _generate_stream(self, messages): + try: + stream = self._client.chat.completions.create( + model=self._model, + messages=messages, + temperature=config.LOCAL_MODEL_TEMPERATURE, + max_tokens=config.LOCAL_MODEL_MAX_NEW_TOKENS, + stream=True, + ) + except Exception: + logger.exception("Groq streaming API call failed (model=%s)", self._model) + yield ( + "Sorry, I couldn't reach the language model service right now. " + "Please try again in a moment." + ) + return + + for chunk in stream: + delta = chunk.choices[0].delta.content if chunk.choices else None + if delta: + yield delta diff --git a/llm/local_provider.py b/llm/local_provider.py new file mode 100644 index 0000000000000000000000000000000000000000..594d88fdf0ae72da4be5018efe5808c603ab1f38 --- /dev/null +++ b/llm/local_provider.py @@ -0,0 +1,98 @@ +""" +Local Mistral-7B-Instruct-v0.2 provider. + +Loads the model lazily (on first generate() call, not at import time) so +importing this module never requires a GPU or downloaded weights — this +keeps the rest of the codebase importable/testable without the model +present. + +Requires: transformers, bitsandbytes, torch, accelerate (see requirements.txt) +""" + +import config +from llm.provider import LLMProvider +from logs.logger import get_logger + +logger = get_logger(__name__) + + +class LocalMistralProvider(LLMProvider): + def __init__(self): + self._model = None + self._tokenizer = None + + def _load(self): + """Lazy-load the model + tokenizer on first use.""" + if self._model is not None: + return + + logger.info("Loading local model %s ...", config.LOCAL_MODEL_NAME) + + import torch + from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig + + quantization_config = None + if config.LOCAL_MODEL_LOAD_IN_4BIT: + quantization_config = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_quant_type="nf4", + bnb_4bit_compute_dtype=torch.float16, + bnb_4bit_use_double_quant=True, + ) + + self._tokenizer = AutoTokenizer.from_pretrained(config.LOCAL_MODEL_NAME) + self._model = AutoModelForCausalLM.from_pretrained( + config.LOCAL_MODEL_NAME, + quantization_config=quantization_config, + device_map="auto", + ) + + logger.info("Local model loaded.") + + def generate(self, prompt: str, stream: bool = False): + self._load() + + import torch + + inputs = self._tokenizer(prompt, return_tensors="pt").to(self._model.device) + + if not stream: + with torch.no_grad(): + output_ids = self._model.generate( + **inputs, + max_new_tokens=config.LOCAL_MODEL_MAX_NEW_TOKENS, + temperature=config.LOCAL_MODEL_TEMPERATURE, + do_sample=config.LOCAL_MODEL_TEMPERATURE > 0, + ) + full_text = self._tokenizer.decode( + output_ids[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True + ) + return full_text.strip() + + return self._generate_streaming(inputs) + + def _generate_streaming(self, inputs): + import threading + + import torch + from transformers import TextIteratorStreamer + + streamer = TextIteratorStreamer( + self._tokenizer, skip_prompt=True, skip_special_tokens=True + ) + + generation_kwargs = dict( + **inputs, + max_new_tokens=config.LOCAL_MODEL_MAX_NEW_TOKENS, + temperature=config.LOCAL_MODEL_TEMPERATURE, + do_sample=config.LOCAL_MODEL_TEMPERATURE > 0, + streamer=streamer, + ) + + thread = threading.Thread(target=self._model.generate, kwargs=generation_kwargs) + thread.start() + + for text_chunk in streamer: + yield text_chunk + + thread.join() diff --git a/llm/prompts.py b/llm/prompts.py new file mode 100644 index 0000000000000000000000000000000000000000..a0c5b5c7582dc2cffbfa2e51deba51f86abbd5a4 --- /dev/null +++ b/llm/prompts.py @@ -0,0 +1,203 @@ +""" +Prompt builders. + +Every prompt follows this fixed order: + + System Prompt + -> Retrieved Context + -> Relevant mem0 Context + -> Recent Conversation History + -> Current User Question + +One builder per route type, so each route's prompt shape can evolve +independently: + + build_single_prompt() + build_comparison_prompt() + build_followup_prompt() + build_direct_prompt() + build_reject_prompt() + +These builders only assemble strings — they never call the LLM themselves +(see llm/generate.py for that). +""" + +SYSTEM_PROMPT = ( + "You are a patient, precise Data Structures and Algorithms (DSA) tutor. " + "Answer using ONLY the information provided in the retrieved context below. " + "If the retrieved context does not contain enough information to answer " + "confidently, say so explicitly rather than guessing or inventing details. " + "Explain concepts clearly, use the time/space complexity and examples given " + "when relevant, and adapt your explanation style to any user preferences " + "provided in the memory context." +) + + +# --------------------------------------------------------------------------- +# Shared formatting helpers +# --------------------------------------------------------------------------- + +def _format_chunks(chunks: list, heading: str = None) -> str: + if not chunks: + return "" + lines = [] + if heading: + lines.append(f"### {heading}") + for chunk in chunks: + meta = chunk.get("metadata", {}) + section = meta.get("section", "") + topic = meta.get("topic", "") + lines.append(f"[{topic} - {section}]\n{chunk.get('text', '')}") + return "\n\n".join(lines) + + +def _format_mem0_context(mem0_context) -> str: + if not mem0_context: + return "" + if isinstance(mem0_context, str): + body = mem0_context + else: + # list of preference strings/facts returned by memory/mem0_client.py + body = "\n".join(f"- {item}" for item in mem0_context) + return f"### User Preferences / Memory\n{body}" + + +def _format_history(recent_messages: list) -> str: + if not recent_messages: + return "" + lines = ["### Recent Conversation"] + for message in recent_messages: + role = message.get("role", "user").capitalize() + content = message.get("content", "") + lines.append(f"{role}: {content}") + return "\n".join(lines) + + +def _assemble(*sections: str) -> str: + """Joins non-empty sections with blank lines, preserving fixed order.""" + return "\n\n".join(s for s in sections if s) + + +# --------------------------------------------------------------------------- +# Per-route builders +# --------------------------------------------------------------------------- + +def build_single_prompt( + chunks: list, + mem0_context=None, + recent_history: list = None, + query: str = "", +) -> str: + """Single-topic route, e.g. 'Explain Merge Sort'.""" + retrieved = _format_chunks(chunks, heading="Retrieved Context") + mem0_section = _format_mem0_context(mem0_context) + history_section = _format_history(recent_history) + question_section = f"### Current Question\n{query}" + + return _assemble( + SYSTEM_PROMPT, retrieved, mem0_section, history_section, question_section + ) + + +def build_comparison_prompt( + comparison_context: dict, + mem0_context=None, + recent_history: list = None, + query: str = "", +) -> str: + """Comparison route, e.g. 'Difference between BFS and DFS'.""" + retrieved_sections = [] + for topic, chunks in comparison_context.items(): + retrieved_sections.append(_format_chunks(chunks, heading=f"Retrieved Context: {topic}")) + retrieved = "\n\n".join(s for s in retrieved_sections if s) + + mem0_section = _format_mem0_context(mem0_context) + history_section = _format_history(recent_history) + question_section = f"### Current Question\n{query}" + + instruction = ( + "Compare the topics above directly: highlight the key differences in " + "definition, time/space complexity, and when to use each." + ) + + return _assemble( + SYSTEM_PROMPT, instruction, retrieved, mem0_section, history_section, question_section + ) + + +def build_followup_prompt( + followup_context: dict, + mem0_context=None, + recent_history: list = None, + query: str = "", +) -> str: + """ + Follow-up route, e.g. 'Can you explain that again?'. Reuses recent + conversation as primary context; only includes `additional_chunks` if + the follow-up agent decided a fresh retrieval was necessary. + """ + additional_chunks = followup_context.get("additional_chunks", []) + retrieved = _format_chunks(additional_chunks, heading="Additional Retrieved Context") + + mem0_section = _format_mem0_context(mem0_context) + # Prefer the followup_context's own recent_messages if the caller didn't + # pass a separate recent_history override. + history_section = _format_history( + recent_history or followup_context.get("recent_messages", []) + ) + question_section = f"### Current Question\n{query}" + + instruction = ( + "This is a follow-up question. Use the recent conversation above to " + "understand what the user is referring to before answering." + ) + + WORKED_EXAMPLE_INSTRUCTION = ( + "If the user provides their own concrete data (a list, array, or set of " + "numbers) and asks you to apply a search/sort/algorithm to it, walk " + "through the exact steps of the algorithm from the retrieved context " + "on their data. Do not substitute a different algorithm than the one " + "being discussed, even if their data doesn't perfectly fit it (e.g. " + "note if the list isn't sorted, but still apply the same algorithm's " + "logic unless the user explicitly asks for a different one)." + ) + + return _assemble( + SYSTEM_PROMPT, instruction, WORKED_EXAMPLE_INSTRUCTION, retrieved, mem0_section, history_section, question_section, + ) + + +def build_direct_prompt( + mem0_context=None, + recent_history: list = None, + query: str = "", +) -> str: + """ + Fallback builder for cases with no retrieved KB context at all (e.g. a + general greeting, or a follow-up so light it needs no context beyond + conversation history). Still follows the same section ordering, simply + omitting the "Retrieved Context" section. + """ + mem0_section = _format_mem0_context(mem0_context) + history_section = _format_history(recent_history) + question_section = f"### Current Question\n{query}" + + return _assemble(SYSTEM_PROMPT, mem0_section, history_section, question_section) + + +def build_reject_prompt(query: str = "") -> str: + """ + Out-of-scope route, e.g. 'Who won the World Cup?'. Kept deliberately + simple/static — no retrieved context, no memory, no history — since the + goal is just a polite, on-brand refusal, not a reasoned answer. + """ + instruction = ( + "The user's question is outside the scope of this DSA tutor " + "(it isn't a data structures / algorithms question, or the knowledge " + "base has no relevant information). Politely explain that you can " + "only help with Data Structures and Algorithms topics, and invite " + "them to ask a DSA-related question instead. Keep it brief and friendly." + ) + question_section = f"### User's Message\n{query}" + + return _assemble(SYSTEM_PROMPT, instruction, question_section) diff --git a/llm/provider.py b/llm/provider.py new file mode 100644 index 0000000000000000000000000000000000000000..bbe94bfe118691caaf8b1dcebf82f5abcc7c54fe --- /dev/null +++ b/llm/provider.py @@ -0,0 +1,33 @@ +""" +LLM provider interface. + +The rest of the application (router, chat endpoints, evaluation) must +NEVER call a specific model directly. Everything goes through +generate(prompt) on whatever provider is configured via +config.LLM_PROVIDER. Switching from local Mistral to Groq or OpenAI later +should require changing config only — no changes to callers. +""" + +from abc import ABC, abstractmethod + + +class LLMProvider(ABC): + """Base interface every provider (local, groq, openai, ...) must implement.""" + + @abstractmethod + def generate(self, prompt: str, stream: bool = False): + """ + Args: + prompt: the fully-constructed prompt string (see llm/prompts.py). + stream: if True, returns a generator yielding text chunks as + they're produced, instead of a single string. Providers + that can't stream should fall back to yielding the full + response once. + + Returns: + str if stream=False, otherwise a generator[str]. + """ + raise NotImplementedError + + def name(self) -> str: + return self.__class__.__name__ diff --git a/logs/__init__.py b/logs/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/logs/app.log b/logs/app.log new file mode 100644 index 0000000000000000000000000000000000000000..ef7e96a37a6997bd2d7a584dbb6e358e42268700 --- /dev/null +++ b/logs/app.log @@ -0,0 +1,440 @@ +2026-08-02 13:43:39,527 | INFO | dsa_chatbot.__main__ | Loading embedding model: all-MiniLM-L6-v2 +2026-08-02 13:44:07,549 | WARNING | dsa_chatbot.__main__ | The following KB files have no topic_metadata.json entry and will be skipped: ['01_knapsack.txt', 'activity_selection_problem.txt', 'backtracking_overview.txt', 'bit_manipulation_basics.txt', 'coin_change_problem.txt', 'dijkstras_algorithm.txt', 'dynamic_programming_overview.txt', 'edit_distance.txt', 'fractional_knapsack.txt', 'greedy_algorithms_overview.txt', 'hash_set_vs_hash_map.txt', 'hash_table.txt', 'kruskals_algorithm_mst.txt', 'longest_common_subsequence.txt', 'longest_increasing_subsequence.txt', 'n-queens_problem.txt', 'prims_algorithm_mst.txt', 'recursion_overview.txt', 'subset_sum_power_set_generation.txt', 'xor_tricks.txt'] +2026-08-02 13:44:08,102 | INFO | dsa_chatbot.__main__ | Built 98 chunks from 54 knowledge-base files. +2026-08-02 13:44:08,102 | INFO | dsa_chatbot.__main__ | Embedding 98 chunks... +2026-08-02 13:44:13,562 | INFO | dsa_chatbot.__main__ | Index build complete. Collection 'dsa_knowledge_base' now has 98 chunks. +2026-08-02 13:45:19,505 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain how binary search works' | topic=Binary Search +2026-08-02 13:45:19,648 | INFO | dsa_chatbot.rag.retriever | Query='Explain how binary search works' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:19,658 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is the time complexity of merge sort?' | topic=Insertion Sort +2026-08-02 13:45:19,680 | INFO | dsa_chatbot.rag.retriever | Query='What is the time complexity of merge sort?' | candidates=1 | passed_threshold=0 | threshold=0.45 +2026-08-02 13:45:19,681 | WARNING | dsa_chatbot.agents.router | SINGLE_TOPIC query had no chunks pass the similarity threshold: 'What is the time complexity of merge sort?' +2026-08-02 13:45:19,688 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain quick sort with an example' | topic=Bucket Sort +2026-08-02 13:45:19,718 | INFO | dsa_chatbot.rag.retriever | Query='Explain quick sort with an example' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:19,727 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a Trie used for?' | topic=Insertion Sort +2026-08-02 13:45:19,752 | INFO | dsa_chatbot.rag.retriever | Query='What is a Trie used for?' | candidates=1 | passed_threshold=0 | threshold=0.45 +2026-08-02 13:45:19,753 | WARNING | dsa_chatbot.agents.router | SINGLE_TOPIC query had no chunks pass the similarity threshold: 'What is a Trie used for?' +2026-08-02 13:45:19,763 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a doubly linked list?' | topic=Doubly Linked List +2026-08-02 13:45:19,791 | INFO | dsa_chatbot.rag.retriever | Query='What is a doubly linked list?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:19,796 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain how a stack works' | topic=Stack +2026-08-02 13:45:19,817 | INFO | dsa_chatbot.rag.retriever | Query='Explain how a stack works' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:19,825 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is topological sort used for?' | topic=Insertion Sort +2026-08-02 13:45:19,848 | INFO | dsa_chatbot.rag.retriever | Query='What is topological sort used for?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:19,852 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: "Explain Dijkstra's Algorithm" | topic=Dijkstra's Algorithm +2026-08-02 13:45:19,899 | INFO | dsa_chatbot.rag.retriever | Query="Explain Dijkstra's Algorithm" | candidates=0 | passed_threshold=0 | threshold=0.45 +2026-08-02 13:45:19,899 | WARNING | dsa_chatbot.agents.router | SINGLE_TOPIC query had no chunks pass the similarity threshold: "Explain Dijkstra's Algorithm" +2026-08-02 13:45:19,905 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a segment tree?' | topic=Insertion Sort +2026-08-02 13:45:19,929 | INFO | dsa_chatbot.rag.retriever | Query='What is a segment tree?' | candidates=1 | passed_threshold=0 | threshold=0.45 +2026-08-02 13:45:19,929 | WARNING | dsa_chatbot.agents.router | SINGLE_TOPIC query had no chunks pass the similarity threshold: 'What is a segment tree?' +2026-08-02 13:45:19,935 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain the sliding window technique' | topic=Sliding Window Technique +2026-08-02 13:45:19,958 | INFO | dsa_chatbot.rag.retriever | Query='Explain the sliding window technique' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:19,966 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'What is the difference between BFS and DFS?' | topics=['Breadth-First Search', 'Depth-First Search', 'Insertion Sort'] +2026-08-02 13:45:19,995 | INFO | dsa_chatbot.rag.retriever | Query='Breadth-First Search' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 13:45:20,013 | INFO | dsa_chatbot.rag.retriever | Query='Depth-First Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,033 | INFO | dsa_chatbot.rag.retriever | Query='Insertion Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,042 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Compare merge sort vs quick sort' | topics=['Bucket Sort', 'Merge Sort', 'Quick Sort'] +2026-08-02 13:45:20,062 | INFO | dsa_chatbot.rag.retriever | Query='Bucket Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,087 | INFO | dsa_chatbot.rag.retriever | Query='Merge Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,107 | INFO | dsa_chatbot.rag.retriever | Query='Quick Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,115 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Difference between a stack and a queue' | topics=['Queue', 'Stack'] +2026-08-02 13:45:20,168 | INFO | dsa_chatbot.rag.retriever | Query='Queue' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 13:45:20,190 | INFO | dsa_chatbot.rag.retriever | Query='Stack' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,215 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: "Which one is better, Prim's Algorithm or Kruskal's Algorithm, for MST?" | topics=["Dijkstra's Algorithm", 'Insertion Sort', "Kruskal's Algorithm (MST)", "Prim's Algorithm (MST)"] +2026-08-02 13:45:20,245 | INFO | dsa_chatbot.rag.retriever | Query="Dijkstra's Algorithm" | candidates=0 | passed_threshold=0 | threshold=0.45 +2026-08-02 13:45:20,271 | INFO | dsa_chatbot.rag.retriever | Query='Insertion Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,305 | INFO | dsa_chatbot.rag.retriever | Query="Kruskal's Algorithm (MST)" | candidates=0 | passed_threshold=0 | threshold=0.45 +2026-08-02 13:45:20,330 | INFO | dsa_chatbot.rag.retriever | Query="Prim's Algorithm (MST)" | candidates=0 | passed_threshold=0 | threshold=0.45 +2026-08-02 13:45:20,330 | WARNING | dsa_chatbot.agents.comparison | Comparison agent: no chunks passed the similarity threshold for topic "Dijkstra's Algorithm" +2026-08-02 13:45:20,330 | WARNING | dsa_chatbot.agents.comparison | Comparison agent: no chunks passed the similarity threshold for topic "Kruskal's Algorithm (MST)" +2026-08-02 13:45:20,330 | WARNING | dsa_chatbot.agents.comparison | Comparison agent: no chunks passed the similarity threshold for topic "Prim's Algorithm (MST)" +2026-08-02 13:45:20,331 | WARNING | dsa_chatbot.agents.router | COMPARISON query missing sufficient context for one or more topics: ["Dijkstra's Algorithm", 'Insertion Sort', "Kruskal's Algorithm (MST)", "Prim's Algorithm (MST)"] +2026-08-02 13:45:20,341 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Compare AVL Tree and Binary Search Tree' | topics=['AVL Tree', 'Binary Search', 'Binary Search Tree', 'Linear Search', 'Trie'] +2026-08-02 13:45:20,362 | INFO | dsa_chatbot.rag.retriever | Query='AVL Tree' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 13:45:20,380 | INFO | dsa_chatbot.rag.retriever | Query='Binary Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,403 | INFO | dsa_chatbot.rag.retriever | Query='Binary Search Tree' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,426 | INFO | dsa_chatbot.rag.retriever | Query='Linear Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 13:45:20,448 | INFO | dsa_chatbot.rag.retriever | Query='Trie' | candidates=5 | passed_threshold=4 | threshold=0.45 +2026-08-02 13:45:20,459 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'Who won the football World Cup in 2022?' +2026-08-02 13:45:20,466 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: "What's a good recipe for chocolate cake?" +2026-08-02 13:45:20,474 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is the capital of France?' | topic=Insertion Sort +2026-08-02 13:45:20,495 | INFO | dsa_chatbot.rag.retriever | Query='What is the capital of France?' | candidates=1 | passed_threshold=0 | threshold=0.45 +2026-08-02 13:45:20,495 | WARNING | dsa_chatbot.agents.router | SINGLE_TOPIC query had no chunks pass the similarity threshold: 'What is the capital of France?' +2026-08-02 13:45:20,499 | INFO | dsa_chatbot.agents.router | Routed to FOLLOWUP: 'Can you explain that again?' +2026-08-02 13:45:20,517 | INFO | dsa_chatbot.agents.followup | Follow-up query reuses existing conversation context (topics=['Merge Sort', 'Insertion Sort']); skipping fresh retrieval. +2026-08-02 13:45:20,530 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'Give me another example' +2026-08-02 13:45:20,531 | INFO | dsa_chatbot.__main__ | Evaluation complete: {'total_questions': 20, 'route_accuracy': 0.9, 'avg_topic_recall': 0.65, 'retrieval_latency_ms_avg': 51.74, 'retrieval_latency_ms_p95': 140.89} +2026-08-02 13:59:25,086 | INFO | dsa_chatbot.__main__ | Loading embedding model: all-MiniLM-L6-v2 +2026-08-02 13:59:31,186 | INFO | dsa_chatbot.__main__ | Existing collection 'dsa_knowledge_base' deleted. +2026-08-02 13:59:31,554 | INFO | dsa_chatbot.__main__ | Built 150 chunks from 54 knowledge-base files. +2026-08-02 13:59:31,555 | INFO | dsa_chatbot.__main__ | Embedding 150 chunks... +2026-08-02 13:59:39,737 | INFO | dsa_chatbot.__main__ | Index build complete. Collection 'dsa_knowledge_base' now has 150 chunks. +2026-08-02 14:00:37,066 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain how binary search works' | topic=Binary Search +2026-08-02 14:00:37,234 | INFO | dsa_chatbot.rag.retriever | Query='Explain how binary search works' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,248 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is the time complexity of merge sort?' | topic=Insertion Sort +2026-08-02 14:00:37,273 | INFO | dsa_chatbot.rag.retriever | Query='What is the time complexity of merge sort?' | candidates=1 | passed_threshold=0 | threshold=0.45 +2026-08-02 14:00:37,274 | WARNING | dsa_chatbot.agents.router | SINGLE_TOPIC query had no chunks pass the similarity threshold: 'What is the time complexity of merge sort?' +2026-08-02 14:00:37,288 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain quick sort with an example' | topic=Bucket Sort +2026-08-02 14:00:37,310 | INFO | dsa_chatbot.rag.retriever | Query='Explain quick sort with an example' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,321 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a Trie used for?' | topic=Insertion Sort +2026-08-02 14:00:37,345 | INFO | dsa_chatbot.rag.retriever | Query='What is a Trie used for?' | candidates=1 | passed_threshold=0 | threshold=0.45 +2026-08-02 14:00:37,346 | WARNING | dsa_chatbot.agents.router | SINGLE_TOPIC query had no chunks pass the similarity threshold: 'What is a Trie used for?' +2026-08-02 14:00:37,357 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a doubly linked list?' | topic=Doubly Linked List +2026-08-02 14:00:37,378 | INFO | dsa_chatbot.rag.retriever | Query='What is a doubly linked list?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,386 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain how a stack works' | topic=Stack +2026-08-02 14:00:37,405 | INFO | dsa_chatbot.rag.retriever | Query='Explain how a stack works' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,415 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is topological sort used for?' | topic=Insertion Sort +2026-08-02 14:00:37,433 | INFO | dsa_chatbot.rag.retriever | Query='What is topological sort used for?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,439 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: "Explain Dijkstra's Algorithm" | topic=Dijkstra's Algorithm +2026-08-02 14:00:37,463 | INFO | dsa_chatbot.rag.retriever | Query="Explain Dijkstra's Algorithm" | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,471 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a segment tree?' | topic=Insertion Sort +2026-08-02 14:00:37,493 | INFO | dsa_chatbot.rag.retriever | Query='What is a segment tree?' | candidates=1 | passed_threshold=0 | threshold=0.45 +2026-08-02 14:00:37,494 | WARNING | dsa_chatbot.agents.router | SINGLE_TOPIC query had no chunks pass the similarity threshold: 'What is a segment tree?' +2026-08-02 14:00:37,501 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain the sliding window technique' | topic=Sliding Window Technique +2026-08-02 14:00:37,521 | INFO | dsa_chatbot.rag.retriever | Query='Explain the sliding window technique' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,539 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'What is the difference between BFS and DFS?' | topics=['Breadth-First Search', 'Depth-First Search', 'Insertion Sort'] +2026-08-02 14:00:37,597 | INFO | dsa_chatbot.rag.retriever | Query='Breadth-First Search' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:00:37,624 | INFO | dsa_chatbot.rag.retriever | Query='Depth-First Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,647 | INFO | dsa_chatbot.rag.retriever | Query='Insertion Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,666 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Compare merge sort vs quick sort' | topics=['Bucket Sort', 'Merge Sort', 'Quick Sort'] +2026-08-02 14:00:37,699 | INFO | dsa_chatbot.rag.retriever | Query='Bucket Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,725 | INFO | dsa_chatbot.rag.retriever | Query='Merge Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,751 | INFO | dsa_chatbot.rag.retriever | Query='Quick Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,767 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Difference between a stack and a queue' | topics=['Queue', 'Stack'] +2026-08-02 14:00:37,821 | INFO | dsa_chatbot.rag.retriever | Query='Queue' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:00:37,851 | INFO | dsa_chatbot.rag.retriever | Query='Stack' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,892 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: "Which one is better, Prim's Algorithm or Kruskal's Algorithm, for MST?" | topics=["Dijkstra's Algorithm", 'Insertion Sort', "Kruskal's Algorithm (MST)", "Prim's Algorithm (MST)"] +2026-08-02 14:00:37,919 | INFO | dsa_chatbot.rag.retriever | Query="Dijkstra's Algorithm" | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,944 | INFO | dsa_chatbot.rag.retriever | Query='Insertion Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:37,977 | INFO | dsa_chatbot.rag.retriever | Query="Kruskal's Algorithm (MST)" | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:00:38,005 | INFO | dsa_chatbot.rag.retriever | Query="Prim's Algorithm (MST)" | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:00:38,022 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Compare AVL Tree and Binary Search Tree' | topics=['AVL Tree', 'Binary Search', 'Binary Search Tree', 'Linear Search', 'Trie'] +2026-08-02 14:00:38,043 | INFO | dsa_chatbot.rag.retriever | Query='AVL Tree' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:00:38,058 | INFO | dsa_chatbot.rag.retriever | Query='Binary Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:38,081 | INFO | dsa_chatbot.rag.retriever | Query='Binary Search Tree' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:38,109 | INFO | dsa_chatbot.rag.retriever | Query='Linear Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:00:38,126 | INFO | dsa_chatbot.rag.retriever | Query='Trie' | candidates=5 | passed_threshold=4 | threshold=0.45 +2026-08-02 14:00:38,138 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'Who won the football World Cup in 2022?' +2026-08-02 14:00:38,152 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: "What's a good recipe for chocolate cake?" +2026-08-02 14:00:38,164 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is the capital of France?' | topic=Insertion Sort +2026-08-02 14:00:38,186 | INFO | dsa_chatbot.rag.retriever | Query='What is the capital of France?' | candidates=1 | passed_threshold=0 | threshold=0.45 +2026-08-02 14:00:38,186 | WARNING | dsa_chatbot.agents.router | SINGLE_TOPIC query had no chunks pass the similarity threshold: 'What is the capital of France?' +2026-08-02 14:00:38,195 | INFO | dsa_chatbot.agents.router | Routed to FOLLOWUP: 'Can you explain that again?' +2026-08-02 14:00:38,226 | INFO | dsa_chatbot.agents.followup | Follow-up query reuses existing conversation context (topics=['Merge Sort', 'Insertion Sort']); skipping fresh retrieval. +2026-08-02 14:00:38,236 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'Give me another example' +2026-08-02 14:00:38,237 | INFO | dsa_chatbot.__main__ | Evaluation complete: {'total_questions': 20, 'route_accuracy': 0.9, 'avg_topic_recall': 0.75, 'retrieval_latency_ms_avg': 59.26, 'retrieval_latency_ms_p95': 155.61} +2026-08-02 14:03:52,099 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain how binary search works' | topic=Binary Search +2026-08-02 14:03:52,274 | INFO | dsa_chatbot.rag.retriever | Query='Explain how binary search works' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,288 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is the time complexity of merge sort?' | topic=Merge Sort +2026-08-02 14:03:52,311 | INFO | dsa_chatbot.rag.retriever | Query='What is the time complexity of merge sort?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,324 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain quick sort with an example' | topic=Quick Sort +2026-08-02 14:03:52,350 | INFO | dsa_chatbot.rag.retriever | Query='Explain quick sort with an example' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,362 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a Trie used for?' | topic=Trie +2026-08-02 14:03:52,384 | INFO | dsa_chatbot.rag.retriever | Query='What is a Trie used for?' | candidates=5 | passed_threshold=4 | threshold=0.45 +2026-08-02 14:03:52,398 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a doubly linked list?' | topic=Doubly Linked List +2026-08-02 14:03:52,419 | INFO | dsa_chatbot.rag.retriever | Query='What is a doubly linked list?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,430 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain how a stack works' | topic=Stack +2026-08-02 14:03:52,450 | INFO | dsa_chatbot.rag.retriever | Query='Explain how a stack works' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,462 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is topological sort used for?' | topic=Topological Sort +2026-08-02 14:03:52,483 | INFO | dsa_chatbot.rag.retriever | Query='What is topological sort used for?' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:03:52,490 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: "Explain Dijkstra's Algorithm" | topic=Dijkstra's Algorithm +2026-08-02 14:03:52,516 | INFO | dsa_chatbot.rag.retriever | Query="Explain Dijkstra's Algorithm" | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,522 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a segment tree?' | topic=Segment Tree +2026-08-02 14:03:52,547 | INFO | dsa_chatbot.rag.retriever | Query='What is a segment tree?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,561 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain the sliding window technique' | topic=Sliding Window Technique +2026-08-02 14:03:52,582 | INFO | dsa_chatbot.rag.retriever | Query='Explain the sliding window technique' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,605 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'What is the difference between BFS and DFS?' | topics=['Breadth-First Search', 'Depth-First Search'] +2026-08-02 14:03:52,627 | INFO | dsa_chatbot.rag.retriever | Query='Breadth-First Search' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:03:52,649 | INFO | dsa_chatbot.rag.retriever | Query='Depth-First Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,662 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Compare merge sort vs quick sort' | topics=['Merge Sort', 'Quick Sort'] +2026-08-02 14:03:52,685 | INFO | dsa_chatbot.rag.retriever | Query='Merge Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,706 | INFO | dsa_chatbot.rag.retriever | Query='Quick Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,722 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Difference between a stack and a queue' | topics=['Queue', 'Stack'] +2026-08-02 14:03:52,749 | INFO | dsa_chatbot.rag.retriever | Query='Queue' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:03:52,767 | INFO | dsa_chatbot.rag.retriever | Query='Stack' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,793 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: "Which one is better, Prim's Algorithm or Kruskal's Algorithm, for MST?" +2026-08-02 14:03:52,814 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Compare AVL Tree and Binary Search Tree' | topics=['AVL Tree', 'Binary Search', 'Binary Search Tree'] +2026-08-02 14:03:52,836 | INFO | dsa_chatbot.rag.retriever | Query='AVL Tree' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:03:52,852 | INFO | dsa_chatbot.rag.retriever | Query='Binary Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,873 | INFO | dsa_chatbot.rag.retriever | Query='Binary Search Tree' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:03:52,886 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'Who won the football World Cup in 2022?' +2026-08-02 14:03:52,904 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: "What's a good recipe for chocolate cake?" +2026-08-02 14:03:52,917 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'What is the capital of France?' +2026-08-02 14:03:52,924 | INFO | dsa_chatbot.agents.router | Routed to FOLLOWUP: 'Can you explain that again?' +2026-08-02 14:03:52,949 | INFO | dsa_chatbot.agents.followup | Follow-up query reuses existing conversation context (topics=['Merge Sort']); skipping fresh retrieval. +2026-08-02 14:03:52,950 | INFO | dsa_chatbot.agents.router | Routed to FOLLOWUP: 'Give me another example' +2026-08-02 14:03:52,973 | INFO | dsa_chatbot.agents.followup | Follow-up query reuses existing conversation context (topics=['Merge Sort']); skipping fresh retrieval. +2026-08-02 14:03:52,974 | INFO | dsa_chatbot.__main__ | Evaluation complete: {'total_questions': 20, 'route_accuracy': 0.95, 'avg_topic_recall': 0.95, 'retrieval_latency_ms_avg': 44.4, 'retrieval_latency_ms_p95': 85.11} +2026-08-02 14:06:23,677 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain how binary search works' | topic=Binary Search +2026-08-02 14:06:23,808 | INFO | dsa_chatbot.rag.retriever | Query='Explain how binary search works' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:23,820 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is the time complexity of merge sort?' | topic=Merge Sort +2026-08-02 14:06:23,838 | INFO | dsa_chatbot.rag.retriever | Query='What is the time complexity of merge sort?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:23,848 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain quick sort with an example' | topic=Quick Sort +2026-08-02 14:06:23,872 | INFO | dsa_chatbot.rag.retriever | Query='Explain quick sort with an example' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:23,881 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a Trie used for?' | topic=Trie +2026-08-02 14:06:23,904 | INFO | dsa_chatbot.rag.retriever | Query='What is a Trie used for?' | candidates=5 | passed_threshold=4 | threshold=0.45 +2026-08-02 14:06:23,917 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a doubly linked list?' | topic=Doubly Linked List +2026-08-02 14:06:23,938 | INFO | dsa_chatbot.rag.retriever | Query='What is a doubly linked list?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:23,946 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain how a stack works' | topic=Stack +2026-08-02 14:06:23,968 | INFO | dsa_chatbot.rag.retriever | Query='Explain how a stack works' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:23,980 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is topological sort used for?' | topic=Topological Sort +2026-08-02 14:06:24,001 | INFO | dsa_chatbot.rag.retriever | Query='What is topological sort used for?' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:06:24,007 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: "Explain Dijkstra's Algorithm" | topic=Dijkstra's Algorithm +2026-08-02 14:06:24,029 | INFO | dsa_chatbot.rag.retriever | Query="Explain Dijkstra's Algorithm" | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:24,038 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'What is a segment tree?' | topic=Segment Tree +2026-08-02 14:06:24,065 | INFO | dsa_chatbot.rag.retriever | Query='What is a segment tree?' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:24,076 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'Explain the sliding window technique' | topic=Sliding Window Technique +2026-08-02 14:06:24,099 | INFO | dsa_chatbot.rag.retriever | Query='Explain the sliding window technique' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:24,113 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'What is the difference between BFS and DFS?' | topics=['Breadth-First Search', 'Depth-First Search'] +2026-08-02 14:06:24,134 | INFO | dsa_chatbot.rag.retriever | Query='Breadth-First Search' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:06:24,151 | INFO | dsa_chatbot.rag.retriever | Query='Depth-First Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:24,166 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Compare merge sort vs quick sort' | topics=['Merge Sort', 'Quick Sort'] +2026-08-02 14:06:24,190 | INFO | dsa_chatbot.rag.retriever | Query='Merge Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:24,215 | INFO | dsa_chatbot.rag.retriever | Query='Quick Sort' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:24,229 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Difference between a stack and a queue' | topics=['Queue', 'Stack'] +2026-08-02 14:06:24,253 | INFO | dsa_chatbot.rag.retriever | Query='Queue' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:06:24,281 | INFO | dsa_chatbot.rag.retriever | Query='Stack' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:24,320 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: "Which one is better, Prim's Algorithm or Kruskal's Algorithm, for MST?" | topics=["Kruskal's Algorithm (MST)", "Prim's Algorithm (MST)"] +2026-08-02 14:06:24,354 | INFO | dsa_chatbot.rag.retriever | Query="Kruskal's Algorithm (MST)" | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:06:24,380 | INFO | dsa_chatbot.rag.retriever | Query="Prim's Algorithm (MST)" | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:06:24,395 | INFO | dsa_chatbot.agents.router | Routed to COMPARISON: 'Compare AVL Tree and Binary Search Tree' | topics=['AVL Tree', 'Binary Search', 'Binary Search Tree'] +2026-08-02 14:06:24,419 | INFO | dsa_chatbot.rag.retriever | Query='AVL Tree' | candidates=5 | passed_threshold=5 | threshold=0.45 +2026-08-02 14:06:24,444 | INFO | dsa_chatbot.rag.retriever | Query='Binary Search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:24,465 | INFO | dsa_chatbot.rag.retriever | Query='Binary Search Tree' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 14:06:24,489 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'Who won the football World Cup in 2022?' +2026-08-02 14:06:24,503 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: "What's a good recipe for chocolate cake?" +2026-08-02 14:06:24,521 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'What is the capital of France?' +2026-08-02 14:06:24,533 | INFO | dsa_chatbot.agents.router | Routed to FOLLOWUP: 'Can you explain that again?' +2026-08-02 14:06:24,565 | INFO | dsa_chatbot.agents.followup | Follow-up query reuses existing conversation context (topics=['Merge Sort']); skipping fresh retrieval. +2026-08-02 14:06:24,566 | INFO | dsa_chatbot.agents.router | Routed to FOLLOWUP: 'Give me another example' +2026-08-02 14:06:24,590 | INFO | dsa_chatbot.agents.followup | Follow-up query reuses existing conversation context (topics=['Merge Sort']); skipping fresh retrieval. +2026-08-02 14:06:24,590 | INFO | dsa_chatbot.__main__ | Evaluation complete: {'total_questions': 20, 'route_accuracy': 1.0, 'avg_topic_recall': 1.0, 'retrieval_latency_ms_avg': 46.26, 'retrieval_latency_ms_p95': 101.1} +2026-08-02 16:00:13,397 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'explain binary search' | topic=Binary Search +2026-08-02 16:00:14,003 | INFO | dsa_chatbot.rag.retriever | Query='explain binary search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 16:00:15,631 | INFO | dsa_chatbot.llm.generate | LLM provider initialized: GroqProvider +2026-08-02 16:00:17,326 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=None | route=single_topic | topics=['Binary Search'] | session_id=None +2026-08-02 16:00:56,629 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: '2,3,1,6,5,4 find 5 this using this search step by step' +2026-08-02 16:00:57,493 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=None | route=out_of_scope | topics=[] | session_id=None +2026-08-02 16:07:37,423 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'explain binary search' | topic=Binary Search +2026-08-02 16:07:37,480 | INFO | dsa_chatbot.rag.retriever | Query='explain binary search' | candidates=1 | passed_threshold=1 | threshold=0.45 +2026-08-02 16:07:41,129 | INFO | dsa_chatbot.memory.mem0_client | Initializing mem0 (local chroma + huggingface embedder)... +2026-08-02 16:07:51,945 | ERROR | dsa_chatbot.memory.mem0_client | mem0 get_relevant_context failed for user_id=1 +Traceback (most recent call last): + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 110, in get_relevant_context + results = self._get_memory().search(query, user_id=user_id, limit=limit) + ~~~~~~~~~~~~~~~~^^ + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 72, in _get_memory + self._memory = Memory.from_config(_build_config_dict()) + ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 59, in from_config + return cls(config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 37, in __init__ + self.llm = LlmFactory.create(self.config.llm.provider, self.config.llm.config) + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\utils\factory.py", line 34, in create + return llm_instance(base_config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\llms\openai.py", line 28, in __init__ + self.client = OpenAI(api_key=api_key, base_url=base_url) + ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\openai\_client.py", line 135, in __init__ + raise OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + ) +openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable +2026-08-02 16:07:53,458 | INFO | dsa_chatbot.memory.mem0_client | Initializing mem0 (local chroma + huggingface embedder)... +2026-08-02 16:07:57,115 | ERROR | dsa_chatbot.memory.mem0_client | mem0 add_interaction failed for user_id=1 +Traceback (most recent call last): + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 93, in add_interaction + self._get_memory().add(messages, user_id=user_id, infer=self.infer) + ~~~~~~~~~~~~~~~~^^ + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 72, in _get_memory + self._memory = Memory.from_config(_build_config_dict()) + ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 59, in from_config + return cls(config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 37, in __init__ + self.llm = LlmFactory.create(self.config.llm.provider, self.config.llm.config) + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\utils\factory.py", line 34, in create + return llm_instance(base_config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\llms\openai.py", line 28, in __init__ + self.client = OpenAI(api_key=api_key, base_url=base_url) + ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\openai\_client.py", line 135, in __init__ + raise OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + ) +openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable +2026-08-02 16:07:57,120 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=1 | route=single_topic | topics=['Binary Search'] | session_id=1 +2026-08-02 16:31:21,411 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'explain binary search' | topic=Binary Search +2026-08-02 16:31:21,670 | INFO | dsa_chatbot.rag.retriever | Query='explain binary search' | candidates=1 | passed_threshold=1 | threshold=-1.00 +2026-08-02 16:31:24,341 | INFO | dsa_chatbot.memory.mem0_client | Initializing mem0 (local chroma + huggingface embedder)... +2026-08-02 16:31:29,511 | ERROR | dsa_chatbot.memory.mem0_client | mem0 get_relevant_context failed for user_id=1 +Traceback (most recent call last): + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 110, in get_relevant_context + results = self._get_memory().search(query, user_id=user_id, limit=limit) + ~~~~~~~~~~~~~~~~^^ + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 72, in _get_memory + self._memory = Memory.from_config(_build_config_dict()) + ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 59, in from_config + return cls(config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 37, in __init__ + self.llm = LlmFactory.create(self.config.llm.provider, self.config.llm.config) + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\utils\factory.py", line 34, in create + return llm_instance(base_config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\llms\openai.py", line 28, in __init__ + self.client = OpenAI(api_key=api_key, base_url=base_url) + ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\openai\_client.py", line 135, in __init__ + raise OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + ) +openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable +2026-08-02 16:31:30,363 | INFO | dsa_chatbot.llm.generate | LLM provider initialized: GroqProvider +2026-08-02 16:31:31,945 | INFO | dsa_chatbot.memory.mem0_client | Initializing mem0 (local chroma + huggingface embedder)... +2026-08-02 16:31:35,540 | ERROR | dsa_chatbot.memory.mem0_client | mem0 add_interaction failed for user_id=1 +Traceback (most recent call last): + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 93, in add_interaction + self._get_memory().add(messages, user_id=user_id, infer=self.infer) + ~~~~~~~~~~~~~~~~^^ + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 72, in _get_memory + self._memory = Memory.from_config(_build_config_dict()) + ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 59, in from_config + return cls(config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 37, in __init__ + self.llm = LlmFactory.create(self.config.llm.provider, self.config.llm.config) + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\utils\factory.py", line 34, in create + return llm_instance(base_config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\llms\openai.py", line 28, in __init__ + self.client = OpenAI(api_key=api_key, base_url=base_url) + ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\openai\_client.py", line 135, in __init__ + raise OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + ) +openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable +2026-08-02 16:31:35,545 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=1 | route=single_topic | topics=['Binary Search'] | session_id=2 +2026-08-02 16:32:22,442 | INFO | dsa_chatbot.agents.router | Routed to FOLLOWUP: 'in this array [65,32,43,23,78,96] how will you find 23 using this search' +2026-08-02 16:32:24,207 | INFO | dsa_chatbot.agents.followup | Follow-up query reuses existing conversation context (topics=['Binary Search', 'Linear Search']); skipping fresh retrieval. +2026-08-02 16:32:24,208 | INFO | dsa_chatbot.memory.mem0_client | Initializing mem0 (local chroma + huggingface embedder)... +2026-08-02 16:32:27,835 | ERROR | dsa_chatbot.memory.mem0_client | mem0 get_relevant_context failed for user_id=1 +Traceback (most recent call last): + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 110, in get_relevant_context + results = self._get_memory().search(query, user_id=user_id, limit=limit) + ~~~~~~~~~~~~~~~~^^ + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 72, in _get_memory + self._memory = Memory.from_config(_build_config_dict()) + ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 59, in from_config + return cls(config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 37, in __init__ + self.llm = LlmFactory.create(self.config.llm.provider, self.config.llm.config) + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\utils\factory.py", line 34, in create + return llm_instance(base_config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\llms\openai.py", line 28, in __init__ + self.client = OpenAI(api_key=api_key, base_url=base_url) + ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\openai\_client.py", line 135, in __init__ + raise OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + ) +openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable +2026-08-02 16:32:29,805 | INFO | dsa_chatbot.memory.mem0_client | Initializing mem0 (local chroma + huggingface embedder)... +2026-08-02 16:32:33,863 | ERROR | dsa_chatbot.memory.mem0_client | mem0 add_interaction failed for user_id=1 +Traceback (most recent call last): + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 93, in add_interaction + self._get_memory().add(messages, user_id=user_id, infer=self.infer) + ~~~~~~~~~~~~~~~~^^ + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 72, in _get_memory + self._memory = Memory.from_config(_build_config_dict()) + ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 59, in from_config + return cls(config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 37, in __init__ + self.llm = LlmFactory.create(self.config.llm.provider, self.config.llm.config) + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\utils\factory.py", line 34, in create + return llm_instance(base_config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\llms\openai.py", line 28, in __init__ + self.client = OpenAI(api_key=api_key, base_url=base_url) + ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\openai\_client.py", line 135, in __init__ + raise OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + ) +openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable +2026-08-02 16:32:33,869 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=1 | route=followup | topics=[] | session_id=2 +2026-08-02 16:33:48,648 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'compare this search with linear search' | topic=Linear Search +2026-08-02 16:33:48,715 | INFO | dsa_chatbot.rag.retriever | Query='compare this search with linear search' | candidates=1 | passed_threshold=1 | threshold=-1.00 +2026-08-02 16:33:48,716 | INFO | dsa_chatbot.memory.mem0_client | Initializing mem0 (local chroma + huggingface embedder)... +2026-08-02 16:33:52,390 | ERROR | dsa_chatbot.memory.mem0_client | mem0 get_relevant_context failed for user_id=1 +Traceback (most recent call last): + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 110, in get_relevant_context + results = self._get_memory().search(query, user_id=user_id, limit=limit) + ~~~~~~~~~~~~~~~~^^ + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 72, in _get_memory + self._memory = Memory.from_config(_build_config_dict()) + ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 59, in from_config + return cls(config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 37, in __init__ + self.llm = LlmFactory.create(self.config.llm.provider, self.config.llm.config) + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\utils\factory.py", line 34, in create + return llm_instance(base_config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\llms\openai.py", line 28, in __init__ + self.client = OpenAI(api_key=api_key, base_url=base_url) + ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\openai\_client.py", line 135, in __init__ + raise OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + ) +openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable +2026-08-02 16:33:54,095 | INFO | dsa_chatbot.memory.mem0_client | Initializing mem0 (local chroma + huggingface embedder)... +2026-08-02 16:33:58,206 | ERROR | dsa_chatbot.memory.mem0_client | mem0 add_interaction failed for user_id=1 +Traceback (most recent call last): + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 93, in add_interaction + self._get_memory().add(messages, user_id=user_id, infer=self.infer) + ~~~~~~~~~~~~~~~~^^ + File "D:\scratch\chatbot\chatbot\memory\mem0_client.py", line 72, in _get_memory + self._memory = Memory.from_config(_build_config_dict()) + ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 59, in from_config + return cls(config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\memory\main.py", line 37, in __init__ + self.llm = LlmFactory.create(self.config.llm.provider, self.config.llm.config) + ~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\utils\factory.py", line 34, in create + return llm_instance(base_config) + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\mem0\llms\openai.py", line 28, in __init__ + self.client = OpenAI(api_key=api_key, base_url=base_url) + ~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "D:\scratch\chatbot\chatbot\venv\Lib\site-packages\openai\_client.py", line 135, in __init__ + raise OpenAIError( + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable" + ) +openai.OpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable +2026-08-02 16:33:58,212 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=1 | route=single_topic | topics=['Linear Search'] | session_id=2 +2026-08-02 19:53:48,973 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'how are you' +2026-08-02 19:53:49,894 | INFO | dsa_chatbot.llm.generate | LLM provider initialized: GroqProvider +2026-08-02 19:53:51,450 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=None | route=out_of_scope | topics=[] | session_id=None +2026-08-02 22:34:22,716 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: "explain dijikstra's algorithm" | topic=Dijkstra's Algorithm +2026-08-02 22:34:23,350 | INFO | dsa_chatbot.rag.retriever | Query="explain dijikstra's algorithm" | candidates=1 | passed_threshold=1 | threshold=-1.00 +2026-08-02 22:34:24,785 | INFO | dsa_chatbot.llm.generate | LLM provider initialized: GroqProvider +2026-08-02 22:34:26,608 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=None | route=single_topic | topics=["Dijkstra's Algorithm"] | session_id=None +2026-08-02 22:41:13,334 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'explain bfs' | topic=Breadth-First Search +2026-08-02 22:41:13,572 | INFO | dsa_chatbot.rag.retriever | Query='explain bfs' | candidates=5 | passed_threshold=5 | threshold=-1.00 +2026-08-02 22:41:14,419 | INFO | dsa_chatbot.llm.generate | LLM provider initialized: GroqProvider +2026-08-02 22:41:15,798 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=None | route=single_topic | topics=['Breadth-First Search'] | session_id=None +2026-08-02 22:41:56,906 | INFO | dsa_chatbot.agents.router | Routed to OUT_OF_SCOPE: 'compare with other graph parsing algorithms' +2026-08-02 22:41:57,470 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=None | route=out_of_scope | topics=[] | session_id=None +2026-08-02 22:51:55,471 | INFO | dsa_chatbot.agents.router | Routed to SINGLE_TOPIC: 'explain dfs' | topic=Depth-First Search +2026-08-02 22:51:55,687 | INFO | dsa_chatbot.rag.retriever | Query='explain dfs' | candidates=1 | passed_threshold=1 | threshold=-1.00 +2026-08-02 22:51:56,559 | INFO | dsa_chatbot.llm.generate | LLM provider initialized: GroqProvider +2026-08-02 22:51:57,814 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=None | route=single_topic | topics=['Depth-First Search'] | session_id=None +2026-08-02 22:52:27,161 | INFO | dsa_chatbot.agents.router | Routed to FOLLOWUP: 'compare it with other graph traversal algorithms' +2026-08-02 22:52:28,250 | INFO | dsa_chatbot.agents.followup | Follow-up query reuses existing conversation context (topics=['Depth-First Search', 'Stack']); skipping fresh retrieval. +2026-08-02 22:52:29,862 | INFO | dsa_chatbot.chat_service | Chat turn complete | user_id=None | route=followup | topics=[] | session_id=None diff --git a/logs/logger.py b/logs/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..82a81cfe18366aac27553bedeb1a6fbd89c1813f --- /dev/null +++ b/logs/logger.py @@ -0,0 +1,48 @@ +""" +Centralized logging setup. + +Every module does `from logs.logger import get_logger` and +`logger = get_logger(__name__)` rather than configuring logging itself. +Writes to both console and logs/app.log (paths from config.py). +""" + +import logging +import os +import sys + +import config + +_CONFIGURED = False + + +def _configure_root(): + global _CONFIGURED + if _CONFIGURED: + return + + os.makedirs(config.LOG_DIR, exist_ok=True) + + root = logging.getLogger("dsa_chatbot") + root.setLevel(config.LOG_LEVEL) + root.propagate = False + + formatter = logging.Formatter( + "%(asctime)s | %(levelname)-8s | %(name)s | %(message)s" + ) + + if not root.handlers: + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setFormatter(formatter) + root.addHandler(stream_handler) + + file_handler = logging.FileHandler(config.LOG_FILE_PATH, encoding="utf-8") + file_handler.setFormatter(formatter) + root.addHandler(file_handler) + + _CONFIGURED = True + + +def get_logger(name: str) -> logging.Logger: + """Returns a named logger nested under the shared 'dsa_chatbot' root.""" + _configure_root() + return logging.getLogger(f"dsa_chatbot.{name}") diff --git a/memory/__init__.py b/memory/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/memory/mem0_client.py b/memory/mem0_client.py new file mode 100644 index 0000000000000000000000000000000000000000..1d215f0af3b2991af0fedff517af3d3d8fdbd53b --- /dev/null +++ b/memory/mem0_client.py @@ -0,0 +1,137 @@ +""" +mem0 client wrapper. + +Responsibility boundary (do not blur this): + - mem0 -> user preferences + long-term conversational memory + (e.g. "prefers beginner explanations", "likes step-by-step examples") + - ChromaDB -> DSA knowledge embeddings (rag/retriever.py) — NEVER touched here + - SQLite -> raw message/session transcripts (history/) — NEVER touched here + +mem0 runs in local/open-source mode: a local Chroma vector store (separate +collection from the knowledge base) + a local sentence-transformers +embedder (same model as the KB for consistency, configurable independently +if needed). + +Fact extraction (infer=True) would use an LLM to summarize what to +remember. To avoid depending on the (expensive, 4-bit-quantized) local +Mistral model just to log a preference, this wrapper uses infer=False by +default — messages are stored as-is and semantic search over them still +works for retrieval. Set infer=True if you want mem0 to run LLM-based +fact extraction using the same provider configured in llm/generate.py. +""" + +import os + +import config +from logs.logger import get_logger + +logger = get_logger(__name__) + +_MEM0_COLLECTION_NAME = "user_preferences_memory" + + +def _build_config_dict() -> dict: + os.makedirs(config.MEM0_LOCAL_STORAGE_DIR, exist_ok=True) + return { + "vector_store": { + "provider": "chroma", + "config": { + "collection_name": _MEM0_COLLECTION_NAME, + "path": os.path.join(config.MEM0_LOCAL_STORAGE_DIR, "chroma_db"), + }, + }, + "embedder": { + "provider": "huggingface", + "config": { + "model": f"sentence-transformers/{config.EMBEDDING_MODEL_NAME}", + }, + }, + "history_db_path": os.path.join(config.MEM0_LOCAL_STORAGE_DIR, "mem0_history.db"), + } + + +class Mem0Client: + """ + Thin wrapper around mem0.Memory scoped to a single responsibility: + user preferences and long-term conversational memory. + """ + + def __init__(self, infer: bool = False): + self.infer = infer + self._memory = None + self._enabled = config.USE_MEM0 + + if not self._enabled: + logger.info("USE_MEM0 is False — Mem0Client will be a no-op.") + + def _get_memory(self): + if self._memory is None: + from mem0 import Memory + + logger.info("Initializing mem0 (local chroma + huggingface embedder)...") + self._memory = Memory.from_config(_build_config_dict()) + return self._memory + + def add_interaction(self, user_id: str, user_message: str, assistant_message: str = None) -> None: + """ + Store a turn of conversation for long-term memory purposes (NOT + transcript storage — that's history/service.py + SQLite). + + Only call this for things worth remembering long-term (preferences, + recurring topics of interest, stated skill level) — not every raw + message. The router/chat layer decides when this is worth calling; + this wrapper doesn't filter content itself. + """ + if not self._enabled: + return + + messages = [{"role": "user", "content": user_message}] + if assistant_message: + messages.append({"role": "assistant", "content": assistant_message}) + + try: + self._get_memory().add(messages, user_id=user_id, infer=self.infer) + except Exception: + logger.exception("mem0 add_interaction failed for user_id=%s", user_id) + + def get_relevant_context(self, user_id: str, query: str, limit: int = 5) -> list: + """ + Returns a list of plain-text memory strings relevant to `query`, + ready to hand to llm/prompts.py's mem0_context parameter. + + Returns [] if mem0 is disabled, the user has no memories yet, or a + lookup error occurs — callers should treat that as "no memory + context available", not as an error condition. + """ + if not self._enabled: + return [] + + try: + results = self._get_memory().search(query, user_id=user_id, limit=limit) + except Exception: + logger.exception("mem0 get_relevant_context failed for user_id=%s", user_id) + return [] + + # mem0's search() returns {"results": [{"memory": "...", "score": ...}, ...]} + # in v2, or a plain list in some versions — handle both. + items = results.get("results", results) if isinstance(results, dict) else results + return [item.get("memory", "") for item in items if item.get("memory")] + + def delete_all_for_user(self, user_id: str) -> None: + """Useful for account deletion / a 'forget me' feature.""" + if not self._enabled: + return + try: + self._get_memory().delete_all(user_id=user_id) + except Exception: + logger.exception("mem0 delete_all_for_user failed for user_id=%s", user_id) + + +_CLIENT_SINGLETON = None + + +def get_mem0_client() -> Mem0Client: + global _CLIENT_SINGLETON + if _CLIENT_SINGLETON is None: + _CLIENT_SINGLETON = Mem0Client() + return _CLIENT_SINGLETON diff --git a/rag/__init__.py b/rag/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/rag/build_index.py b/rag/build_index.py new file mode 100644 index 0000000000000000000000000000000000000000..3e423c51cc797a0ee51d54060ac3d12d76c7f7c8 --- /dev/null +++ b/rag/build_index.py @@ -0,0 +1,211 @@ +""" +RAG index builder. + +Reads every knowledge-base .txt file, splits it into semantic chunks based +on the existing structural headings (DEFINITION, TIME_COMPLEXITY, +SPACE_COMPLEXITY, USE_WHEN, EXAMPLE) rather than arbitrary fixed-size +windows, attaches per-topic metadata from topic_metadata.json, embeds each +chunk with sentence-transformers, and stores everything in a persistent +ChromaDB collection. + +This script NEVER modifies the source .txt files. They are treated as +immutable. Run it whenever the knowledge base or topic_metadata.json +changes: + + python -m rag.build_index +""" + +import json +import os +import re +import sys + +import chromadb +from sentence_transformers import SentenceTransformer + +# Allow running as a standalone script (python rag/build_index.py) as well +# as a module (python -m rag.build_index). +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import config +from logs.logger import get_logger + +logger = get_logger(__name__) + +# Matches a section header line, e.g. "DEFINITION:" or "TIME_COMPLEXITY:" +_HEADER_PATTERN = re.compile( + r"^(" + "|".join(config.KB_SECTION_HEADERS) + r"):\s*$", re.MULTILINE +) + +_TOPIC_LINE_PATTERN = re.compile(r"^TOPIC:\s*(.+)$", re.MULTILINE) + + +def _parse_sections(file_text: str) -> dict: + """ + Split a KB file's text into {section_name: section_body} using the + structural headers defined in config.KB_SECTION_HEADERS. + + Any text before the first recognized header (e.g. a "TOPIC:" line) is + ignored here — the topic name is expected to come from + topic_metadata.json, not parsed out of the file body. + """ + matches = list(_HEADER_PATTERN.finditer(file_text)) + sections = {} + + for i, match in enumerate(matches): + header = match.group(1) + start = match.end() + end = matches[i + 1].start() if i + 1 < len(matches) else len(file_text) + body = file_text[start:end].strip() + if body: + sections[header] = body + + return sections + + +def _load_topic_metadata() -> dict: + if not os.path.exists(config.TOPIC_METADATA_PATH): + raise FileNotFoundError( + f"topic_metadata.json not found at {config.TOPIC_METADATA_PATH}. " + "Every knowledge-base file must have a matching entry." + ) + with open(config.TOPIC_METADATA_PATH, "r", encoding="utf-8") as f: + return json.load(f) + + +def _list_kb_files() -> list: + if not os.path.isdir(config.KNOWLEDGE_BASE_DIR): + raise FileNotFoundError( + f"Knowledge base directory not found: {config.KNOWLEDGE_BASE_DIR}" + ) + return sorted( + f + for f in os.listdir(config.KNOWLEDGE_BASE_DIR) + if f.endswith(".txt") + ) + + +def build_chunks() -> list: + """ + Returns a list of dicts, each representing one chunk ready for embedding: + { + "id": "binary_search.txt::DEFINITION", + "text": "...", + "metadata": { + "topic": "...", + "category": "...", + "section": "...", + "difficulty": "...", + "source": "binary_search.txt", + "chunk_id": 0 + } + } + """ + topic_metadata = _load_topic_metadata() + kb_files = _list_kb_files() + + missing_metadata = [f for f in kb_files if f not in topic_metadata] + if missing_metadata: + logger.warning( + "The following KB files have no topic_metadata.json entry and " + "will be skipped: %s", + missing_metadata, + ) + + chunks = [] + chunk_counter = 0 + + for filename in kb_files: + if filename not in topic_metadata: + continue + + file_path = os.path.join(config.KNOWLEDGE_BASE_DIR, filename) + with open(file_path, "r", encoding="utf-8") as f: + file_text = f.read() + + sections = _parse_sections(file_text) + if not sections: + logger.warning( + "No recognized section headers (%s) found in %s. Skipping.", + config.KB_SECTION_HEADERS, + filename, + ) + continue + + meta = topic_metadata[filename] + + for section_name, section_body in sections.items(): + chunk = { + "id": f"{filename}::{section_name}", + "text": f"{meta.get('topic', filename)} - {section_name}: {section_body}", + "metadata": { + "topic": meta.get("topic", ""), + "category": meta.get("category", ""), + "section": section_name, + "difficulty": meta.get("difficulty", ""), + "source": filename, + "prerequisites": ", ".join(meta.get("prerequisites", [])), + "chunk_id": chunk_counter, + }, + } + chunks.append(chunk) + chunk_counter += 1 + + logger.info("Built %d chunks from %d knowledge-base files.", len(chunks), len(kb_files)) + return chunks + + +def build_index(reset: bool = True) -> None: + """ + Embed all chunks and persist them into the ChromaDB collection. + + Args: + reset: if True, drops and recreates the collection so the index is + rebuilt from scratch (safe default whenever the KB or + metadata changes). If False, upserts into the existing + collection. + """ + os.makedirs(config.CHROMA_DB_DIR, exist_ok=True) + + logger.info("Loading embedding model: %s", config.EMBEDDING_MODEL_NAME) + embedder = SentenceTransformer(config.EMBEDDING_MODEL_NAME) + + client = chromadb.PersistentClient(path=config.CHROMA_DB_DIR) + + if reset: + try: + client.delete_collection(config.CHROMA_COLLECTION_NAME) + logger.info("Existing collection '%s' deleted.", config.CHROMA_COLLECTION_NAME) + except Exception: + pass # collection didn't exist yet + + collection = client.get_or_create_collection(name=config.CHROMA_COLLECTION_NAME) + + chunks = build_chunks() + if not chunks: + logger.warning("No chunks produced — index will be empty.") + return + + texts = [c["text"] for c in chunks] + ids = [c["id"] for c in chunks] + metadatas = [c["metadata"] for c in chunks] + + logger.info("Embedding %d chunks...", len(texts)) + embeddings = embedder.encode(texts, show_progress_bar=False).tolist() + + collection.upsert( + ids=ids, + embeddings=embeddings, + documents=texts, + metadatas=metadatas, + ) + + logger.info( + "Index build complete. Collection '%s' now has %d chunks.", + config.CHROMA_COLLECTION_NAME, + collection.count(), + ) + + +if __name__ == "__main__": + build_index(reset=True) diff --git a/rag/generate.py b/rag/generate.py new file mode 100644 index 0000000000000000000000000000000000000000..6dd0f61529180ed52eb02efa7dc3733d5fc657b7 --- /dev/null +++ b/rag/generate.py @@ -0,0 +1,147 @@ +""" +Batch-generates DSA knowledge base .txt files using the Groq API. + +Setup: + pip install groq python-dotenv + Create a .env file with: GROQ_API_KEY=your_key_here + +Run: + python generate_knowledge_base.py + +Output: + Files are written to ./knowledge_base/.txt +""" + +import os +import re +import time +from dotenv import load_dotenv +from groq import Groq + +load_dotenv() + +client = Groq(api_key=os.environ.get("GROQ_API_KEY")) +MODEL = "llama-3.3-70b-versatile" # swap for another Groq-hosted model if you prefer +OUTPUT_DIR = "knowledge_base" +DELAY_BETWEEN_CALLS = 1.5 # seconds, be nice to rate limits + +os.makedirs(OUTPUT_DIR, exist_ok=True) + +# ── PROMPT TEMPLATE ───────────────────────────────────────────────────────── +PROMPT_TEMPLATE = """You are writing a knowledge base entry for a DSA (Data Structures & Algorithms) +tutoring RAG chatbot. Write in your own original words — do NOT copy verbatim from +GeeksforGeeks, textbooks, or any other source. Paraphrase and synthesize instead. +Keep explanations beginner-friendly but technically accurate. + +Generate the entry for this topic: {topic} + +Output ONLY in the exact following plain text structure, no markdown formatting, +no extra commentary before or after: + +TOPIC: {topic} + +DEFINITION: +<2-3 sentence clear definition of what it is and what problem it solves> + +TIME_COMPLEXITY: + + +SPACE_COMPLEXITY: + + +USE_WHEN: +<1-2 sentences: scenarios/conditions where this is the right tool> + +AVOID_WHEN: +<1-2 sentences: scenarios where this is a poor choice, and what to use instead> + +EXAMPLE: + + +REAL_WORLD_ANALOGY: +<1-2 sentence relatable comparison to something non-technical> + +SOURCE_NOTE: + +""" + +# ── TOPIC LIST ─────────────────────────────────────────────────────────────── +TOPICS = [ + # Arrays & Searching + "Linear Search", "Binary Search", "Two Pointer Technique", "Sliding Window Technique", + # Sorting + "Bubble Sort", "Selection Sort", "Insertion Sort", "Merge Sort", "Quick Sort", + "Heap Sort", "Counting Sort","radix sort","bucket sort", + # Linked Lists + "Singly Linked List", "Doubly Linked List", "Circular Linked List", + # Stacks & Queues + "Stack", "Queue", "Circular Queue", "Deque", "Monotonic Stack", + # Trees + "Binary Tree", "Binary Search Tree", "AVL Tree", "Heap (Min/Max)", "Trie", + "Segment Tree", "Fenwick Tree (Binary Indexed Tree)", + # Graphs + "BFS", "DFS", "Dijkstra's Algorithm", "Bellman-Ford Algorithm", + "Floyd-Warshall Algorithm", "Kruskal's Algorithm (MST)", "Prim's Algorithm (MST)", + "Topological Sort", "Union-Find (Disjoint Set)", + # Dynamic Programming + "Dynamic Programming Overview", "0/1 Knapsack", "Longest Common Subsequence", + "Longest Increasing Subsequence", "Coin Change Problem", "Edit Distance", + # Greedy + "Greedy Algorithms Overview", "Activity Selection Problem", "Fractional Knapsack", + # Hashing + "Hash Table", "Hash Set vs Hash Map", + # Recursion & Backtracking + "Recursion Overview", "Backtracking Overview", "N-Queens Problem", + "Subset Sum / Power Set Generation", + # Bit Manipulation + "Bit Manipulation Basics", "XOR Tricks", +] + + +def to_snake_case(topic: str) -> str: + """Converts a topic name into a safe snake_case filename.""" + cleaned = re.sub(r"[^\w\s-]", "", topic) # strip punctuation like ()/,' + cleaned = re.sub(r"\s+", "_", cleaned.strip()) # spaces -> underscores + return cleaned.lower() + + +def generate_entry(topic: str) -> str: + prompt = PROMPT_TEMPLATE.format(topic=topic) + response = client.chat.completions.create( + model=MODEL, + messages=[{"role": "user", "content": prompt}], + temperature=0.4, + max_tokens=1024, + ) + return response.choices[0].message.content.strip() + + +def main(): + print(f"Generating {len(TOPICS)} knowledge base files into ./{OUTPUT_DIR}/\n") + + for i, topic in enumerate(TOPICS, start=1): + filename = to_snake_case(topic) + ".txt" + filepath = os.path.join(OUTPUT_DIR, filename) + + if os.path.exists(filepath): + print(f"[{i}/{len(TOPICS)}] SKIP (already exists): {filename}") + continue + + print(f"[{i}/{len(TOPICS)}] Generating: {topic} -> {filename}") + try: + content = generate_entry(topic) + with open(filepath, "w", encoding="utf-8") as f: + f.write(content) + except Exception as e: + print(f" ERROR generating '{topic}': {e}") + + time.sleep(DELAY_BETWEEN_CALLS) + + print("\nDone. Review files in ./knowledge_base/ before indexing into ChromaDB.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/rag/knowledge_base/01_knapsack.txt b/rag/knowledge_base/01_knapsack.txt new file mode 100644 index 0000000000000000000000000000000000000000..c3f9322fb31a5e46b6f41ea5e395ca6bcd9a777a --- /dev/null +++ b/rag/knowledge_base/01_knapsack.txt @@ -0,0 +1,37 @@ +TOPIC: 0/1 Knapsack + +DEFINITION: +The 0/1 Knapsack problem is a classic problem in computer science where you have a set of items, each with a weight and a value, and you need to determine the subset of items to include in a knapsack of limited capacity such that the total value is maximized. This problem solves the issue of optimizing the selection of items when resources are limited. The "0/1" refers to the fact that each item is either included or excluded from the knapsack, with no partial inclusions allowed. + +TIME_COMPLEXITY: +The time complexity of the 0/1 Knapsack problem is O(nW), where n is the number of items and W is the capacity of the knapsack. This is because the problem is typically solved using dynamic programming, which involves filling up a 2D table of size n x W. + +SPACE_COMPLEXITY: +The space complexity of the 0/1 Knapsack problem is O(nW), which is used to store the 2D table that keeps track of the maximum value that can be obtained with a given capacity and number of items. + +USE_WHEN: +The 0/1 Knapsack problem is useful when you need to make a selection from a set of items, each with its own value and cost, and you have limited resources. This problem is particularly relevant in scenarios such as budgeting, resource allocation, and investment decisions. + +AVOID_WHEN: +You should avoid using the 0/1 Knapsack problem when the items can be divided or when the cost of an item is not directly proportional to its value, in which case other problems like the Fractional Knapsack problem may be more suitable. + +EXAMPLE: +Suppose we have a knapsack with a capacity of 5 units and the following items: + - Item A: weight 2, value 10 + - Item B: weight 3, value 8 + - Item C: weight 1, value 5 +We can solve this problem using dynamic programming as follows: + - Initialize a table with zeros: + [0, 0, 0, 0, 0, 0] + - Fill up the table: + - For Item A: [0, 0, 10, 10, 10, 10] + - For Item B: [0, 0, 10, 10, 10, 18] + - For Item C: [0, 5, 10, 15, 15, 18] + The maximum value we can get is 18, which is obtained by including Item B and Item C: + +CHECKMARK: + +REAL_WORLD_ANALOGY: +The 0/1 Knapsack problem is similar to going on a trip and having to pack a suitcase with limited space, where you need to decide which items to bring and which to leave behind in order to maximize your overall satisfaction. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/activity_selection_problem.txt b/rag/knowledge_base/activity_selection_problem.txt new file mode 100644 index 0000000000000000000000000000000000000000..f66c01fb5bc62730e08a5eec6d6a580e1a883ddc --- /dev/null +++ b/rag/knowledge_base/activity_selection_problem.txt @@ -0,0 +1,41 @@ +TOPIC: Activity Selection Problem + +DEFINITION: The Activity Selection Problem is a classic problem in computer science and operations research that involves selecting the maximum number of activities that can be performed by a single person or resource, given a set of activities with start and end times. The goal is to choose the activities that do not conflict with each other, meaning that the end time of one activity is less than or equal to the start time of the next activity. This problem is useful in a variety of real-world scenarios, such as scheduling meetings or tasks. + +TIME_COMPLEXITY: O(n log n) due to the sorting of activities by their end times, which is necessary to apply the greedy algorithm that solves the problem. + +SPACE_COMPLEXITY: O(n) for storing the list of activities, where n is the number of activities. + +USE_WHEN: This problem is the right tool when you need to schedule a set of tasks or activities that have specific start and end times, and you want to maximize the number of activities that can be performed without any conflicts. It is particularly useful when the activities are mutually exclusive, meaning that only one activity can be performed at a time. + +AVOID_WHEN: This problem is a poor choice when the activities are not mutually exclusive, or when there are additional constraints that need to be considered, such as resource availability or priority levels. In such cases, more advanced scheduling algorithms or techniques, such as dynamic programming or integer programming, may be more suitable. + +EXAMPLE: +Consider the following set of activities with their start and end times: + [1] (1, 4) + [2] (3, 5) + [3] (0, 6) + [4] (5, 7) + [5] (3, 8) + [6] (5, 9) + [7] (6, 10) +First, sort the activities by their end times: + [1] (1, 4) + [2] (3, 5) + [4] (5, 7) + [7] (6, 10) + [5] (3, 8) + [6] (5, 9) + [3] (0, 6) +Then, apply the greedy algorithm: + Select [1] (1, 4) + Select [4] (5, 7) + Select [7] (6, 10) is not selected because it conflicts with [4] +The final selected activities are: + [1] (1, 4) + [4] (5, 7) + Checkmark: The maximum number of non-conflicting activities is 2. + +REAL_WORLD_ANALOGY: The Activity Selection Problem is similar to scheduling meetings in a conference room, where you want to maximize the number of meetings that can be held without any conflicts or overlaps. + +SOURCE_NOTE: Concepts referenced from general knowledge of greedy algorithms and scheduling problems. \ No newline at end of file diff --git a/rag/knowledge_base/avl_tree.txt b/rag/knowledge_base/avl_tree.txt new file mode 100644 index 0000000000000000000000000000000000000000..046f6ff4faa8db2dee63408dc0df337684bb7134 --- /dev/null +++ b/rag/knowledge_base/avl_tree.txt @@ -0,0 +1,63 @@ +TOPIC: AVL Tree + +DEFINITION: +An AVL tree is a self-balancing binary search tree that ensures the height of the tree remains relatively small by rotating nodes when the balance factor becomes too large, thus solving the problem of maintaining efficient search, insertion, and deletion operations. This balance is crucial for preventing the tree from becoming skewed, which would lead to reduced performance. + +TIME_COMPLEXITY: +The time complexity for search, insertion, and deletion in an AVL tree is O(log n) in all cases, as the tree's height is kept logarithmic by the self-balancing mechanism. + +SPACE_COMPLEXITY: +The space complexity of an AVL tree is O(n), where n is the number of nodes in the tree, as each node requires a constant amount of space to store its value, child pointers, and balance factor. + +USE_WHEN: +Use an AVL tree when you need to frequently insert, delete, and search for elements in a dataset while maintaining an efficient order, such as in database indexing or file systems. This data structure is particularly useful when the dataset is constantly changing. + +AVOID_WHEN: +Avoid using an AVL tree when the dataset is mostly static or when memory is extremely limited, as the overhead of maintaining the balance factor and performing rotations can be unnecessary or too costly; in such cases, a simpler binary search tree or even an array might be more suitable. + +EXAMPLE: +Let's insert elements 5, 2, 8, 3, 9 into an AVL tree step by step: + - Start with an empty tree. + - Insert 5: [5] + - Insert 2: [5] + / + [2] + - Insert 8: [5] + / \ + [2] [8] + - Insert 3: Since 3 is between 2 and 5, we insert it as the right child of 2. + [5] + / \ + [2] [8] + \ + [3] + - Insert 9: The tree becomes unbalanced, so we rotate: + [5] + / \ + [2] [8] + \ \ + [3] [9] + After rotation: + [5] + / \ + [2] [8] + \ / \ + [3] [9] + Final balanced tree: + [5] + / \ + [2] [8] + \ \ + [3] [9] + Result: + The final balanced AVL tree is: + [5] + / \ + [2] [8] + \ \ + [3] [9] + +REAL_WORLD_ANALOGY: +An AVL tree can be thought of like a librarian who constantly rearranges books on shelves to ensure that the shelves remain balanced and it's easy to find any book, similar to how the tree balances itself to maintain efficient data retrieval. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/backtracking_overview.txt b/rag/knowledge_base/backtracking_overview.txt new file mode 100644 index 0000000000000000000000000000000000000000..1c693b49cf256c8af12f48d2ee9635faa232f008 --- /dev/null +++ b/rag/knowledge_base/backtracking_overview.txt @@ -0,0 +1,42 @@ +TOPIC: Backtracking Overview + +DEFINITION: +Backtracking is a problem-solving strategy used in algorithms to find a solution by exploring all possible options in a systematic way. It involves recursively trying different paths until a valid solution is found, and if a path does not lead to a solution, it backtracks to the previous step and tries another path. This approach is particularly useful for solving constraint satisfaction problems. + +TIME_COMPLEXITY: +The time complexity of backtracking algorithms can vary, but in the worst case, it is typically O(n!), where n is the number of options to choose from, because it may need to explore all possible combinations. The average and best cases depend on the specific problem and the order in which options are explored. + +SPACE_COMPLEXITY: +The space complexity of backtracking algorithms is usually O(n), where n is the maximum depth of the recursion tree, because that's how much space is needed to store the current path being explored. + +USE_WHEN: +Backtracking is the right tool to use when dealing with problems that have multiple possible solutions and require exploring all options in a systematic way, such as puzzles, scheduling, or resource allocation problems. It's particularly useful when the problem can be broken down into smaller sub-problems and solved recursively. + +AVOID_WHEN: +Backtracking is a poor choice for problems that have a very large solution space or require finding the optimal solution quickly, because it can be slow and inefficient. In such cases, other algorithms like dynamic programming or greedy algorithms may be more suitable. + +EXAMPLE: +Suppose we want to find all possible ways to arrange the letters A, B, and C. We can use backtracking to solve this problem: + Start with an empty string: [] + Try adding A: [A] + Try adding B: [A, B] + Try adding C: [A, B, C] + Try adding C: [A, C] + Try adding B: [A, C, B] + Try adding B: [B] + Try adding A: [B, A] + Try adding C: [B, A, C] + Try adding C: [B, C] + Try adding A: [B, C, A] + Try adding C: [C] + Try adding A: [C, A] + Try adding B: [C, A, B] + Try adding B: [C, B] + Try adding A: [C, B, A] +The final solutions are: [A, B, C] , [A, C, B] , [B, A, C] , [B, C, A] , [C, A, B] , [C, B, A] + +REAL_WORLD_ANALOGY: +Backtracking is similar to trying different routes to reach a destination, where you explore each path until you find the one that leads you to your goal, and if a path doesn't work out, you go back to the previous intersection and try another route. + +SOURCE_NOTE: +Concepts referenced from general knowledge of algorithms and problem-solving strategies. \ No newline at end of file diff --git a/rag/knowledge_base/bellman-ford_algorithm.txt b/rag/knowledge_base/bellman-ford_algorithm.txt new file mode 100644 index 0000000000000000000000000000000000000000..c265464a76f5d1f7f57655994d578be6f44d4ec8 --- /dev/null +++ b/rag/knowledge_base/bellman-ford_algorithm.txt @@ -0,0 +1,49 @@ +TOPIC: Bellman-Ford Algorithm + +DEFINITION: The Bellman-Ford algorithm is a graph search algorithm that finds the shortest path from a source vertex to all other vertices in a weighted graph. It can handle negative weight edges and can detect negative weight cycles, making it a versatile tool for solving various graph-related problems. This algorithm is particularly useful for finding the shortest path in graphs where the edge weights can be negative. + +TIME_COMPLEXITY: The time complexity of the Bellman-Ford algorithm is O(VE), where V is the number of vertices and E is the number of edges, because it relaxes all edges V-1 times. + +SPACE_COMPLEXITY: The space complexity is O(V), as the algorithm needs to store the distance to each vertex from the source vertex. + +USE_WHEN: The Bellman-Ford algorithm is the right tool when you need to find the shortest path in a graph with negative weight edges, or when you need to detect negative weight cycles. It's particularly useful in scenarios where other algorithms like Dijkstra's may fail due to negative weights. + +AVOID_WHEN: You should avoid using the Bellman-Ford algorithm when the graph is very large and dense, as its O(VE) time complexity can be slow, and in such cases, more efficient algorithms like Dijkstra's or A* may be more suitable for graphs with non-negative weights. + +EXAMPLE: + Let's consider a graph with 4 vertices (A, B, C, D) and 5 edges: + A -> B (weight: -1) + A -> C (weight: 4) + B -> C (weight: 3) + B -> D (weight: 2) + C -> D (weight: 2) + Initially, the distance to all vertices is infinity, except for A which is 0. + A: 0 + B: inf + C: inf + D: inf + After the first iteration: + A: 0 + B: -1 + C: inf + D: inf + After the second iteration: + A: 0 + B: -1 + C: 2 + D: inf + After the third iteration: + A: 0 + B: -1 + C: 2 + D: 1 + After the fourth iteration, no changes occur, so we stop. + The shortest distances are: + A to B: -1 + A to C: 2 + A to D: 1 + Checkmark: The shortest path from A to D is A -> B -> D with a total weight of 1. + +REAL_WORLD_ANALOGY: The Bellman-Ford algorithm can be thought of as finding the cheapest route to travel between cities, where the cost of traveling between cities can sometimes be negative, such as getting a refund for a part of the journey. + +SOURCE_NOTE: Concepts referenced from general knowledge of graph algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/bfs.txt b/rag/knowledge_base/bfs.txt new file mode 100644 index 0000000000000000000000000000000000000000..f86cfc699c72df52c6dbc0fd607265f6d979c64c --- /dev/null +++ b/rag/knowledge_base/bfs.txt @@ -0,0 +1,29 @@ +TOPIC: BFS + +DEFINITION: +Breadth-First Search (BFS) is a graph traversal algorithm that explores all the nodes at a given depth level before moving on to the next level, solving the problem of finding the shortest path between two nodes in an unweighted graph. It's particularly useful for searching graphs or trees level by level, starting from a given source node. This approach ensures that the algorithm visits all the nodes closest to the source before moving further away. + +TIME_COMPLEXITY: +The time complexity of BFS is O(V + E), where V represents the number of vertices (nodes) and E represents the number of edges, because in the worst case, the algorithm visits every node and edge once. + +SPACE_COMPLEXITY: +The space complexity of BFS is O(V), as the algorithm uses a queue to store nodes to be visited, and in the worst case, the queue can hold all the nodes at the deepest level. + +USE_WHEN: +BFS is the right tool when you need to find the shortest path in an unweighted graph or when you want to traverse a graph level by level, such as in web crawlers or social network friend suggestions. It's particularly useful when the graph is very large and you want to avoid the overhead of more complex algorithms. + +AVOID_WHEN: +BFS is a poor choice when dealing with very deep graphs or trees, as it can be slow and memory-intensive, and in such cases, a depth-first search (DFS) or more specialized algorithms like Dijkstra's or A* might be more suitable. + +EXAMPLE: +Consider a graph with nodes A, B, C, D, and E, where A is connected to B and C, B is connected to D, and C is connected to E. A BFS traversal starting from A would visit the nodes in the following order: + A + -> B, C + -> D, E +The final result is a visited set {A, B, C, D, E} + +REAL_WORLD_ANALOGY: +BFS can be thought of as exploring a city by first visiting all the neighboring houses, then moving on to the next block, and so on, similar to how a fire department might search a neighborhood. + +SOURCE_NOTE: +Concepts referenced from general knowledge of graph traversal algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/binary_search.txt b/rag/knowledge_base/binary_search.txt new file mode 100644 index 0000000000000000000000000000000000000000..eda52aea6aa5dd1d5c9f16ca470227031f413544 --- /dev/null +++ b/rag/knowledge_base/binary_search.txt @@ -0,0 +1,25 @@ +TOPIC: Binary Search + +DEFINITION: Binary search is an efficient algorithm for finding an item from a sorted list of items. It works by repeatedly dividing in half the portion of the list that could contain the item, until you've narrowed the possible locations to just one. This approach solves the problem of quickly locating a specific element within a large sorted dataset. + +TIME_COMPLEXITY: The time complexity of binary search is O(log n), where n is the number of items in the list. This is because with each comparison, the algorithm effectively halves the search space, leading to a logarithmic number of steps. + +SPACE_COMPLEXITY: The space complexity is O(1), as binary search only requires a constant amount of additional space to store the indices and the target value, regardless of the size of the input list. + +USE_WHEN: Use binary search when you need to find an item in a large sorted list, and speed is crucial. This is particularly useful in scenarios where the list is too large to search linearly, and the data is already sorted or can be sorted efficiently. + +AVOID_WHEN: Avoid using binary search when the list is unsorted, as the algorithm relies on the data being sorted to work correctly. In such cases, a linear search or a different sorting algorithm followed by binary search might be more appropriate. + +EXAMPLE: + Suppose we have a sorted list [2, 5, 8, 12, 16, 23] and we're looking for the number 12. + Start with the entire list: [2, 5, 8, 12, 16, 23] + -> Compare the middle element (8) with the target (12). Since 12 > 8, we know 12 must be in the right half. + Narrow down to the right half: [12, 16, 23] + -> Compare the middle element (16) with the target (12). Since 12 < 16, we know 12 must be in the left half of this subset. + Narrow down further: [12] + -> The only element left is 12, which matches our target: + + +REAL_WORLD_ANALOGY: Binary search is similar to finding a specific book in a library where the books are arranged alphabetically by title. You start by looking at the middle shelf, then move to the left or right half depending on whether the book you're looking for comes before or after the middle book alphabetically, repeating this process until you find the book. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/binary_search_tree.txt b/rag/knowledge_base/binary_search_tree.txt new file mode 100644 index 0000000000000000000000000000000000000000..79554a93dca1d5f3ebf7e42035a9eb1202f3e92b --- /dev/null +++ b/rag/knowledge_base/binary_search_tree.txt @@ -0,0 +1,35 @@ +TOPIC: Binary Search Tree + +DEFINITION: A Binary Search Tree (BST) is a data structure in which each node has at most two children (i.e., left child and right child) and each node represents a value. This structure allows for efficient storage and retrieval of data by ensuring that all values to the left of a node are less than the node's value, and all values to the right are greater. This property enables fast searching, inserting, and deleting of nodes. + +TIME_COMPLEXITY: The time complexity of a Binary Search Tree is O(log n) for best and average cases, and O(n) for the worst case, which occurs when the tree becomes a skewed tree (essentially a linked list). + +SPACE_COMPLEXITY: The space complexity of a Binary Search Tree is O(n), where n is the number of nodes in the tree, as each node requires a constant amount of space to store its value and references to its children. + +USE_WHEN: Use a Binary Search Tree when you need to store and retrieve large amounts of data efficiently, and the data needs to be ordered or searchable. This is particularly useful in scenarios where data is constantly being inserted or deleted. + +AVOID_WHEN: Avoid using a Binary Search Tree when the data is mostly static or when the tree may become heavily unbalanced, as this can lead to poor performance; in such cases, a self-balancing binary search tree like an AVL tree or a Red-Black tree may be more suitable. + +EXAMPLE: +Suppose we have a BST with the following nodes: + 5 + / \ + 3 7 + / \ \ + 2 4 8 +We want to insert the value 6. + First, we start at the root node (5) and compare it to the value we want to insert (6). + Since 6 is greater than 5, we move to the right child (7). + Then, we compare 6 to 7; since 6 is less than 7, we move to the left child (which doesn't exist), + so we create a new node with the value 6 as the left child of 7. + The resulting tree is: + 5 + / \ + 3 7 + / \ / \ + 2 4 6 8 + The insertion is successful + +REAL_WORLD_ANALOGY: A Binary Search Tree can be thought of as a phonebook where each entry is a person's name, and the names are arranged alphabetically; when you look up a name, you can quickly find it by comparing it to the names on either side of the current page. + +SOURCE_NOTE: Concepts referenced from general knowledge of data structures and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/binary_tree.txt b/rag/knowledge_base/binary_tree.txt new file mode 100644 index 0000000000000000000000000000000000000000..09891e9c4e4b79c3d34ce8d0a0d2881722d285aa --- /dev/null +++ b/rag/knowledge_base/binary_tree.txt @@ -0,0 +1,43 @@ +TOPIC: Binary Tree + +DEFINITION: +A binary tree is a data structure in which each node has at most two children, referred to as the left child and the right child. This structure allows for efficient storage and retrieval of data, particularly when the data has a hierarchical relationship. It solves the problem of organizing and searching large amounts of data by providing a systematic way to traverse and access the data. + +TIME_COMPLEXITY: +The time complexity of binary tree operations such as search, insert, and delete can vary: O(1) for the best case when the tree is balanced and the operation is performed at the root, O(log n) for the average case when the tree is balanced, and O(n) for the worst case when the tree is skewed. + +SPACE_COMPLEXITY: +The space complexity of a binary tree is O(n), where n is the number of nodes in the tree, since each node requires a constant amount of space to store its value and references to its children. + +USE_WHEN: +Use a binary tree when you need to store and retrieve data efficiently, especially when the data has a natural hierarchical structure, such as a file system or a database index. It's also useful when you need to perform operations like insertion, deletion, and search frequently. + +AVOID_WHEN: +Avoid using a binary tree when the data is too large to fit into memory or when the tree becomes severely unbalanced, leading to poor performance. In such cases, consider using other data structures like hash tables or self-balancing trees like AVL or Red-Black trees. + +EXAMPLE: +Suppose we have a binary tree with the following nodes: + 4 + / \ + 2 6 + / \ \ + 1 3 5 +To insert a new node with value 7, we start at the root (4) and move to the right child (6), then to the right child again (which doesn't exist), so we create a new node (7) as the right child of 6: + 4 + / \ + 2 6 + / \ / \ + 1 3 5 7 +The resulting tree is now: + 4 + / \ + 2 6 + / \ / \ + 1 3 5 7 +Result: + +REAL_WORLD_ANALOGY: +A binary tree can be thought of as a family tree, where each person (node) has at most two children, and the relationships between family members are organized in a hierarchical structure. + +SOURCE_NOTE: +Concepts referenced from general knowledge of data structures and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/bit_manipulation_basics.txt b/rag/knowledge_base/bit_manipulation_basics.txt new file mode 100644 index 0000000000000000000000000000000000000000..ee495e584af1460b803f138b382632e8cddd3106 --- /dev/null +++ b/rag/knowledge_base/bit_manipulation_basics.txt @@ -0,0 +1,24 @@ +TOPIC: Bit Manipulation Basics + +DEFINITION: Bit manipulation is a technique used to perform operations on the binary representation of numbers, allowing for efficient and low-level control over data. It solves problems that require direct access and modification of individual bits or groups of bits within a binary number. This is particularly useful in situations where memory or computational resources are limited. + +TIME_COMPLEXITY: O(1) - Bit manipulation operations are typically constant time because they involve simple bitwise operations that do not depend on the size of the input. + +SPACE_COMPLEXITY: O(1) - The space complexity is constant because bit manipulation operations usually only require a small, fixed amount of space to store the input and output values. + +USE_WHEN: Bit manipulation is the right tool when working with low-level data representations, such as flags, masks, or bit fields, and when performing operations like setting, clearing, or toggling individual bits. It is also useful when optimizing code for performance or memory usage. + +AVOID_WHEN: Bit manipulation is a poor choice when working with high-level abstractions or complex data structures, and when readability and maintainability are more important than performance. In such cases, using higher-level constructs and libraries may be more appropriate. + +EXAMPLE: + Start with the binary number: 1010 + Apply a bitwise AND operation with the mask: 1100 + 1010 +& 1100 + ---- + 1000 + The result is: 1000 + +REAL_WORLD_ANALOGY: Bit manipulation is similar to working with a combination lock, where each bit represents a specific switch that can be turned on or off to achieve a desired outcome. + +SOURCE_NOTE: Concepts referenced from general knowledge of computer science and programming principles. \ No newline at end of file diff --git a/rag/knowledge_base/bubble_sort.txt b/rag/knowledge_base/bubble_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..0520578b8130a2338e677e3ebb6eafd74270a155 --- /dev/null +++ b/rag/knowledge_base/bubble_sort.txt @@ -0,0 +1,33 @@ +TOPIC: Bubble Sort + +DEFINITION: +Bubble sort is a simple sorting algorithm that repeatedly steps through the list, compares adjacent elements, and swaps them if they are in the wrong order. This process is repeated until the list is sorted, solving the problem of arranging elements in ascending or descending order. It works by iteratively "bubbling" the largest or smallest element to the end or beginning of the list. + +TIME_COMPLEXITY: +The time complexity of bubble sort is O(n) in the best case, O(n^2) on average, and O(n^2) in the worst case, because in the best case the list is already sorted and only one pass is needed, while in the average and worst cases, the algorithm needs to make multiple passes through the list to sort it. + +SPACE_COMPLEXITY: +The space complexity of bubble sort is O(1), because it only uses a constant amount of additional space to store temporary swaps, and does not require any extra space that scales with the size of the input list. + +USE_WHEN: +Bubble sort can be useful when the list is small or nearly sorted, as it has a simple implementation and can be efficient in these cases. It can also be a good teaching tool for introducing the concept of sorting algorithms. + +AVOID_WHEN: +Bubble sort is a poor choice for large lists or lists that are in reverse order, as its average and worst-case time complexities are high, and other algorithms like quicksort or mergesort are generally more efficient and should be used instead. + +EXAMPLE: +Suppose we have the list [5, 2, 8, 3, 1] and we want to sort it in ascending order using bubble sort. + Initial list: [5, 2, 8, 3, 1] + Compare 5 and 2, swap: [2, 5, 8, 3, 1] + Compare 5 and 8, no swap: [2, 5, 8, 3, 1] + Compare 8 and 3, swap: [2, 5, 3, 8, 1] + Compare 8 and 1, swap: [2, 5, 3, 1, 8] + Repeat the process until the list is sorted: + [2, 5, 3, 1, 8] -> [2, 3, 5, 1, 8] -> [2, 3, 1, 5, 8] -> [2, 1, 3, 5, 8] -> [1, 2, 3, 5, 8] + Final sorted list: [1, 2, 3, 5, 8] + +REAL_WORLD_ANALOGY: +Bubble sort is similar to rearranging a set of books on a shelf, where you compare each book to the one next to it and swap them if they are in the wrong order, repeating the process until the books are in the correct order. + +SOURCE_NOTE: +Concepts referenced from general knowledge of sorting algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/bucket_sort.txt b/rag/knowledge_base/bucket_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..3e7545d4a2162edec04a076b24201b522b03f0f9 --- /dev/null +++ b/rag/knowledge_base/bucket_sort.txt @@ -0,0 +1,22 @@ +TOPIC: bucket sort + +DEFINITION: Bucket sort is a comparison-based sorting algorithm that distributes elements into a number of buckets, then sorts each bucket individually, typically using a simpler sorting algorithm. This approach is particularly useful for sorting large datasets of floating-point numbers or other continuous data. By dividing the data into smaller groups, bucket sort can achieve efficient sorting with minimal overhead. + +TIME_COMPLEXITY: The average-case time complexity of bucket sort is O(n + k), where n is the number of elements and k is the number of buckets, assuming a uniform distribution of elements. However, in the worst-case scenario, the time complexity can degrade to O(n^2) if all elements end up in a single bucket. + +SPACE_COMPLEXITY: The space complexity of bucket sort is O(n + k), as the algorithm requires additional memory to store the buckets and their contents. + +USE_WHEN: Bucket sort is particularly effective when dealing with large datasets of continuous data, such as floating-point numbers, and the distribution of elements is relatively uniform. It's also useful when memory is not a concern, and the goal is to achieve efficient sorting with minimal overhead. + +AVOID_WHEN: Bucket sort may not be the best choice for sorting discrete data or datasets with a large number of duplicate elements, as it can lead to poor performance and inefficient use of memory. In such cases, other sorting algorithms like quicksort or mergesort may be more suitable. + +EXAMPLE: + Initial array: [0.3, 0.1, 0.5, 0.2, 0.4] + -> Divide into buckets: [0.1, 0.2] [0.3, 0.4] [0.5] + -> Sort each bucket: [0.1, 0.2] [0.3, 0.4] [0.5] + -> Merge sorted buckets: [0.1, 0.2, 0.3, 0.4, 0.5] + Result: [0.1, 0.2, 0.3, 0.4, 0.5] + +REAL_WORLD_ANALOGY: Bucket sort can be thought of as categorizing a large collection of books into separate shelves based on their genres, and then arranging the books on each shelf in alphabetical order. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/circular_linked_list.txt b/rag/knowledge_base/circular_linked_list.txt new file mode 100644 index 0000000000000000000000000000000000000000..69a6f1a98c1b789576f5323bed259c52cbc3af1e --- /dev/null +++ b/rag/knowledge_base/circular_linked_list.txt @@ -0,0 +1,31 @@ +TOPIC: Circular Linked List + +DEFINITION: +A circular linked list is a type of data structure where the last node points back to the first node, forming a circle. This structure is useful for applications that require efficient insertion and deletion of nodes at any position, as well as traversing the list in a continuous loop. It solves the problem of having to manually handle the connection between the last and first nodes in a traditional linked list. + +TIME_COMPLEXITY: +The time complexity of a circular linked list is O(n) for operations like insertion, deletion, and traversal, where n is the number of nodes in the list. This is because in the worst-case scenario, the entire list needs to be traversed to find the desired node or to complete the operation. + +SPACE_COMPLEXITY: +The space complexity of a circular linked list is O(n), where n is the number of nodes in the list. This is because each node in the list requires a constant amount of space to store its value and the reference to the next node. + +USE_WHEN: +A circular linked list is useful when implementing a buffer or a queue with a fixed size, where the last element is connected to the first element. It's also useful in applications that require efficient insertion and deletion of nodes at any position, such as a music playlist or a browser's history. + +AVOID_WHEN: +A circular linked list is a poor choice when the application requires random access to nodes, as this would require traversing the entire list to find the desired node. In such cases, a different data structure like an array or a tree would be more suitable. + +EXAMPLE: +Suppose we have a circular linked list with the nodes [1] -> [2] -> [3] -> [1]. If we want to insert a new node [4] after [2], the steps would be: + [1] -> [2] -> [3] -> [1] + Insert [4] after [2]: + [1] -> [2] -> [4] -> [3] -> [1] + The resulting list is: + [1] -> [2] -> [4] -> [3] -> [1] +Result: + +REAL_WORLD_ANALOGY: +A circular linked list can be thought of as a merry-go-round, where the last horse is connected to the first horse, forming a continuous loop. Just as you can get on or off the merry-go-round at any horse, you can insert or delete nodes at any position in a circular linked list. + +SOURCE_NOTE: +Concepts referenced from general knowledge of data structures and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/circular_queue.txt b/rag/knowledge_base/circular_queue.txt new file mode 100644 index 0000000000000000000000000000000000000000..f36c573e43761f7692afe8cc9268892cd046852a --- /dev/null +++ b/rag/knowledge_base/circular_queue.txt @@ -0,0 +1,31 @@ +TOPIC: Circular Queue + +DEFINITION: +A circular queue is a type of data structure that follows the First-In-First-Out (FIFO) principle, where the last element is connected to the first element to form a circle, allowing for efficient use of space. This design solves the problem of wasted space in a traditional queue when elements are removed from the front and added to the rear. It enables the queue to wrap around to the beginning when it reaches the end. + +TIME_COMPLEXITY: +The time complexity of a circular queue is O(1) for both enqueue and dequeue operations, as these operations can be performed in constant time regardless of the number of elements in the queue. + +SPACE_COMPLEXITY: +The space complexity of a circular queue is O(n), where n is the number of elements in the queue, as each element requires a constant amount of space. + +USE_WHEN: +A circular queue is useful when implementing job scheduling systems, print queues, or other applications where tasks need to be processed in a FIFO order and memory efficiency is crucial. It's also beneficial when the number of elements is fixed and known in advance. + +AVOID_WHEN: +A circular queue may not be the best choice when the number of elements is highly dynamic or unpredictable, as it can lead to inefficient use of space or require frequent resizing, in which case a dynamic data structure like a linked list might be more suitable. + +EXAMPLE: +Suppose we have a circular queue with a capacity of 5 elements: [1, 2, 3, _, _]. + - Enqueue 4: [1, 2, 3, 4, _] + - Enqueue 5: [1, 2, 3, 4, 5] + - Dequeue: [2, 3, 4, 5, _] (remove 1) + - Enqueue 6: [2, 3, 4, 5, 6] + - Dequeue: [3, 4, 5, 6, _] (remove 2) +The final state is: [3, 4, 5, 6, _] +Result: + +REAL_WORLD_ANALOGY: +A circular queue can be thought of as a merry-go-round, where riders (elements) enter and exit in a circular fashion, and when the ride (queue) is full, new riders can only join when some exit, making space for them. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/coin_change_problem.txt b/rag/knowledge_base/coin_change_problem.txt new file mode 100644 index 0000000000000000000000000000000000000000..d3387aa819f1df96bb13b24a1bb20f917fd54fb6 --- /dev/null +++ b/rag/knowledge_base/coin_change_problem.txt @@ -0,0 +1,32 @@ +TOPIC: Coin Change Problem + +DEFINITION: +The Coin Change Problem is a classic problem in dynamic programming that involves finding the minimum number of coins of different denominations needed to make a certain amount of change. It solves the problem of making change for a given amount using the fewest number of coins possible. This problem is commonly encountered in real-world scenarios where we need to find the most efficient way to make change. + +TIME_COMPLEXITY: +The time complexity of the Coin Change Problem is O(amount * number_of_coins), where amount is the target amount and number_of_coins is the number of different coin denominations available. This is because we need to fill up a table of size amount * number_of_coins to store the minimum number of coins needed for each amount. + +SPACE_COMPLEXITY: +The space complexity of the Coin Change Problem is O(amount), as we need to store the minimum number of coins needed for each amount from 0 to the target amount. + +USE_WHEN: +The Coin Change Problem is useful when we need to find the minimum number of items of different sizes that add up to a certain total, such as making change or packing items of different sizes into a container. This problem can be applied to various scenarios where we need to optimize the number of items used. + +AVOID_WHEN: +The Coin Change Problem is not suitable for scenarios where the number of items is very large or the item sizes are continuous, as the dynamic programming approach may not be efficient in such cases. In these scenarios, other optimization techniques such as greedy algorithms or linear programming may be more suitable. + +EXAMPLE: +Suppose we have coins of denominations 1, 2, and 5, and we want to make change for 6. + - Initialize a table: [0, inf, inf, inf, inf, inf, inf] + - Fill the table: + - For coin 1: [0, 1, 2, 3, 4, 5, 6] + - For coin 2: [0, 1, 1, 2, 2, 3, 3] + - For coin 5: [0, 1, 1, 2, 2, 1, 2] + - The minimum number of coins needed to make change for 6 is 2 (5 + 1) + - Result: + +REAL_WORLD_ANALOGY: +The Coin Change Problem is similar to a cashier trying to give a customer the correct change using the fewest number of bills and coins possible, or a shopper trying to pack items of different sizes into a bag with the least amount of empty space. + +SOURCE_NOTE: +Concepts referenced from general knowledge of dynamic programming and optimization techniques. \ No newline at end of file diff --git a/rag/knowledge_base/counting_sort.txt b/rag/knowledge_base/counting_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..53b66d6638d4556aaf3c84b32b192debfbea934a --- /dev/null +++ b/rag/knowledge_base/counting_sort.txt @@ -0,0 +1,26 @@ +TOPIC: Counting Sort + +DEFINITION: Counting sort is a stable sorting algorithm that works by counting the occurrences of each unique element in the input array and then using these counts to determine the position of each element in the sorted output. This algorithm is particularly useful for sorting integers or other discrete data within a known range. It solves the problem of efficiently sorting small integers or categorical data. + +TIME_COMPLEXITY: The time complexity of counting sort is O(n + k), where n is the number of elements in the input array and k is the range of input values. This is because the algorithm iterates over the input array to count the occurrences of each element and then iterates over the range of input values to construct the sorted output. + +SPACE_COMPLEXITY: The space complexity of counting sort is O(n + k), as the algorithm needs to store the counts of each unique element and the sorted output. + +USE_WHEN: Counting sort is the right tool when the input data consists of small integers or categorical data with a limited range, and memory is not a concern. It is particularly useful in scenarios where the range of input values is not significantly larger than the number of elements in the input array. + +AVOID_WHEN: Counting sort is a poor choice when the range of input values is very large compared to the number of elements in the input array, as this can lead to high memory usage and slow performance; in such cases, other sorting algorithms like quicksort or mergesort may be more suitable. + +EXAMPLE: + Input: [4, 2, 2, 8, 3, 3, 1] + Counting array: [0, 1, 2, 2, 1, 0, 0, 0, 1] (counts of each element from 1 to 8) + Output: + [1] (from count of 1) + [2, 2] (from count of 2) + [3, 3] (from count of 3) + [4] (from count of 4) + [8] (from count of 1) + Result: [1, 2, 2, 3, 3, 4, 8] + +REAL_WORLD_ANALOGY: Counting sort is similar to organizing a collection of books by genre, where you first count how many books belong to each genre and then use these counts to arrange the books on shelves according to their genres. + +SOURCE_NOTE: Concepts referenced from general knowledge of sorting algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/deque.txt b/rag/knowledge_base/deque.txt new file mode 100644 index 0000000000000000000000000000000000000000..8f4a3c4bd3951e78efb1775e819dfa7ccbb2f600 --- /dev/null +++ b/rag/knowledge_base/deque.txt @@ -0,0 +1,30 @@ +TOPIC: Deque + +DEFINITION: +A Deque, short for Double-Ended Queue, is a data structure that allows elements to be added or removed from both the beginning and the end. This makes it particularly useful for applications where elements need to be efficiently inserted or deleted at either end, solving the problem of having to shift all elements when using a regular queue or stack. + +TIME_COMPLEXITY: +O(1) for adding or removing elements from both ends, as these operations can be performed directly without having to shift other elements, making Deques very efficient for such use cases. + +SPACE_COMPLEXITY: +O(n), where n is the number of elements stored in the Deque, as each element occupies a certain amount of space in memory. + +USE_WHEN: +Use a Deque when you need to implement a queue or a stack but also need the flexibility to add or remove elements from both ends efficiently, such as in a browser's history or a text editor's undo/redo functionality. + +AVOID_WHEN: +Avoid using a Deque when you only need to access elements from one end, as a regular queue or stack would be more straightforward and possibly more efficient, or when random access to elements is necessary, in which case an array or list might be a better choice. + +EXAMPLE: +Starting with an empty Deque: [] +1. Add 'A' from the front: ['A'] +2. Add 'B' from the back: ['A', 'B'] +3. Add 'C' from the front: ['C', 'A', 'B'] +4. Remove from the back: ['C', 'A'] +5. Add 'D' from the back: ['C', 'A', 'D'] +Result: ['C', 'A', 'D'] + +REAL_WORLD_ANALOGY: +A Deque can be thought of like a line of people where new people can join from either the front or the back, and people can leave from either end as well, making it easy to manage the line without having to shift everyone. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/dfs.txt b/rag/knowledge_base/dfs.txt new file mode 100644 index 0000000000000000000000000000000000000000..3490aef716103c0e720058afe91c00097e6cedef --- /dev/null +++ b/rag/knowledge_base/dfs.txt @@ -0,0 +1,37 @@ +TOPIC: DFS + +DEFINITION: Depth-First Search (DFS) is a traversal algorithm used to search and explore nodes in a graph or tree data structure. It works by visiting a node and then exploring as far as possible along each of its edges before backtracking, allowing it to efficiently search for a target node or perform other operations. This approach helps solve problems that require examining all possible paths from a given starting point. + +TIME_COMPLEXITY: The time complexity of DFS is O(V + E), where V is the number of vertices (nodes) and E is the number of edges, because in the worst case, it visits each node and edge once. + +SPACE_COMPLEXITY: The space complexity of DFS is O(h), where h is the height of the tree (or the maximum depth of the recursion call stack), because it needs to store the current path being explored. + +USE_WHEN: DFS is the right tool when you need to search for a node in a graph or tree, or when you need to perform an operation that requires exploring all possible paths from a given starting point. It's particularly useful in scenarios where the graph is very deep but not very wide. + +AVOID_WHEN: DFS is a poor choice when the graph is very wide and you're looking for the shortest path, as it can get stuck exploring a long branch before backtracking; in such cases, Breadth-First Search (BFS) is a better option. + +EXAMPLE: +Suppose we have a tree with the following structure: + A + / \ + B C + / \ \ + D E F +We start a DFS from node A: +1. Visit A +2. Explore B: + - Visit B + - Explore D: + - Visit D + - Explore E: + - Visit E +3. Backtrack to A, then explore C: + - Visit C + - Explore F: + - Visit F +The final result is: A, B, D, E, C, F +Result: + +REAL_WORLD_ANALOGY: DFS is like exploring a maze by always going as far as you can down a path before turning back, which allows you to systematically cover all areas without getting lost. + +SOURCE_NOTE: Concepts referenced from general knowledge of graph traversal algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/dijkstras_algorithm.txt b/rag/knowledge_base/dijkstras_algorithm.txt new file mode 100644 index 0000000000000000000000000000000000000000..5281adb9c4d4aaa59bb4729b26e5c3e51adbba4c --- /dev/null +++ b/rag/knowledge_base/dijkstras_algorithm.txt @@ -0,0 +1,46 @@ +TOPIC: Dijkstra's Algorithm + +DEFINITION: Dijkstra's Algorithm is a method for finding the shortest path between two points in a weighted graph or network. It solves the single-source shortest paths problem, where the goal is to determine the minimum distance from a starting node to all other nodes in the graph. This algorithm is particularly useful when the graph contains non-negative edge weights. + +TIME_COMPLEXITY: The time complexity of Dijkstra's Algorithm is O(|E|log|V|) in the worst case, where |E| is the number of edges and |V| is the number of vertices, because it uses a priority queue to efficiently select the next node to visit. + +SPACE_COMPLEXITY: The space complexity is O(|V| + |E|), as the algorithm needs to store the distance to each node and the graph's adjacency list or matrix. + +USE_WHEN: Dijkstra's Algorithm is the right tool when you need to find the shortest path in a weighted graph with non-negative edge weights, such as in network routing or traffic optimization problems. It's particularly useful when the graph is sparse, meaning that most nodes are not directly connected. + +AVOID_WHEN: You should avoid using Dijkstra's Algorithm when the graph contains negative-weight edges, as it can lead to incorrect results; instead, use the Bellman-Ford Algorithm. Additionally, for unweighted graphs, a simpler algorithm like Breadth-First Search (BFS) may be more efficient. + +EXAMPLE: + Start with a graph: + A -> B (weight 2) + A -> C (weight 4) + B -> C (weight 1) + B -> D (weight 5) + C -> D (weight 3) + Initialize distances: + A: 0 + B: infinity + C: infinity + D: infinity + Visit A, update distances: + A: 0 + B: 2 + C: 4 + D: infinity + Visit B, update distances: + A: 0 + B: 2 + C: 3 + D: 7 + Visit C, update distances: + A: 0 + B: 2 + C: 3 + D: 6 + Visit D, no updates needed + Result: shortest path from A to D is A -> B -> C -> D with distance 6 + + +REAL_WORLD_ANALOGY: Dijkstra's Algorithm is like planning a road trip, where you want to find the shortest route between two cities on a map with varying road lengths and traffic conditions. + +SOURCE_NOTE: Concepts referenced from general knowledge of graph algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/doubly_linked_list.txt b/rag/knowledge_base/doubly_linked_list.txt new file mode 100644 index 0000000000000000000000000000000000000000..8943b970d97335c0fe02b50c03acb830c5f123c7 --- /dev/null +++ b/rag/knowledge_base/doubly_linked_list.txt @@ -0,0 +1,23 @@ +TOPIC: Doubly Linked List + +DEFINITION: A doubly linked list is a data structure that consists of nodes, each containing a value and two pointers, one pointing to the next node and one to the previous node, allowing for efficient traversal in both forward and backward directions. This data structure solves the problem of needing to access and manipulate elements in a sequence from both ends. It provides more flexibility than a singly linked list, where only forward traversal is possible. + +TIME_COMPLEXITY: O(1) for insertion and deletion at the beginning or end of the list, and O(n) for searching, where n is the number of elements in the list, as in the worst case, the entire list may need to be traversed. + +SPACE_COMPLEXITY: O(n), where n is the number of elements in the list, as each node requires additional space to store the pointers to the previous and next nodes. + +USE_WHEN: A doubly linked list is the right tool when you need to frequently insert or delete elements at arbitrary positions in a sequence, or when you need to traverse a sequence in both forward and backward directions. This is particularly useful in applications such as database query results or browser history. + +AVOID_WHEN: A doubly linked list is a poor choice when memory is limited, as it requires more space than a singly linked list, and when the sequence is mostly static, as the overhead of maintaining the extra pointers may not be justified; in such cases, an array or a singly linked list may be more suitable. + +EXAMPLE: + Initially, the list is empty: [] + Insert 1 at the beginning: [1] + Insert 2 at the end: [1 -> 2] + Insert 3 at the beginning: [3 -> 1 -> 2] + Delete the middle element (1): [3 -> 2] + Check the final state: [3 -> 2] + +REAL_WORLD_ANALOGY: A doubly linked list can be thought of as a two-way street, where you can move in both directions, similar to how you can drive or walk in both directions on a road, and each intersection (node) has a sign pointing to the previous and next intersection. + +SOURCE_NOTE: Concepts referenced from general knowledge of data structures and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/dynamic_programming_overview.txt b/rag/knowledge_base/dynamic_programming_overview.txt new file mode 100644 index 0000000000000000000000000000000000000000..09181edb20c0ca135a18679ed135ae3a5064aa1b --- /dev/null +++ b/rag/knowledge_base/dynamic_programming_overview.txt @@ -0,0 +1,26 @@ +TOPIC: Dynamic Programming Overview + +DEFINITION: Dynamic programming is a method for solving complex problems by breaking them down into smaller subproblems, solving each subproblem only once, and storing the solutions to subproblems to avoid redundant computation. This approach helps to solve problems that have overlapping subproblems or that can be decomposed into smaller subproblems. It's particularly useful for problems where a naive recursive approach would be inefficient due to the repeated computation of the same subproblems. + +TIME_COMPLEXITY: O(n^2) for many dynamic programming problems, where n is the size of the input, although this can vary depending on the specific problem being solved. The key insight is that dynamic programming avoids the exponential time complexity of a naive recursive approach by storing and reusing solutions to subproblems. + +SPACE_COMPLEXITY: O(n) for storing the solutions to subproblems in a table, where n is the size of the input. The space is used to store the results of subproblems so that they can be quickly looked up instead of recomputed. + +USE_WHEN: Use dynamic programming when a problem can be broken down into smaller subproblems, and the solution to the larger problem depends on the solutions to these subproblems. This approach is particularly useful when the subproblems overlap, meaning that some subproblems may be identical or have similar solutions. + +AVOID_WHEN: Avoid using dynamic programming for problems that do not have overlapping subproblems or that cannot be decomposed into smaller subproblems, as the overhead of storing and managing the solutions to subproblems may outweigh any potential benefits. In such cases, a simple recursive or iterative approach may be more suitable. + +EXAMPLE: +Consider the problem of finding the minimum number of coins needed to make change for a given amount, using coins of denominations 1, 2, and 5. + Start with amount 0, which requires 0 coins. + For amount 1, the minimum number of coins is 1 (using a coin of denomination 1). + For amount 2, the minimum number of coins is 1 (using a coin of denomination 2). + For amount 3, the minimum number of coins is 2 (using coins of denomination 1 and 2). + ... + The final result for amount 6 is: + [0: 0, 1: 1, 2: 1, 3: 2, 4: 2, 5: 1, 6: 1] + Checkmark: The minimum number of coins needed to make change for 6 is 1 (using a coin of denomination 5 and a coin of denomination 1). + +REAL_WORLD_ANALOGY: Dynamic programming is like planning a road trip by breaking it down into smaller segments, solving each segment once, and storing the results to avoid getting lost or having to re-route. This approach helps to efficiently find the best route by avoiding redundant calculations. + +SOURCE_NOTE: Concepts synthesized from general knowledge of algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/edit_distance.txt b/rag/knowledge_base/edit_distance.txt new file mode 100644 index 0000000000000000000000000000000000000000..665c0db883e259b68cef7c47e2214d540a2bcaec --- /dev/null +++ b/rag/knowledge_base/edit_distance.txt @@ -0,0 +1,24 @@ +TOPIC: Edit Distance + +DEFINITION: The edit distance between two strings is the minimum number of operations, such as insertions, deletions, and substitutions, required to transform one string into another. This concept solves the problem of measuring the similarity between two strings, which is crucial in various applications like spell-checking, data compression, and bioinformatics. It provides a quantitative measure of how different two strings are. + +TIME_COMPLEXITY: The time complexity of calculating the edit distance is O(m*n), where m and n are the lengths of the two strings, because we need to fill in a 2D matrix of size m x n to compute the edit distances between substrings. + +SPACE_COMPLEXITY: The space complexity is O(m*n) as well, which is used to store the 2D matrix that keeps track of the edit distances between substrings. + +USE_WHEN: Use the edit distance algorithm when you need to compare the similarity between two strings, such as in spell-checking or plagiarism detection applications. It's particularly useful when you want to identify the minimum number of operations required to transform one string into another. + +AVOID_WHEN: Avoid using the edit distance algorithm when dealing with very long strings, as its time and space complexity can be prohibitively expensive; in such cases, consider using approximate algorithms or heuristics instead. + +EXAMPLE: + Transform "kitten" into "sitting" + 1. Start with "kitten" + 2. Substitute "k" with "s" -> "sitten" + 3. Substitute "e" with "i" -> "sittin" + 4. Append "g" -> "sitting" + Result: 3 operations + + +REAL_WORLD_ANALOGY: The edit distance is similar to calculating the minimum number of steps required to rearrange the letters on a Scrabble board to form a new word, where each step involves adding, removing, or replacing a letter. + +SOURCE_NOTE: Concepts referenced from general knowledge of dynamic programming and string algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/fenwick_tree_binary_indexed_tree.txt b/rag/knowledge_base/fenwick_tree_binary_indexed_tree.txt new file mode 100644 index 0000000000000000000000000000000000000000..6e9ddaf651f808fd3792a2b9a4c0fa2283d9ac87 --- /dev/null +++ b/rag/knowledge_base/fenwick_tree_binary_indexed_tree.txt @@ -0,0 +1,35 @@ +TOPIC: Fenwick Tree (Binary Indexed Tree) + +DEFINITION: +A Fenwick Tree, also known as a Binary Indexed Tree, is a data structure used to efficiently calculate prefix sums and update elements in an array. It solves the problem of quickly computing cumulative sums and making range updates by using a tree-like structure to store the cumulative sums. + +TIME_COMPLEXITY: +The time complexity of a Fenwick Tree is O(log n) for both update and query operations, where n is the number of elements in the array, because each operation involves traversing a path in the tree that is at most log n levels deep. + +SPACE_COMPLEXITY: +The space complexity of a Fenwick Tree is O(n), as it requires an array of size n to store the tree nodes, with each node representing a cumulative sum. + +USE_WHEN: +Use a Fenwick Tree when you need to frequently calculate prefix sums or make range updates in an array, such as in scenarios involving dynamic cumulative sum calculations or updates to a large dataset. This data structure is particularly useful when the array is too large to fit into memory or when updates and queries need to be performed quickly. + +AVOID_WHEN: +Avoid using a Fenwick Tree when the array is relatively small or when random access to individual elements is necessary, as the overhead of the tree structure may outweigh its benefits; in such cases, a simple array or a hash table may be a better choice. + +EXAMPLE: +Suppose we have an array [3, 2, -1, 6, 5] and we want to calculate the prefix sum up to the 4th index using a Fenwick Tree: + Initial array: [3, 2, -1, 6, 5] + Initialize tree: [3, 5, 4, 10, 15] + To calculate prefix sum up to 4th index: + Start at index 4 (value 15) + Subtract value at index 3 (value 10): 15 - 10 = 5 + Add value at index 2 (value 4): 5 + 4 = 9 + Subtract value at index 1 (value 5): 9 - 5 = 4 + Add value at index 0 (value 3): 4 + 3 = 7 + Result: + Checkmark: The prefix sum up to the 4th index is 15. + +REAL_WORLD_ANALOGY: +A Fenwick Tree can be thought of as a hierarchical accounting system, where each node in the tree represents a cumulative sum of a subset of accounts, allowing for efficient calculation of totals and updates to individual accounts. + +SOURCE_NOTE: +Concepts referenced from general knowledge of data structures and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/floyd-warshall_algorithm.txt b/rag/knowledge_base/floyd-warshall_algorithm.txt new file mode 100644 index 0000000000000000000000000000000000000000..aad0cf3ccbe98997365f0f7af9a0a30673e93600 --- /dev/null +++ b/rag/knowledge_base/floyd-warshall_algorithm.txt @@ -0,0 +1,41 @@ +TOPIC: Floyd-Warshall Algorithm + +DEFINITION: +The Floyd-Warshall algorithm is a dynamic programming approach used to find the shortest path between all pairs of vertices in a weighted graph. It solves the all-pairs shortest paths problem, which is essential in various applications such as network routing and traffic optimization. This algorithm can handle both positive and negative weight edges. + +TIME_COMPLEXITY: +The time complexity of the Floyd-Warshall algorithm is O(V^3), where V is the number of vertices in the graph. This is because the algorithm involves three nested loops, each iterating over all vertices. + +SPACE_COMPLEXITY: +The space complexity is O(V^2), which is used to store the distance matrix that keeps track of the shortest distances between all pairs of vertices. + +USE_WHEN: +Use the Floyd-Warshall algorithm when you need to find the shortest path between all pairs of vertices in a weighted graph, especially when the graph is dense or when you need to handle negative weight edges. This algorithm is particularly useful in applications where the graph is relatively small and the all-pairs shortest paths are required. + +AVOID_WHEN: +Avoid using the Floyd-Warshall algorithm for very large graphs or sparse graphs, as its cubic time complexity can be inefficient. In such cases, consider using other algorithms like Dijkstra's or Bellman-Ford, which are more efficient for single-source shortest paths or sparse graphs. + +EXAMPLE: +Consider a graph with vertices A, B, C, and the following edges: + A -> B (weight: 2) + B -> C (weight: 3) + A -> C (weight: 5) +The distance matrix is initially: + [0, 2, inf] + [inf, 0, 3] + [inf, inf, 0] +After applying the Floyd-Warshall algorithm: + 1. Consider vertex A: no changes + 2. Consider vertex B: update A->C through B (2+3=5), but A->C is already 5, so no change + 3. Consider vertex C: update A->C through C (2+0=2) is not possible since C is not between A and B, but update B->C through C (3+0=3) is not better, so no change +The final distance matrix is: + [0, 2, 5] + [inf, 0, 3] + [inf, inf, 0] + The shortest path from A to C is through B with a total weight of 5 + + +REAL_WORLD_ANALOGY: +The Floyd-Warshall algorithm can be thought of as finding the shortest route between all pairs of cities in a road network, similar to how a GPS navigation system calculates the fastest route between two points. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/fractional_knapsack.txt b/rag/knowledge_base/fractional_knapsack.txt new file mode 100644 index 0000000000000000000000000000000000000000..227df7872ee350fb9cb23262e8c5c5fb4d483865 --- /dev/null +++ b/rag/knowledge_base/fractional_knapsack.txt @@ -0,0 +1,34 @@ +TOPIC: Fractional Knapsack + +DEFINITION: The Fractional Knapsack problem is a classic problem in computer science where we have a set of items, each with a weight and a value, and a knapsack with a limited capacity. The goal is to determine the optimal subset of items to include in the knapsack to maximize the total value while not exceeding the knapsack's capacity, with the added twist that items can be taken fractionally. This problem solves the issue of optimizing resource allocation when the resources are limited and the items can be divided. + +TIME_COMPLEXITY: O(n log n) in the worst case, due to the sorting of items based on their value-to-weight ratio, which is necessary to find the optimal solution. + +SPACE_COMPLEXITY: O(n) for storing the items and their corresponding values and weights. + +USE_WHEN: The Fractional Knapsack problem is the right tool to use when we need to allocate limited resources to a set of items with different values and weights, and the items can be divided into smaller parts. This is particularly useful in scenarios such as portfolio optimization or resource allocation in financial markets. + +AVOID_WHEN: The Fractional Knapsack problem is a poor choice when the items cannot be divided, in which case the 0/1 Knapsack problem is more suitable. It's also not the best choice when the number of items is very large and the knapsack capacity is relatively small, as the sorting step can become computationally expensive. + +EXAMPLE: + Suppose we have a knapsack with a capacity of 10 units and the following items: + - Item A: weight 5, value 30 + - Item B: weight 3, value 20 + - Item C: weight 2, value 15 + First, we sort the items by their value-to-weight ratio: + - Item C: 7.5 (15/2) + - Item B: 6.67 (20/3) + - Item A: 6 (30/5) + Then, we start filling the knapsack with the item with the highest ratio: + - Take all of Item C (2 units, value 15) + -> Knapsack: [Item C], capacity left: 8 + - Take all of Item B (3 units, value 20) + -> Knapsack: [Item C, Item B], capacity left: 5 + - Take 5/5 of Item A (5 units, value 30) + -> Knapsack: [Item C, Item B, Item A], capacity left: 0 + The total value of the items in the knapsack is: 15 + 20 + 30 = 65 + Checkmark: + +REAL_WORLD_ANALOGY: The Fractional Knapsack problem is similar to deciding how to allocate your budget to different investments, such as stocks or bonds, where you can invest a fraction of your budget in each option. + +SOURCE_NOTE: Concepts referenced from general knowledge of algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/greedy_algorithms_overview.txt b/rag/knowledge_base/greedy_algorithms_overview.txt new file mode 100644 index 0000000000000000000000000000000000000000..2ea54140b96fd8f03f10f0a099add908a92ec194 --- /dev/null +++ b/rag/knowledge_base/greedy_algorithms_overview.txt @@ -0,0 +1,30 @@ +TOPIC: Greedy Algorithms Overview + +DEFINITION: +Greedy algorithms are a type of algorithmic strategy that solves problems by making the locally optimal choice at each step, with the hope that these local choices will lead to a globally optimal solution. This approach is useful for solving optimization problems, where the goal is to find the best solution among a set of possible solutions. By choosing the best option at each step, greedy algorithms aim to find a solution that is optimal or close to optimal. + +TIME_COMPLEXITY: +The time complexity of greedy algorithms can vary depending on the specific problem, but it is often O(n), where n is the number of elements being processed, because the algorithm typically makes a single pass through the data. + +SPACE_COMPLEXITY: +The space complexity of greedy algorithms is usually O(1), because the algorithm only needs to keep track of a limited amount of information, such as the current solution and any relevant parameters. + +USE_WHEN: +Greedy algorithms are a good choice when the problem has the following properties: the problem can be broken down into smaller sub-problems, and the optimal solution to the larger problem can be constructed from the optimal solutions of the sub-problems. This is often the case in problems that involve finding the shortest path, the minimum spanning tree, or the optimal scheduling. + +AVOID_WHEN: +Greedy algorithms are not the best choice when the problem requires considering the global optimality, and the locally optimal choices do not necessarily lead to a globally optimal solution. In such cases, other algorithms like dynamic programming may be more suitable. + +EXAMPLE: +Consider a problem where we need to make change for 6 units using coins of denominations 1, 2, and 5. + Start with 6 units + -> subtract 5 ( largest possible coin) = 1 unit left + -> subtract 1 ( largest possible coin) = 0 units left + Result: [5, 1] + Checkmark: The greedy algorithm finds a valid solution by making the locally optimal choice at each step. + +REAL_WORLD_ANALOGY: +The greedy algorithm is similar to a person trying to make change for a purchase, where they try to use the largest denomination bills and coins first to minimize the total number of items used. + +SOURCE_NOTE: +Concepts referenced from general knowledge of algorithms and data structures, synthesized for clarity and conciseness. \ No newline at end of file diff --git a/rag/knowledge_base/hash_set_vs_hash_map.txt b/rag/knowledge_base/hash_set_vs_hash_map.txt new file mode 100644 index 0000000000000000000000000000000000000000..cabad622d5e74d7975a42711e336546f507badd5 --- /dev/null +++ b/rag/knowledge_base/hash_set_vs_hash_map.txt @@ -0,0 +1,31 @@ +TOPIC: Hash Set vs Hash Map + +DEFINITION: +A hash set and a hash map are two fundamental data structures used for storing and retrieving data efficiently. The primary difference between them lies in their purpose: a hash set is used to store unique elements, whereas a hash map stores key-value pairs. This distinction makes them suitable for solving different problems, such as keeping track of unique items versus storing and retrieving data by a specific key. + +TIME_COMPLEXITY: +The average time complexity for both hash sets and hash maps is O(1) for basic operations like insertion, deletion, and search, assuming a good hash function is used. However, in the worst-case scenario, it can degrade to O(n) if there are many collisions. + +SPACE_COMPLEXITY: +The space complexity for both data structures is O(n), where n is the number of elements stored, as each element requires a certain amount of space to store. + +USE_WHEN: +Use a hash set when you need to store a collection of unique items and quickly check for membership, such as in a spell checker to store unique words. Use a hash map when you need to associate each item with additional data, such as in a phonebook to store names and phone numbers. + +AVOID_WHEN: +Avoid using a hash set when you need to store key-value pairs or when order matters, as hash sets do not maintain any particular order and do not support key-value pairs. Instead, use a hash map or other data structures like a tree map or a linked hash map. + +EXAMPLE: +Consider adding unique names to a hash set: + - Start with an empty set: [] + - Add "John": ["John"] + - Add "Alice": ["John", "Alice"] + - Add "John" again: still ["John", "Alice"] because duplicates are ignored + - Check if "Bob" is in the set: no, because it was never added + Result: ["John", "Alice"] + +REAL_WORLD_ANALOGY: +A hash set is like a list of unique attendees at a party, where each name is recorded only once. A hash map is like a phonebook where each name is associated with a phone number, allowing you to look up the number by the name. + +SOURCE_NOTE: +Concepts referenced from general knowledge of data structures and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/hash_table.txt b/rag/knowledge_base/hash_table.txt new file mode 100644 index 0000000000000000000000000000000000000000..eb2e9c36a2afceaaece36c6c34c9148ecdfe033d --- /dev/null +++ b/rag/knowledge_base/hash_table.txt @@ -0,0 +1,38 @@ +TOPIC: Hash Table + +DEFINITION: +A hash table is a data structure that efficiently stores and retrieves key-value pairs by mapping keys to specific indices of an array using a hash function, allowing for fast lookups and insertions. This solves the problem of quickly finding and accessing specific data in a large collection. It enables constant-time performance for basic operations, making it a fundamental component in many algorithms. + +TIME_COMPLEXITY: +The average time complexity of a hash table is O(1) for search, insert, and delete operations, assuming a good hash function that minimizes collisions. However, in the worst-case scenario where all keys hash to the same index, the time complexity can degrade to O(n), where n is the number of elements in the table. + +SPACE_COMPLEXITY: +The space complexity of a hash table is O(n), where n is the number of key-value pairs stored, as each pair requires a constant amount of space. + +USE_WHEN: +Hash tables are particularly useful when you need to frequently look up, insert, or delete elements in a large dataset, and the keys are unique or can be made unique. They are also useful when memory is not a concern, and fast average-case performance is crucial. + +AVOID_WHEN: +Avoid using hash tables when the order of elements matters, or when the dataset is too large to fit into memory, as this can lead to page faults and slow performance; in such cases, consider using a balanced binary search tree or a disk-based database instead. + +EXAMPLE: +Suppose we have a hash table with 5 slots and we want to insert the following key-value pairs: (apple, 5), (banana, 7), (orange, 3), (grape, 2), (mango, 4). + Initially, the table is empty: + [ ] [ ] [ ] [ ] [ ] + We insert (apple, 5) and the hash function maps 'apple' to index 2: + [ ] [ ] [(apple, 5)] [ ] [ ] + Next, we insert (banana, 7) which maps to index 4: + [ ] [ ] [(apple, 5)] [ ] [(banana, 7)] + Then, (orange, 3) maps to index 2, but since it's already occupied, we handle the collision: + [ ] [ ] [(apple, 5) -> (orange, 3)] [ ] [(banana, 7)] + After inserting (grape, 2) to index 1 and (mango, 4) to index 3: + [ ] [(grape, 2)] [(apple, 5) -> (orange, 3)] [(mango, 4)] [(banana, 7)] + The final state of the hash table is as shown above, with some collisions handled + Checkmark: + The hash table now contains all the inserted key-value pairs, allowing for fast lookup by key. + +REAL_WORLD_ANALOGY: +A hash table is similar to a library's catalog system, where each book is assigned a unique identifier (key) that maps to a specific shelf location (index), allowing for quick retrieval of the book. + +SOURCE_NOTE: +Concepts referenced from general knowledge of data structures and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/heap_minmax.txt b/rag/knowledge_base/heap_minmax.txt new file mode 100644 index 0000000000000000000000000000000000000000..c8a5aff6592c90ff21d44f6b16e15d463706ebdf --- /dev/null +++ b/rag/knowledge_base/heap_minmax.txt @@ -0,0 +1,32 @@ +TOPIC: Heap (Min/Max) + +DEFINITION: +A heap is a specialized tree-based data structure that satisfies the heap property, where the parent node is either greater than (max heap) or less than (min heap) its child nodes, allowing for efficient extraction of the maximum or minimum element. This property makes heaps particularly useful for solving problems that require frequent insertion and removal of the largest or smallest item. + +TIME_COMPLEXITY: +The time complexity of heap operations is O(log n) for insertion and deletion, as the heap needs to be rebalanced after these operations, while extraction of the maximum or minimum element is O(1) since it's always at the root. + +SPACE_COMPLEXITY: +The space complexity of a heap is O(n), where n is the number of elements in the heap, as each element occupies a single node in the tree. + +USE_WHEN: +Heaps are the right tool when you need to efficiently manage a collection of items and frequently extract the largest or smallest item, such as in priority queues or event handling systems. They're particularly useful when the collection is constantly changing. + +AVOID_WHEN: +Heaps are a poor choice when you need to search for arbitrary elements or perform range queries, as the tree structure doesn't support these operations efficiently; in such cases, a balanced search tree like an AVL tree or a hash table might be more suitable. + +EXAMPLE: +Suppose we have a min heap with the following elements: [5, 3, 8, 2, 9]. To insert a new element 1, we first add it to the end: [5, 3, 8, 2, 9, 1]. Then, we bubble it up to its correct position: + [5, 3, 8, 2, 9, 1] + -> [5, 3, 8, 2, 9] + -> swap 1 with 2: [5, 3, 8, 1, 9] + -> swap 1 with 3: [5, 1, 8, 3, 9] + -> swap 1 with 5: [1, 5, 8, 3, 9]. +The resulting min heap is: [1, 5, 8, 3, 9] +Checkmark: the smallest element 1 is now at the root. + +REAL_WORLD_ANALOGY: +A heap can be thought of as a pile of boxes, where the largest or smallest box is always on top and easily accessible, and when a new box is added, it's placed at the bottom and then moved up to its correct position based on its size. + +SOURCE_NOTE: +Concepts referenced from general knowledge of data structures and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/heap_sort.txt b/rag/knowledge_base/heap_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..8cb307f37788b1dc9eb2e08b8207384405cf38b3 --- /dev/null +++ b/rag/knowledge_base/heap_sort.txt @@ -0,0 +1,29 @@ +TOPIC: Heap Sort + +DEFINITION: Heap Sort is a comparison-based sorting algorithm that uses a binary heap data structure to sort elements in ascending or descending order. It solves the problem of rearranging a collection of items into a specific order, making it useful for organizing and searching large datasets. By leveraging the properties of a heap, Heap Sort efficiently rearranges elements to achieve the desired sorted order. + +TIME_COMPLEXITY: The time complexity of Heap Sort is O(n log n) in all cases (best, average, and worst), because it involves building a heap and then repeatedly removing the largest (or smallest) element, which takes logarithmic time for each of the n elements. + +SPACE_COMPLEXITY: The space complexity of Heap Sort is O(1), as it only uses a small amount of extra memory for variables and does not require any additional data structures that scale with the input size. + +USE_WHEN: Heap Sort is a good choice when you need to sort large datasets and want a simple, in-place sorting algorithm with a guaranteed O(n log n) time complexity. It's also useful when memory is limited, as it only uses a small amount of extra space. + +AVOID_WHEN: Heap Sort may not be the best choice when the input data is nearly sorted or has a specific structure that can be exploited by other algorithms, such as Insertion Sort or Radix Sort, which can be faster in those cases. + +EXAMPLE: + Initial array: [5, 2, 8, 3, 1] + Build heap: [8, 5, 2, 3, 1] + -> swap 8 and 5: [5, 8, 2, 3, 1] + -> heapify: [8, 5, 2, 3, 1] + Remove max (8): [5, 3, 2, 1] + -> heapify: [5, 3, 2, 1] + Remove max (5): [3, 1, 2] + -> heapify: [3, 1, 2] + Remove max (3): [2, 1] + -> heapify: [2, 1] + Remove max (2): [1] + Final sorted array: [1, 2, 3, 5, 8] + +REAL_WORLD_ANALOGY: Heap Sort can be thought of like a librarian organizing books on shelves, where the librarian first builds a "heap" of books and then repeatedly removes the most important (or largest) book, placing it in its correct position, until the entire collection is sorted. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/insertion_sort.txt b/rag/knowledge_base/insertion_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..4355d96b083ce5dc059ea667b73831083f6b78a6 --- /dev/null +++ b/rag/knowledge_base/insertion_sort.txt @@ -0,0 +1,22 @@ +TOPIC: Insertion Sort + +DEFINITION: Insertion sort is a simple sorting algorithm that works by dividing the input into a sorted and an unsorted region, and then iteratively inserting elements from the unsorted region into the sorted region in the correct position. This process continues until the entire input is sorted, solving the problem of arranging elements in ascending or descending order. It's a straightforward and intuitive method for sorting small datasets. + +TIME_COMPLEXITY: The time complexity of insertion sort is O(n^2) in the worst and average cases, where n is the number of elements being sorted, because in the worst case, each element must be compared with every other element. However, in the best case, when the input is already sorted, the time complexity is O(n) because only one comparison per element is needed. + +SPACE_COMPLEXITY: The space complexity of insertion sort is O(1) because it only uses a constant amount of additional space to store temporary variables, and it sorts the input in-place, without requiring any extra space that scales with the input size. + +USE_WHEN: Insertion sort is a good choice when the input size is small, or when the input is nearly sorted, as it has a simple implementation and performs well in these scenarios. It's also a good choice when memory is limited, as it only uses a constant amount of extra space. + +AVOID_WHEN: Insertion sort is a poor choice for large datasets, as its quadratic time complexity makes it inefficient compared to other sorting algorithms like quicksort or mergesort, which have average-case time complexities of O(n log n). + +EXAMPLE: + Initial array: [5, 2, 8, 3] + -> Insert 2 into sorted [5]: [2, 5, 8, 3] + -> Insert 8 into sorted [2, 5]: [2, 5, 8, 3] + -> Insert 3 into sorted [2, 5, 8]: [2, 3, 5, 8] + Result: [2, 3, 5, 8] + +REAL_WORLD_ANALOGY: Insertion sort is similar to how you might sort a hand of cards, by taking each card and inserting it into the correct position among the cards you've already sorted. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/kruskals_algorithm_mst.txt b/rag/knowledge_base/kruskals_algorithm_mst.txt new file mode 100644 index 0000000000000000000000000000000000000000..1336230506ef26d67087ec932d75af2bd837fe43 --- /dev/null +++ b/rag/knowledge_base/kruskals_algorithm_mst.txt @@ -0,0 +1,32 @@ +TOPIC: Kruskal's Algorithm (MST) + +DEFINITION: +Kruskal's Algorithm is a popular algorithm in graph theory that helps find the Minimum Spanning Tree (MST) of a connected, undirected, and weighted graph. It solves the problem of finding the subset of edges with the minimum total weight that connects all vertices in the graph. This algorithm is particularly useful for network design problems where the goal is to minimize the cost of connecting all nodes. + +TIME_COMPLEXITY: +The time complexity of Kruskal's Algorithm is O(E log E) or O(E log V), where E is the number of edges and V is the number of vertices, because it involves sorting all the edges by their weights. + +SPACE_COMPLEXITY: +The space complexity is O(V + E), as the algorithm needs to store the vertices and edges of the graph, as well as the disjoint-set data structure used to keep track of connected components. + +USE_WHEN: +Kruskal's Algorithm is the right tool when you need to find the Minimum Spanning Tree of a sparse graph, as it is more efficient than other algorithms like Prim's for such cases. It's also useful when the graph is represented as a list of edges. + +AVOID_WHEN: +Kruskal's Algorithm may not be the best choice for dense graphs, as its sorting step can be slower than the incremental approach used in Prim's Algorithm; in such cases, Prim's Algorithm might be more efficient. + +EXAMPLE: +Suppose we have a graph with vertices A, B, C, and edges [A-B:2, A-C:3, B-C:1]. + 1. Sort edges by weight: [B-C:1, A-B:2, A-C:3] + 2. Initialize MST as empty and disjoint sets as {A}, {B}, {C} + 3. Add B-C:1 to MST, merge sets {B}, {C} to {B,C} + 4. Add A-B:2 to MST, merge sets {A}, {B,C} to {A,B,C} + 5. Skip A-C:3 as it forms a cycle +Result: MST = [B-C:1, A-B:2] + + +REAL_WORLD_ANALOGY: +Finding the Minimum Spanning Tree using Kruskal's Algorithm is similar to planning the most cost-efficient road network between cities, where the goal is to connect all cities with the least amount of total road length. + +SOURCE_NOTE: +Concepts referenced from general knowledge of graph theory and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/linear_search.txt b/rag/knowledge_base/linear_search.txt new file mode 100644 index 0000000000000000000000000000000000000000..a4923d3de3cf2bd91c56ae8525f4893fab3e17cf --- /dev/null +++ b/rag/knowledge_base/linear_search.txt @@ -0,0 +1,27 @@ +TOPIC: Linear Search + +DEFINITION: Linear search is a simple algorithm used to find a specific element within a list or array by checking each element one by one. It solves the problem of locating a particular item in an unsorted collection of data. This method is straightforward but can be time-consuming for large datasets. + +TIME_COMPLEXITY: The time complexity of linear search is O(n) in all cases (best, average, and worst), because in the worst-case scenario, the algorithm has to check every element in the list. + +SPACE_COMPLEXITY: The space complexity is O(1), as linear search only uses a constant amount of space to store the target element and the current index being examined. + +USE_WHEN: Linear search is the right tool when the dataset is small or when the data is mostly unsorted, making other search algorithms like binary search impractical. It's also useful when the cost of sorting the data outweighs the cost of performing a linear search. + +AVOID_WHEN: Linear search is a poor choice when dealing with large datasets, as its linear time complexity can lead to slow performance; in such cases, algorithms like binary search or hash-based searches are more efficient. + +EXAMPLE: + Suppose we have a list [3, 1, 4, 2] and we're looking for the element 2. + Start with the first element: [3, 1, 4, 2] + -> Check if 3 matches 2: no, move to the next element + [3, 1, 4, 2] + -> Check if 1 matches 2: no, move to the next element + [3, 1, 4, 2] + -> Check if 4 matches 2: no, move to the next element + [3, 1, 4, 2] + -> Check if 2 matches 2: yes, found! + Result: Element 2 found at index 3 + +REAL_WORLD_ANALOGY: Linear search is similar to looking for a specific book in a library by checking each shelf one by one, as opposed to using a catalog system that allows for faster lookup. + +SOURCE_NOTE: Concepts referenced from general knowledge of algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/longest_common_subsequence.txt b/rag/knowledge_base/longest_common_subsequence.txt new file mode 100644 index 0000000000000000000000000000000000000000..d5da6f08dede456b252de7794ea9a4a30b3890b2 --- /dev/null +++ b/rag/knowledge_base/longest_common_subsequence.txt @@ -0,0 +1,28 @@ +TOPIC: Longest Common Subsequence + +DEFINITION: The Longest Common Subsequence (LCS) problem is a classic problem in computer science that involves finding the longest sequence of characters or elements that is common to two or more sequences. It solves the problem of identifying the longest contiguous or non-contiguous substring that appears in multiple sequences, which has numerous applications in data comparison and analysis. + +TIME_COMPLEXITY: The time complexity of the LCS problem is O(m*n), where m and n are the lengths of the two input sequences, because the problem is typically solved using dynamic programming, which involves filling a 2D table of size m x n. + +SPACE_COMPLEXITY: The space complexity is O(m*n), which is used to store the 2D table that keeps track of the lengths of common subsequences. + +USE_WHEN: The LCS problem is useful when comparing two or more sequences to identify similarities or patterns, such as in bioinformatics, data compression, or plagiarism detection. It is particularly useful when the sequences are long and the common subsequences are short. + +AVOID_WHEN: The LCS problem may not be the best choice when the sequences are very short or when the common subsequences are very long, as the problem can be solved more efficiently using other methods, such as brute force or suffix trees. + +EXAMPLE: + Sequence 1: [A, B, C, D, E] + Sequence 2: [A, C, E, F, G] + Step 1: Initialize a 2D table to store the lengths of common subsequences + [0, 0, 0, 0, 0] + [0, 0, 0, 0, 0] + Step 2: Fill the table using dynamic programming + [1, 1, 1, 1, 1] + [0, 1, 1, 1, 1] + Step 3: Backtrack to find the longest common subsequence + [A, C, E] + Result: The longest common subsequence is [A, C, E] + +REAL_WORLD_ANALOGY: The LCS problem is similar to finding the longest common thread between two or more stories, where the thread represents the common elements or events that appear in multiple stories. + +SOURCE_NOTE: Concepts referenced from general knowledge of dynamic programming and sequence comparison algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/longest_increasing_subsequence.txt b/rag/knowledge_base/longest_increasing_subsequence.txt new file mode 100644 index 0000000000000000000000000000000000000000..e01cf3af7b7c41a2ea22d718913f40fc1414b9c6 --- /dev/null +++ b/rag/knowledge_base/longest_increasing_subsequence.txt @@ -0,0 +1,29 @@ +TOPIC: Longest Increasing Subsequence + +DEFINITION: The Longest Increasing Subsequence (LIS) problem is a classic problem in computer science that involves finding the longest subsequence of a given sequence where each element is greater than its previous element. This problem is useful for identifying patterns in data where order matters, such as in time series analysis or genetic sequencing. It solves the problem of extracting a meaningful subset of data that shows a consistent increasing trend. + +TIME_COMPLEXITY: The time complexity of the Longest Increasing Subsequence problem is O(n^2) in the worst case, where n is the length of the input sequence, because it involves comparing each element with every other element. However, using dynamic programming, the time complexity can be improved to O(n log n). + +SPACE_COMPLEXITY: The space complexity is O(n), which is used to store the lengths of the longest increasing subsequences ending at each position. + +USE_WHEN: The Longest Increasing Subsequence is the right tool when you need to identify a pattern in a sequence of data where each element is related to its previous element, such as in financial analysis or signal processing. It's particularly useful when the sequence has a mix of increasing and decreasing trends. + +AVOID_WHEN: Avoid using the Longest Increasing Subsequence when the sequence is very large and the elements are mostly random or uncorrelated, as the algorithm's time complexity can become impractical. In such cases, consider using approximate algorithms or heuristics that can provide a good trade-off between accuracy and computational efficiency. + +EXAMPLE: + Start with the sequence: [10, 22, 9, 33, 21, 50] + Initialize LIS: [1, 1, 1, 1, 1, 1] + Compare elements: + 10 < 22, update LIS: [1, 2, 1, 1, 1, 1] + 10 < 9, no update + 22 > 9, update LIS: [1, 2, 1, 1, 1, 1] + 9 < 33, update LIS: [1, 2, 1, 3, 1, 1] + 22 < 33, update LIS: [1, 2, 1, 3, 1, 1] + 9 < 21, update LIS: [1, 2, 1, 3, 2, 1] + 33 > 21, no update + 21 < 50, update LIS: [1, 2, 1, 3, 2, 4] + The longest increasing subsequence is: [10, 22, 33, 50] + +REAL_WORLD_ANALOGY: The Longest Increasing Subsequence problem is similar to finding the longest sequence of good weather days in a year, where each day's weather is compared to the previous day's to determine if it's an improvement. + +SOURCE_NOTE: Concepts referenced from general knowledge of dynamic programming and sequence analysis. \ No newline at end of file diff --git a/rag/knowledge_base/merge_sort.txt b/rag/knowledge_base/merge_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..c270566404e079ba2b7ef8fbfc4a2348183187ad --- /dev/null +++ b/rag/knowledge_base/merge_sort.txt @@ -0,0 +1,24 @@ +TOPIC: Merge Sort + +DEFINITION: Merge sort is a popular sorting algorithm that divides a list of elements into smaller sublists, sorts each sublist, and then merges the sorted sublists to produce a single, sorted list. This approach helps solve the problem of efficiently sorting large datasets by breaking them down into more manageable pieces. It's particularly useful for sorting lists that are too large to fit into memory. + +TIME_COMPLEXITY: The time complexity of merge sort is O(n log n) in all cases (best, average, and worst), because the algorithm always divides the list in half and sorts each half recursively, resulting in a logarithmic number of levels and a linear amount of work at each level. + +SPACE_COMPLEXITY: The space complexity of merge sort is O(n), because the algorithm needs to create temporary arrays to hold the merged sublists, and in the worst case, it may need to create an array of the same size as the original list. + +USE_WHEN: Merge sort is a good choice when you need to sort a large list of elements and you have enough extra memory to hold the temporary arrays. It's also a good choice when stability is important, because merge sort is a stable sorting algorithm. + +AVOID_WHEN: Merge sort may not be the best choice when memory is limited, because it requires extra space to hold the temporary arrays. In such cases, an in-place sorting algorithm like quicksort or heapsort may be a better choice. + +EXAMPLE: + Initial list: [5, 2, 8, 3, 1, 6, 4] + Divide into sublists: [5, 2, 8, 3] and [1, 6, 4] + Divide again: [5, 2] and [8, 3] and [1, 6] and [4] + Sort each sublist: [2, 5] and [3, 8] and [1, 6] and [4] + Merge sublists: [2, 3, 5, 8] and [1, 4, 6] + Merge final sublists: [1, 2, 3, 4, 5, 6, 8] + Result: [1, 2, 3, 4, 5, 6, 8] + +REAL_WORLD_ANALOGY: Merge sort is like sorting a large stack of papers by dividing the stack into smaller piles, sorting each pile, and then merging the sorted piles into a single, organized stack. + +SOURCE_NOTE: Concepts referenced from general knowledge of algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/monotonic_stack.txt b/rag/knowledge_base/monotonic_stack.txt new file mode 100644 index 0000000000000000000000000000000000000000..9ce6155fca8f9257d8fcfa3923aa45970608e1bc --- /dev/null +++ b/rag/knowledge_base/monotonic_stack.txt @@ -0,0 +1,25 @@ +TOPIC: Monotonic Stack + +DEFINITION: A monotonic stack is a data structure that maintains a stack of elements in either monotonically increasing or decreasing order, allowing for efficient retrieval of the next larger or smaller element. This data structure is particularly useful in solving problems that involve finding the next greater or smaller element in a sequence. It helps in reducing the time complexity by avoiding unnecessary comparisons. + +TIME_COMPLEXITY: O(n) - The best, average, and worst-case time complexities are all linear because each element is pushed and popped from the stack exactly once. + +SPACE_COMPLEXITY: O(n) - The space complexity is linear because in the worst-case scenario, the stack can contain up to n elements, where n is the number of elements in the input sequence. + +USE_WHEN: Use a monotonic stack when you need to find the next greater or smaller element for each element in a sequence, such as in problems involving finding the next greater element in an array. This data structure is particularly useful when the sequence is large and the elements are distinct. + +AVOID_WHEN: Avoid using a monotonic stack when the sequence is very small or when the elements are mostly the same, as the overhead of maintaining the stack may outweigh its benefits; in such cases, a simple iterative approach might be more efficient. + +EXAMPLE: +Consider the sequence [3, 1, 2, 4] and we want to find the next greater element for each element. +- Start with an empty stack: [] +- Push 3: [3] +- Push 1: [3, 1] (since 1 < 3, we don't pop) +- Push 2: [3, 1, 2] (since 2 > 1, pop 1 and update next greater for 1 as 2) -> [3, 2] +- Push 4: [3, 2, 4] (since 4 > 2, pop 2 and update next greater for 2 as 4) -> [3, 4] (since 4 > 3, pop 3 and update next greater for 3 as 4) -> [4] +- Result: Next greater elements are [4, 2, 4, -1] (assuming -1 for the last element since there's no greater element after it) + + +REAL_WORLD_ANALOGY: A monotonic stack can be thought of as a pile of plates where each plate is either larger or smaller than the one below it, and you're trying to find the next larger or smaller plate for each one in the pile. + +SOURCE_NOTE: Concepts referenced from general data structures and algorithms principles. \ No newline at end of file diff --git a/rag/knowledge_base/n-queens_problem.txt b/rag/knowledge_base/n-queens_problem.txt new file mode 100644 index 0000000000000000000000000000000000000000..0690eb1274900655e76ac2c15cb5bc223baf3bc8 --- /dev/null +++ b/rag/knowledge_base/n-queens_problem.txt @@ -0,0 +1,43 @@ +TOPIC: N-Queens Problem + +DEFINITION: The N-Queens Problem is a classic problem in computer science that involves placing N queens on an NxN chessboard such that no two queens attack each other. This problem is a classic example of a constraint satisfaction problem, where the goal is to find a configuration that satisfies a set of constraints. + +TIME_COMPLEXITY: The time complexity of the N-Queens Problem is O(N!) in the worst case, as there are N! possible configurations of the queens on the board, and in the worst case, we may need to explore all of them to find a solution. + +SPACE_COMPLEXITY: The space complexity is O(N), as we need to store the current configuration of the queens on the board, which requires O(N) space. + +USE_WHEN: The N-Queens Problem is a useful tool when we need to solve a constraint satisfaction problem with a small number of variables, and we can afford to use a brute-force approach to find a solution. This problem is also useful for learning about backtracking algorithms. + +AVOID_WHEN: The N-Queens Problem is a poor choice when we need to solve a large-scale constraint satisfaction problem, as the time complexity is exponential in the number of variables, and a more efficient approach such as constraint programming or local search may be more suitable. + +EXAMPLE: + Start with an empty 4x4 chessboard: + [ . . . . ] + [ . . . . ] + [ . . . . ] + [ . . . . ] + Place the first queen in the first row: + [ Q . . . ] + [ . . . . ] + [ . . . . ] + [ . . . . ] + Place the second queen in the third column of the second row: + [ Q . . . ] + [ . . Q . ] + [ . . . . ] + [ . . . . ] + Place the third queen in the second column of the third row: + [ Q . . . ] + [ . . Q . ] + [ . Q . . ] + [ . . . . ] + Place the fourth queen in the fourth column of the fourth row: + [ Q . . . ] + [ . . Q . ] + [ . Q . . ] + [ . . . Q ] + Checkmark: This is a valid solution to the 4-Queens Problem. + +REAL_WORLD_ANALOGY: The N-Queens Problem is similar to planning a seating arrangement for a dinner party, where we need to seat a group of people at a table such that no two people who don't get along are seated next to each other. + +SOURCE_NOTE: Concepts referenced from general knowledge of constraint satisfaction problems and backtracking algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/prims_algorithm_mst.txt b/rag/knowledge_base/prims_algorithm_mst.txt new file mode 100644 index 0000000000000000000000000000000000000000..8975f4f4fb5c02cbeb75cdcc96c2efabf6b18da3 --- /dev/null +++ b/rag/knowledge_base/prims_algorithm_mst.txt @@ -0,0 +1,30 @@ +TOPIC: Prim's Algorithm (MST) + +DEFINITION: +Prim's Algorithm is a method used to find the Minimum Spanning Tree (MST) of a connected, undirected, and weighted graph. It solves the problem of selecting the subset of edges in a graph that connect all the vertices together while minimizing the total edge cost. This is particularly useful in network design and optimization problems. + +TIME_COMPLEXITY: +The time complexity of Prim's Algorithm is O(E + V log V) in the worst case, where E is the number of edges and V is the number of vertices, due to the use of a priority queue to efficiently select the next edge to add to the MST. + +SPACE_COMPLEXITY: +The space complexity is O(V + E), which is used to store the graph and the MST. + +USE_WHEN: +Prim's Algorithm is the right tool when you need to find the minimum spanning tree of a connected graph, especially in scenarios where the graph is dense or has a large number of vertices. It's particularly useful in network design problems, such as designing a telecommunications network or a transportation system. + +AVOID_WHEN: +You should avoid using Prim's Algorithm when the graph is very sparse or when you need to find the shortest path between two specific vertices, in which case Dijkstra's Algorithm or other shortest path algorithms would be more suitable. + +EXAMPLE: +Consider a graph with vertices A, B, C, and edges [A-B:2, A-C:3, B-C:1]. + Start with vertex A, and the MST is [A]. + Select the edge A-B:2, and the MST becomes [A, B]. + Select the edge B-C:1, and the MST becomes [A, B, C]. + The final MST is [A-B:2, B-C:1] with a total cost of 3 + Checkmark: The minimum spanning tree has been found. + +REAL_WORLD_ANALOGY: +Finding the Minimum Spanning Tree using Prim's Algorithm is similar to building a network of roads between cities while trying to minimize the total cost of construction, ensuring that all cities are connected. + +SOURCE_NOTE: +Concepts referenced from general knowledge of graph theory and algorithms. \ No newline at end of file diff --git a/rag/knowledge_base/queue.txt b/rag/knowledge_base/queue.txt new file mode 100644 index 0000000000000000000000000000000000000000..29cbf47a97a9aa61015bf873be2c2a1fc023a67b --- /dev/null +++ b/rag/knowledge_base/queue.txt @@ -0,0 +1,29 @@ +TOPIC: Queue + +DEFINITION: +A queue is a linear data structure that follows the First-In-First-Out (FIFO) principle, where elements are added from one end and removed from the other, allowing for efficient management of items in a specific order. This helps solve problems that require processing items in a sequential manner, such as job scheduling or print queues. + +TIME_COMPLEXITY: +The time complexity for enqueue and dequeue operations in a queue is O(1), as these operations involve adding or removing an element from the end or front of the queue, which takes constant time. + +SPACE_COMPLEXITY: +The space complexity of a queue is O(n), where n is the number of elements in the queue, as each element occupies a certain amount of space in memory. + +USE_WHEN: +Use a queue when you need to process items in a specific order, such as handling requests in the order they were received, or when you need to manage a pool of resources that are allocated and deallocated in a particular sequence. + +AVOID_WHEN: +Avoid using a queue when you need to frequently access or remove elements from the middle of the sequence, as this can be inefficient; in such cases, consider using a different data structure like a linked list or an array. + +EXAMPLE: +Initial queue: [1, 2, 3] +Enqueue 4: [1, 2, 3, 4] +Dequeue: [2, 3, 4] +Enqueue 5: [2, 3, 4, 5] +Dequeue: [3, 4, 5] +Result: [3, 4, 5] + +REAL_WORLD_ANALOGY: +A queue is similar to a line of people waiting to buy tickets at a movie theater, where the first person in line is the first to be served, and each person waits their turn in the order they arrived. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/quick_sort.txt b/rag/knowledge_base/quick_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..fc54b997b354e6980e8572cfa3252fce1af0374b --- /dev/null +++ b/rag/knowledge_base/quick_sort.txt @@ -0,0 +1,24 @@ +TOPIC: Quick Sort + +DEFINITION: Quick Sort is a popular sorting algorithm that efficiently arranges elements in an array in ascending or descending order by selecting a pivot element, partitioning the array around it, and recursively sorting the sub-arrays. This process solves the problem of sorting large datasets by breaking it down into smaller, more manageable chunks. It's a divide-and-conquer algorithm that's widely used due to its simplicity and performance. + +TIME_COMPLEXITY: The time complexity of Quick Sort is O(n^2) in the worst case, O(n log n) on average, and O(n log n) in the best case, depending on the choice of pivot and the initial order of the elements, with the average case being the most common scenario due to the randomization of pivot selection. + +SPACE_COMPLEXITY: The space complexity of Quick Sort is O(log n), as it uses recursive function calls to sort the sub-arrays, which can go up to a depth of log n in the call stack. + +USE_WHEN: Quick Sort is the right tool when you need to sort large datasets efficiently, especially when memory is limited, as it only requires a small amount of extra memory for the recursive call stack. It's also suitable for datasets that are partially sorted or have a mix of ordered and random elements. + +AVOID_WHEN: Quick Sort is a poor choice when the dataset is already sorted in reverse order or has a specific pattern that can cause the worst-case scenario, in which case other algorithms like Merge Sort or Heap Sort might be more suitable. + +EXAMPLE: + Initial array: [5, 2, 8, 3, 1] + Select pivot: 5 + Partition: [2, 3, 1] | 5 | [8] + Recursively sort left: [1, 2, 3] + Recursively sort right: [8] + Combine: [1, 2, 3] | 5 | [8] + Result: [1, 2, 3, 5, 8] + +REAL_WORLD_ANALOGY: Quick Sort is similar to how you would sort a stack of papers by selecting a pivot paper, putting all the papers with a lower value to its left and all the papers with a higher value to its right, and then recursively sorting the two piles. + +SOURCE_NOTE: Concepts referenced from general knowledge of algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/radix_sort.txt b/rag/knowledge_base/radix_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..f1f909f731f04b49c637bf3780ad7532018f6ebb --- /dev/null +++ b/rag/knowledge_base/radix_sort.txt @@ -0,0 +1,38 @@ +TOPIC: radix sort + +DEFINITION: Radix sort is a non-comparative sorting algorithm that sorts integers or strings by processing individual digits or characters from least significant to most significant. This approach allows radix sort to efficiently sort large datasets of integers or strings with a fixed length. It solves the problem of sorting data with a large range of values. + +TIME_COMPLEXITY: The time complexity of radix sort is O(nk), where n is the number of elements and k is the number of digits in the radix sort, because it needs to iterate over each digit of each element. + +SPACE_COMPLEXITY: The space complexity of radix sort is O(n + k), as it requires additional space to store the buckets for each digit. + +USE_WHEN: Radix sort is the right tool when sorting large datasets of integers or strings with a fixed length, as it can take advantage of the structure of the data to achieve high performance. It is particularly useful when the range of input values is large. + +AVOID_WHEN: Radix sort is a poor choice when sorting data with a large number of unique elements or variable-length strings, as it can be less efficient than other sorting algorithms like quicksort or mergesort. + +EXAMPLE: + Initial array: [170, 45, 75, 90, 802, 24, 2, 66] + Sorting by least significant digit: + 0: [170, 90] + 2: [2, 24] + 4: [45] + 5: [75] + 6: [66] + 2: [802] + Combine: [2, 24, 45, 66, 75, 90, 170, 802] + Sorting by next digit: + 0: [2, 24] + 1: [170] + 4: [45] + 6: [66] + 7: [75, 90, 802] + Combine: [2, 24, 45, 66, 75, 90, 170, 802] + Sorting by most significant digit: + 0: [2, 24, 45, 66, 75, 90] + 1: [170] + 8: [802] + Final sorted array: [2, 24, 45, 66, 75, 90, 170, 802] + +REAL_WORLD_ANALOGY: Radix sort is similar to sorting a collection of books by their ISBN numbers, where you first sort by the last digit, then the second-to-last, and so on, until you have sorted by the entire number. + +SOURCE_NOTE: Concepts referenced from general knowledge of sorting algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/recursion_overview.txt b/rag/knowledge_base/recursion_overview.txt new file mode 100644 index 0000000000000000000000000000000000000000..3472d6e0a173d5090cd223f8394a3d0e7fff3334 --- /dev/null +++ b/rag/knowledge_base/recursion_overview.txt @@ -0,0 +1,28 @@ +TOPIC: Recursion Overview + +DEFINITION: Recursion is a programming technique where a function invokes itself to solve a problem by breaking it down into smaller instances of the same problem. This approach helps in solving complex problems that can be decomposed into simpler sub-problems, making it easier to understand and implement the solution. It's particularly useful for problems that have a recursive structure. + +TIME_COMPLEXITY: The time complexity of recursion can vary depending on the problem, but in general, it's O(2^n) in the worst case for problems like tree traversals, O(n) for problems like array sum, and can be optimized to O(log n) for problems like binary search, due to the repeated function calls. + +SPACE_COMPLEXITY: The space complexity of recursion is typically O(n), where n is the maximum depth of the recursion call stack, as each recursive call adds a new layer to the call stack. + +USE_WHEN: Recursion is the right tool when dealing with problems that have a recursive structure, such as tree or graph traversals, or when the problem can be broken down into smaller sub-problems of the same type. It's particularly useful when the problem has a clear base case and a recursive case. + +AVOID_WHEN: Recursion can be a poor choice when dealing with large problems or problems that require a high degree of efficiency, as the repeated function calls can lead to stack overflow errors, and in such cases, iterative solutions are often preferred. + +EXAMPLE: +Consider a simple recursive function to calculate the sum of an array: [1, 2, 3, 4] + -> sum([1, 2, 3, 4]) = 1 + sum([2, 3, 4]) + -> sum([2, 3, 4]) = 2 + sum([3, 4]) + -> sum([3, 4]) = 3 + sum([4]) + -> sum([4]) = 4 + sum([]) + -> sum([]) = 0 (base case) + <- sum([4]) = 4 + <- sum([3, 4]) = 7 + <- sum([2, 3, 4]) = 9 + <- sum([1, 2, 3, 4]) = 10 + Checkmark: sum = 10 + +REAL_WORLD_ANALOGY: Recursion is similar to a set of Russian nesting dolls, where each doll contains a smaller version of itself, and the process of opening each doll to find the smaller one is similar to the recursive function calls. + +SOURCE_NOTE: Concepts referenced from general knowledge of programming principles and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/segment_tree.txt b/rag/knowledge_base/segment_tree.txt new file mode 100644 index 0000000000000000000000000000000000000000..f636f44bc1044d10abda0f11bd75f936485aad4e --- /dev/null +++ b/rag/knowledge_base/segment_tree.txt @@ -0,0 +1,34 @@ +TOPIC: Segment Tree + +DEFINITION: A segment tree is a binary tree data structure that allows for efficient querying and updating of ranges within an array. It solves the problem of performing range queries, such as finding the sum or minimum value within a range, in logarithmic time. This is particularly useful when dealing with large datasets and frequent range queries. + +TIME_COMPLEXITY: The time complexity of segment tree operations is O(log n) for both query and update operations, where n is the size of the input array. This is because each node in the tree represents a range of values, and traversing the tree to find a specific range takes logarithmic time. + +SPACE_COMPLEXITY: The space complexity of a segment tree is O(n), as the tree requires additional space to store the range information for each node. + +USE_WHEN: Use a segment tree when you need to perform frequent range queries on a large dataset, such as finding the sum or minimum value within a range. This data structure is particularly useful when the range queries are overlapping or when the dataset is too large to fit into memory. + +AVOID_WHEN: Avoid using a segment tree when the dataset is small or when the range queries are non-overlapping, as the overhead of creating and maintaining the tree may outweigh the benefits. In such cases, a simple array or a hash table may be a more suitable choice. + +EXAMPLE: +Suppose we have an array [1, 3, 5, 7, 9] and we want to find the sum of the range [2, 4]. + Create the segment tree: + [1, 3, 5, 7, 9] + / \ + [1, 3] [5, 7, 9] + / \ / \ +[1] [3] [5] [7, 9] + / \ + [7] [9] + To find the sum of the range [2, 4], we traverse the tree: + Start at the root [1, 3, 5, 7, 9] + -> Go to the right child [5, 7, 9] + -> Go to the left child [5] + -> Go to the right child [7, 9] + -> Go to the left child [7] + The sum of the range [2, 4] is 5 + 7 = 12 + Checkmark: The result is 12. + +REAL_WORLD_ANALOGY: A segment tree can be thought of as a hierarchical filing system, where each folder represents a range of files, and querying a range is like finding a specific folder and its contents. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/selection_sort.txt b/rag/knowledge_base/selection_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..ab3d4a1a8191008561aa3026526c8dadd3e534b4 --- /dev/null +++ b/rag/knowledge_base/selection_sort.txt @@ -0,0 +1,31 @@ +TOPIC: Selection Sort + +DEFINITION: +Selection sort is a simple comparison-based sorting algorithm that works by repeatedly finding the minimum element from the unsorted part of the list and swapping it with the first unsorted element. This process continues until the entire list is sorted, solving the problem of arranging elements in ascending or descending order. + +TIME_COMPLEXITY: +The time complexity of selection sort is O(n^2) in all cases (best, average, and worst), because it involves two nested loops that iterate over the list, resulting in quadratic time complexity. + +SPACE_COMPLEXITY: +The space complexity of selection sort is O(1), as it only uses a constant amount of additional space to store temporary variables, regardless of the size of the input list. + +USE_WHEN: +Selection sort is suitable when the list is small or when memory writes are expensive, as it minimizes the number of swaps. It's also a good choice for educational purposes, as its simplicity makes it easy to understand and implement. + +AVOID_WHEN: +Selection sort is not suitable for large lists, as its quadratic time complexity makes it inefficient compared to other sorting algorithms like quicksort or mergesort, which have average-case time complexities of O(n log n). + +EXAMPLE: +Suppose we have the list [5, 2, 8, 3, 1]. The selection sort algorithm works as follows: + Initial list: [5, 2, 8, 3, 1] + Find minimum (1) and swap with first element: [1, 2, 8, 3, 5] + Find minimum in remaining list (2) and swap with first unsorted element: [1, 2, 8, 3, 5] + Find minimum in remaining list (3) and swap with first unsorted element: [1, 2, 3, 8, 5] + Find minimum in remaining list (5) and swap with first unsorted element: [1, 2, 3, 5, 8] + Find minimum in remaining list (8) and swap with first unsorted element: [1, 2, 3, 5, 8] + Result: [1, 2, 3, 5, 8] + +REAL_WORLD_ANALOGY: +Selection sort is similar to how you might organize a set of books on a shelf, where you repeatedly find the book that belongs in the next spot and move it there, until the entire shelf is organized. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/singly_linked_list.txt b/rag/knowledge_base/singly_linked_list.txt new file mode 100644 index 0000000000000000000000000000000000000000..0814b8e00e6f6f5c28acdc92d5a093af461f596b --- /dev/null +++ b/rag/knowledge_base/singly_linked_list.txt @@ -0,0 +1,31 @@ +TOPIC: Singly Linked List + +DEFINITION: +A singly linked list is a linear data structure where each element, or node, contains a value and a reference, or link, to the next node in the list. This allows for efficient insertion and deletion of nodes at any point in the list, solving the problem of dynamic memory allocation and node rearrangement. It's particularly useful when the list is constantly changing. + +TIME_COMPLEXITY: +The time complexity of a singly linked list is O(1) for insertion and deletion at the beginning of the list, but O(n) for insertion and deletion at the end or at a specific position, where n is the number of nodes in the list. + +SPACE_COMPLEXITY: +The space complexity of a singly linked list is O(n), where n is the number of nodes in the list, as each node requires a constant amount of space to store its value and the reference to the next node. + +USE_WHEN: +A singly linked list is the right tool when you need to frequently insert or delete nodes at arbitrary positions in the list, and memory efficiency is a concern. This data structure is particularly useful in scenarios where the list is constantly changing, such as in a database query result set. + +AVOID_WHEN: +A singly linked list is a poor choice when you need to frequently access nodes at arbitrary positions in the list, as this operation can be slow due to the need to traverse the list from the beginning. In such cases, a doubly linked list or an array-based data structure may be more suitable. + +EXAMPLE: +Suppose we have a singly linked list with the following nodes: [1] -> [2] -> [3]. If we want to insert a new node with value 4 at the end of the list, the steps would be: + Start with the list: [1] -> [2] -> [3] + Create a new node: [4] + Traverse the list to find the last node: [1] -> [2] -> [3] + Update the reference of the last node to point to the new node: [1] -> [2] -> [3] -> [4] + The resulting list is: [1] -> [2] -> [3] -> [4] + Checkmark: The new node is now part of the list. + +REAL_WORLD_ANALOGY: +A singly linked list can be thought of as a train with cars, where each car represents a node and the connection between cars represents the link between nodes. Just as you can add or remove cars from the train, you can insert or delete nodes from the list. + +SOURCE_NOTE: +Concepts referenced from general data structures and algorithms principles. \ No newline at end of file diff --git a/rag/knowledge_base/sliding_window_technique.txt b/rag/knowledge_base/sliding_window_technique.txt new file mode 100644 index 0000000000000000000000000000000000000000..22f2d206abfc21d2c211cb63b7783e2e4cb99ee4 --- /dev/null +++ b/rag/knowledge_base/sliding_window_technique.txt @@ -0,0 +1,23 @@ +TOPIC: Sliding Window Technique + +DEFINITION: The Sliding Window Technique is a problem-solving approach used in array or string problems that involves creating a window of elements, which moves over the data structure to find a specific pattern or property. This technique helps to reduce the time complexity by avoiding redundant calculations and focusing on a specific subset of data at a time. It is particularly useful in problems that require finding a maximum, minimum, or specific pattern within a given range. + +TIME_COMPLEXITY: O(n), where n is the size of the input array or string, because in the worst case, the window will traverse the entire data structure once. + +SPACE_COMPLEXITY: O(1), as the space used does not grow with the size of the input, assuming the window size is constant and does not depend on the input size. + +USE_WHEN: This technique is useful when dealing with array or string problems that require finding a specific pattern or property within a given range, such as finding the maximum sum of a subarray of a certain size. It is particularly effective when the problem can be solved by examining a subset of the data at a time. + +AVOID_WHEN: The Sliding Window Technique may not be the best choice for problems that require examining each element in isolation or problems with complex dependencies between elements, in which case a different approach such as dynamic programming might be more suitable. + +EXAMPLE: +Consider finding the maximum sum of a subarray of size 3 in the array [1, 2, 3, 4, 5]. + Initial window: [1, 2, 3], sum = 6 + -> Move window to the right: [2, 3, 4], sum = 9 + -> Move window to the right: [3, 4, 5], sum = 12 + The maximum sum is 12, which is the result of the subarray [3, 4, 5] + Result: + +REAL_WORLD_ANALOGY: The Sliding Window Technique can be compared to looking through a moving frame of a video, where the frame represents the window, and as it moves, it captures different scenes or patterns. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/stack.txt b/rag/knowledge_base/stack.txt new file mode 100644 index 0000000000000000000000000000000000000000..20fe358ee79f23e64ac37ffa539eb2d2a05d7e38 --- /dev/null +++ b/rag/knowledge_base/stack.txt @@ -0,0 +1,25 @@ +TOPIC: Stack + +DEFINITION: A stack is a linear data structure that follows the Last-In-First-Out (LIFO) principle, meaning the last element added to the stack will be the first one to be removed. This data structure is useful for solving problems that require reversing the order of elements or parsing expressions with nested structures. + +TIME_COMPLEXITY: O(1) for push and pop operations, as they involve adding or removing elements from the top of the stack, making them constant time operations. + +SPACE_COMPLEXITY: O(n), where n is the number of elements in the stack, as each element occupies a portion of memory. + +USE_WHEN: Use a stack when you need to parse expressions with nested structures, such as evaluating postfix expressions or parsing XML/HTML tags, or when you need to implement recursive algorithms iteratively. + +AVOID_WHEN: Avoid using a stack when you need to frequently access elements in the middle of the data structure, as this would require popping all the elements above it, and then pushing them back; in such cases, a different data structure like a list or array would be more suitable. + +EXAMPLE: +Initial stack: [ ] +1. Push 1: [1] +2. Push 2: [1, 2] +3. Push 3: [1, 2, 3] +4. Pop: [1, 2] -> 3 +5. Pop: [1] -> 2 +6. Pop: [] -> 1 +Result: All elements have been popped in reverse order: 3, 2, 1 + +REAL_WORLD_ANALOGY: A stack can be thought of as a pile of plates, where you add and remove plates from the top of the pile, following the LIFO principle. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/subset_sum_power_set_generation.txt b/rag/knowledge_base/subset_sum_power_set_generation.txt new file mode 100644 index 0000000000000000000000000000000000000000..787a806d99ff547cceabc81f5fec9edc841bf247 --- /dev/null +++ b/rag/knowledge_base/subset_sum_power_set_generation.txt @@ -0,0 +1,25 @@ +TOPIC: Subset Sum / Power Set Generation + +DEFINITION: Subset sum and power set generation are related problems where you have a set of numbers and you want to find all possible subsets of these numbers, or specifically, subsets that sum up to a certain target value. This problem is useful when you need to explore all combinations of elements from a given set. The power set generation is a more general problem that includes finding all subsets, while subset sum focuses on subsets with a specific sum. + +TIME_COMPLEXITY: O(2^n) because for each element in the set, there are two choices - include it in the subset or not, leading to 2^n possible subsets in total. + +SPACE_COMPLEXITY: O(2^n) as in the worst case, you might need to store all subsets of the given set, which can be 2^n in number. + +USE_WHEN: Use subset sum or power set generation when you need to consider all possible combinations of elements from a set, such as when solving problems that involve combinatorial optimization. This approach is particularly useful when the set is relatively small and you need to find a specific subset that meets certain criteria. + +AVOID_WHEN: Avoid using subset sum or power set generation for very large sets because the exponential time and space complexity can make the computation impractical. In such cases, consider using approximate algorithms or heuristics that can provide good solutions in a reasonable amount of time. + +EXAMPLE: +Given the set [1, 2, 3], find all subsets: +- Start with an empty set: [] +- For each element in the set, add it to all existing subsets: + - Add 1 to []: [1] + - Add 2 to [] and [1]: [2], [1, 2] + - Add 3 to [], [1], [2], [1, 2]: [3], [1, 3], [2, 3], [1, 2, 3] +- Resulting subsets: [], [1], [2], [3], [1, 2], [1, 3], [2, 3], [1, 2, 3] +Result: + +REAL_WORLD_ANALOGY: Generating all subsets of a set is similar to making a list of all possible teams that can be formed from a group of people, where each person can either be included in or excluded from a team. + +SOURCE_NOTE: Concepts referenced from general knowledge of combinatorial algorithms and data structures. \ No newline at end of file diff --git a/rag/knowledge_base/topic_metadata.json b/rag/knowledge_base/topic_metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..0366bbdbf1dd15160cf9b55371255fcee4414418 --- /dev/null +++ b/rag/knowledge_base/topic_metadata.json @@ -0,0 +1,331 @@ +{ + "linear_search.txt": { + "topic": "Linear Search", + "category": "Searching", + "difficulty": "Easy", + "prerequisites": ["Arrays"] + }, + "binary_search.txt": { + "topic": "Binary Search", + "category": "Searching", + "difficulty": "Easy", + "prerequisites": ["Arrays", "Sorting"] + }, + "two_pointer_technique.txt": { + "topic": "Two Pointer Technique", + "category": "Array Techniques", + "difficulty": "Easy", + "prerequisites": ["Arrays", "Loops"] + }, + "sliding_window_technique.txt": { + "topic": "Sliding Window Technique", + "category": "Array Techniques", + "difficulty": "Medium", + "prerequisites": ["Arrays", "Two Pointer Technique"] + }, + + "bubble_sort.txt": { + "topic": "Bubble Sort", + "category": "Sorting", + "difficulty": "Easy", + "prerequisites": ["Arrays", "Loops"] + }, + "selection_sort.txt": { + "topic": "Selection Sort", + "category": "Sorting", + "difficulty": "Easy", + "prerequisites": ["Arrays", "Loops"] + }, + "insertion_sort.txt": { + "topic": "Insertion Sort", + "category": "Sorting", + "difficulty": "Easy", + "prerequisites": ["Arrays"] + }, + "merge_sort.txt": { + "topic": "Merge Sort", + "category": "Sorting", + "difficulty": "Medium", + "prerequisites": ["Arrays", "Recursion"] + }, + "quick_sort.txt": { + "topic": "Quick Sort", + "category": "Sorting", + "difficulty": "Medium", + "prerequisites": ["Arrays", "Recursion", "Divide and Conquer"] + }, + "heap_sort.txt": { + "topic": "Heap Sort", + "category": "Sorting", + "difficulty": "Medium", + "prerequisites": ["Arrays", "Heap"] + }, + "counting_sort.txt": { + "topic": "Counting Sort", + "category": "Sorting", + "difficulty": "Medium", + "prerequisites": ["Arrays", "Hashing"] + }, + "radix_sort.txt": { + "topic": "Radix Sort", + "category": "Sorting", + "difficulty": "Medium", + "prerequisites": ["Counting Sort", "Number Representation"] + }, + "bucket_sort.txt": { + "topic": "Bucket Sort", + "category": "Sorting", + "difficulty": "Medium", + "prerequisites": ["Arrays", "Hashing"] + }, + + "singly_linked_list.txt": { + "topic": "Singly Linked List", + "category": "Linked Lists", + "difficulty": "Easy", + "prerequisites": ["Pointers", "Structures"] + }, + "doubly_linked_list.txt": { + "topic": "Doubly Linked List", + "category": "Linked Lists", + "difficulty": "Easy", + "prerequisites": ["Pointers", "Singly Linked List"] + }, + "circular_linked_list.txt": { + "topic": "Circular Linked List", + "category": "Linked Lists", + "difficulty": "Medium", + "prerequisites": ["Linked Lists", "Pointers"] + }, + + "stack.txt": { + "topic": "Stack", + "category": "Stacks and Queues", + "difficulty": "Easy", + "prerequisites": ["Arrays", "Linked Lists"] + }, + "queue.txt": { + "topic": "Queue", + "category": "Stacks and Queues", + "difficulty": "Easy", + "prerequisites": ["Arrays", "Linked Lists"] + }, + "circular_queue.txt": { + "topic": "Circular Queue", + "category": "Stacks and Queues", + "difficulty": "Medium", + "prerequisites": ["Queue", "Arrays"] + }, + "deque.txt": { + "topic": "Deque", + "category": "Stacks and Queues", + "difficulty": "Medium", + "prerequisites": ["Queue"] + }, + "monotonic_stack.txt": { + "topic": "Monotonic Stack", + "category": "Stacks and Queues", + "difficulty": "Hard", + "prerequisites": ["Stack", "Arrays"] + }, + + "binary_tree.txt": { + "topic": "Binary Tree", + "category": "Trees", + "difficulty": "Easy", + "prerequisites": ["Recursion"] + }, + "binary_search_tree.txt": { + "topic": "Binary Search Tree", + "category": "Trees", + "difficulty": "Medium", + "prerequisites": ["Binary Tree", "Searching"] + }, + "avl_tree.txt": { + "topic": "AVL Tree", + "category": "Trees", + "difficulty": "Hard", + "prerequisites": ["Binary Search Tree", "Rotations"] + }, + "heap_minmax.txt": { + "topic": "Heap (Min/Max)", + "category": "Trees", + "difficulty": "Medium", + "prerequisites": ["Binary Tree", "Arrays"] + }, + "trie.txt": { + "topic": "Trie", + "category": "Trees", + "difficulty": "Medium", + "prerequisites": ["Trees", "Strings"] + }, + "segment_tree.txt": { + "topic": "Segment Tree", + "category": "Trees", + "difficulty": "Hard", + "prerequisites": ["Trees", "Recursion"] + }, + "fenwick_tree_binary_indexed_tree.txt": { + "topic": "Fenwick Tree (Binary Indexed Tree)", + "category": "Trees", + "difficulty": "Hard", + "prerequisites": ["Arrays", "Prefix Sum"] + }, + + "bfs.txt": { + "topic": "Breadth-First Search", + "category": "Graph Traversal", + "difficulty": "Medium", + "prerequisites": ["Graphs", "Queues"] + }, + "dfs.txt": { + "topic": "Depth-First Search", + "category": "Graph Traversal", + "difficulty": "Medium", + "prerequisites": ["Graphs", "Recursion", "Stacks"] + }, + "dijkstras_algorithm.txt": { + "topic": "Dijkstra's Algorithm", + "category": "Graph Algorithms", + "difficulty": "Medium", + "prerequisites": ["Graphs", "Heap", "BFS"] + }, + "bellman-ford_algorithm.txt": { + "topic": "Bellman-Ford Algorithm", + "category": "Graph Algorithms", + "difficulty": "Hard", + "prerequisites": ["Graphs", "Shortest Path"] + }, + "floyd-warshall_algorithm.txt": { + "topic": "Floyd-Warshall Algorithm", + "category": "Graph Algorithms", + "difficulty": "Hard", + "prerequisites": ["Graphs", "Dynamic Programming"] + }, + "kruskals_algorithm_mst.txt": { + "topic": "Kruskal's Algorithm (MST)", + "category": "Minimum Spanning Tree", + "difficulty": "Medium", + "prerequisites": ["Graphs", "Union-Find"] + }, + "prims_algorithm_mst.txt": { + "topic": "Prim's Algorithm (MST)", + "category": "Minimum Spanning Tree", + "difficulty": "Medium", + "prerequisites": ["Graphs", "Heap"] + }, + "topological_sort.txt": { + "topic": "Topological Sort", + "category": "Graph Traversal", + "difficulty": "Medium", + "prerequisites": ["Graphs", "DFS"] + }, + "union-find_disjoint_set.txt": { + "topic": "Union-Find (Disjoint Set)", + "category": "Graph Algorithms", + "difficulty": "Medium", + "prerequisites": ["Graphs", "Trees"] + }, + "01_knapsack.txt": { + "topic": "0/1 Knapsack", + "category": "Dynamic Programming", + "difficulty": "Medium", + "prerequisites": ["Recursion", "Dynamic Programming Overview"] + }, + "activity_selection_problem.txt": { + "topic": "Activity Selection Problem", + "category": "Greedy", + "difficulty": "Medium", + "prerequisites": ["Sorting", "Greedy Algorithms Overview"] + }, + "backtracking_overview.txt": { + "topic": "Backtracking Overview", + "category": "Recursion and Backtracking", + "difficulty": "Medium", + "prerequisites": ["Recursion"] + }, + "bit_manipulation_basics.txt": { + "topic": "Bit Manipulation Basics", + "category": "Bit Manipulation", + "difficulty": "Easy", + "prerequisites": ["Number Representation"] + }, + "coin_change_problem.txt": { + "topic": "Coin Change Problem", + "category": "Dynamic Programming", + "difficulty": "Medium", + "prerequisites": ["Dynamic Programming Overview"] + }, + "dynamic_programming_overview.txt": { + "topic": "Dynamic Programming Overview", + "category": "Dynamic Programming", + "difficulty": "Medium", + "prerequisites": ["Recursion"] + }, + "edit_distance.txt": { + "topic": "Edit Distance", + "category": "Dynamic Programming", + "difficulty": "Hard", + "prerequisites": ["Dynamic Programming Overview", "Strings"] + }, + "fractional_knapsack.txt": { + "topic": "Fractional Knapsack", + "category": "Greedy", + "difficulty": "Medium", + "prerequisites": ["Sorting", "Greedy Algorithms Overview"] + }, + "greedy_algorithms_overview.txt": { + "topic": "Greedy Algorithms Overview", + "category": "Greedy", + "difficulty": "Medium", + "prerequisites": ["Sorting"] + }, + "hash_set_vs_hash_map.txt": { + "topic": "Hash Set vs Hash Map", + "category": "Hashing", + "difficulty": "Easy", + "prerequisites": ["Hash Table"] + }, + "hash_table.txt": { + "topic": "Hash Table", + "category": "Hashing", + "difficulty": "Easy", + "prerequisites": ["Arrays"] + }, + "longest_common_subsequence.txt": { + "topic": "Longest Common Subsequence", + "category": "Dynamic Programming", + "difficulty": "Medium", + "prerequisites": ["Dynamic Programming Overview", "Strings"] + }, + "longest_increasing_subsequence.txt": { + "topic": "Longest Increasing Subsequence", + "category": "Dynamic Programming", + "difficulty": "Medium", + "prerequisites": ["Dynamic Programming Overview", "Arrays"] + }, + "n-queens_problem.txt": { + "topic": "N-Queens Problem", + "category": "Recursion and Backtracking", + "difficulty": "Hard", + "prerequisites": ["Backtracking Overview"] + }, + "recursion_overview.txt": { + "topic": "Recursion Overview", + "category": "Recursion and Backtracking", + "difficulty": "Easy", + "prerequisites": ["Functions"] + }, + "subset_sum_power_set_generation.txt": { + "topic": "Subset Sum / Power Set Generation", + "category": "Recursion and Backtracking", + "difficulty": "Medium", + "prerequisites": ["Backtracking Overview", "Bit Manipulation Basics"] + }, + "xor_tricks.txt": { + "topic": "XOR Tricks", + "category": "Bit Manipulation", + "difficulty": "Medium", + "prerequisites": ["Bit Manipulation Basics"] + } +} \ No newline at end of file diff --git a/rag/knowledge_base/topological_sort.txt b/rag/knowledge_base/topological_sort.txt new file mode 100644 index 0000000000000000000000000000000000000000..063be1cc3c15f38ce075fa7dae10f8d76cf6de00 --- /dev/null +++ b/rag/knowledge_base/topological_sort.txt @@ -0,0 +1,34 @@ +TOPIC: Topological Sort + +DEFINITION: +A topological sort is an ordering of the vertices in a directed acyclic graph (DAG) such that for every directed edge u -> v, vertex u comes before v in the ordering. This helps solve the problem of ordering tasks or events in a way that respects their dependencies. It's a linear ordering of vertices that preserves the original graph's structure. + +TIME_COMPLEXITY: +The time complexity of a topological sort is O(V + E), where V is the number of vertices and E is the number of edges, because it involves visiting each vertex and edge once. + +SPACE_COMPLEXITY: +The space complexity is O(V), as it requires storing the visited vertices and the ordering, which in the worst case can be as large as the number of vertices. + +USE_WHEN: +Use a topological sort when you need to order a set of tasks or events that have dependencies between them, such as scheduling jobs or courses that have prerequisites. It's particularly useful when the dependencies form a DAG. + +AVOID_WHEN: +Avoid using a topological sort when the graph contains cycles, as it's not possible to find a valid ordering in such cases; instead, consider using a different algorithm or data structure, like a strongly connected components algorithm. + +EXAMPLE: +Consider a simple graph with vertices A, B, C, and D, and edges A -> B, A -> C, B -> D, C -> D. A topological sort would order these vertices as follows: + - Start with A (no incoming edges) + - Add B and C (both have A as a predecessor) + [A, B] + [A, C] + - Add D (has B and C as predecessors) + [A, B, D] + [A, C, D] + The final ordering can be [A, B, C, D] or [A, C, B, D], both are valid + Checkmark: [A, B, C, D] or [A, C, B, D] + +REAL_WORLD_ANALOGY: +A topological sort is similar to planning a meal, where you need to prepare ingredients in a specific order to ensure that each dish is ready on time and that you're using the ingredients in the right order. + +SOURCE_NOTE: +Concepts referenced from general graph theory and algorithm design principles. \ No newline at end of file diff --git a/rag/knowledge_base/trie.txt b/rag/knowledge_base/trie.txt new file mode 100644 index 0000000000000000000000000000000000000000..489dbac902f374ecbacdc60d2600276b243569b5 --- /dev/null +++ b/rag/knowledge_base/trie.txt @@ -0,0 +1,59 @@ +TOPIC: Trie + +DEFINITION: +A Trie, also known as a prefix tree, is a data structure that efficiently stores and retrieves a collection of strings by organizing them in a tree-like structure. It solves the problem of quickly finding all strings that start with a given prefix, making it useful for tasks like autocomplete and spell-checking. + +TIME_COMPLEXITY: +The time complexity of searching, inserting, and deleting operations in a Trie is O(m), where m is the length of the string being searched, inserted, or deleted, because each operation involves traversing the Trie from the root to a leaf node. + +SPACE_COMPLEXITY: +The space complexity of a Trie is O(n*m), where n is the number of strings stored and m is the average length of the strings, because each node in the Trie stores a set of child nodes and a flag indicating whether the node represents the end of a string. + +USE_WHEN: +Use a Trie when you need to store a large number of strings and quickly retrieve all strings that start with a given prefix, such as in autocomplete or spell-checking applications. It's also useful when you need to validate input strings against a set of valid strings. + +AVOID_WHEN: +Avoid using a Trie when you need to store a small number of very long strings, because the space required to store the Trie can be excessive; in such cases, a simple array or linked list of strings may be more efficient. + +EXAMPLE: +Suppose we want to store the strings "cat", "car", and "dog" in a Trie: + - Start with an empty Trie: [] + - Insert "cat": + [root] + | + -- 'c' + | + -- 'a' + | + -- 't' (end of string) + - Insert "car": + [root] + | + -- 'c' + | + -- 'a' + | + -- 'r' (end of string) + -- 't' (end of string) + - Insert "dog": + [root] + | + -- 'c' + | + -- 'a' + | + -- 'r' (end of string) + -- 't' (end of string) + -- 'd' + | + -- 'o' + | + -- 'g' (end of string) + - Search for strings starting with "ca": + [found: "cat", "car"] +Result: + +REAL_WORLD_ANALOGY: +A Trie is similar to a phonebook where all names starting with the same letter are grouped together, making it easy to find all names starting with a given letter. + +SOURCE_NOTE: \ No newline at end of file diff --git a/rag/knowledge_base/two_pointer_technique.txt b/rag/knowledge_base/two_pointer_technique.txt new file mode 100644 index 0000000000000000000000000000000000000000..71b19eb35602bc727d289807f7e05a1879afed66 --- /dev/null +++ b/rag/knowledge_base/two_pointer_technique.txt @@ -0,0 +1,23 @@ +TOPIC: Two Pointer Technique + +DEFINITION: The Two Pointer Technique is a problem-solving approach used in algorithms where two pointers, or references, are utilized to traverse through a data structure, such as an array or linked list, in a coordinated manner. This technique helps solve problems that require comparing or processing elements from different parts of the data structure. It is particularly useful for finding pairs or sequences that satisfy certain conditions. + +TIME_COMPLEXITY: O(n), where n is the number of elements in the data structure, because in the worst case, the pointers may need to traverse the entire data structure. + +SPACE_COMPLEXITY: O(1), as the technique typically only uses a constant amount of space to store the two pointers, regardless of the size of the input data structure. + +USE_WHEN: The Two Pointer Technique is useful when you need to find a pair of elements in a sorted or partially sorted data structure that satisfy a certain condition, such as finding a pair of numbers that add up to a target sum. It is also useful for problems that require comparing elements from different parts of the data structure. + +AVOID_WHEN: This technique may not be the best choice for problems that require random access to elements in the data structure, as the pointers can only move in a linear fashion. In such cases, a different approach, such as using a hash table, may be more suitable. + +EXAMPLE: +Consider finding a pair of numbers in a sorted array [1, 2, 3, 4, 5] that add up to 7. + Start with two pointers, one at the beginning (1) and one at the end (5). + Compare the sum of the values at the two pointers: 1 + 5 = 6, which is less than 7. + Move the left pointer to the right: [1 -> 2, 2, 3, 4, 5 <- 5]. + Compare the sum again: 2 + 5 = 7, which is equal to 7. + Result: [2, 5] + +REAL_WORLD_ANALOGY: The Two Pointer Technique can be thought of as two people working together to find a specific pair of items in a long list, with one person starting from the beginning and the other from the end, and moving towards each other until they find the desired pair. + +SOURCE_NOTE: Concepts referenced from general algorithmic principles and problem-solving strategies. \ No newline at end of file diff --git a/rag/knowledge_base/union-find_disjoint_set.txt b/rag/knowledge_base/union-find_disjoint_set.txt new file mode 100644 index 0000000000000000000000000000000000000000..64b3085d9e045b5684efd3592835fad4e56caa09 --- /dev/null +++ b/rag/knowledge_base/union-find_disjoint_set.txt @@ -0,0 +1,31 @@ +TOPIC: Union-Find (Disjoint Set) + +DEFINITION: +The Union-Find data structure, also known as Disjoint Set, is a way to keep track of a collection of elements that are divided into distinct groups, or sets. It helps solve the problem of determining whether two elements belong to the same set and allows for efficiently merging sets. + +TIME_COMPLEXITY: +The time complexity of Union-Find operations is nearly constant, O(α(n)), where α(n) is the inverse Ackermann function, which grows very slowly. This is because the operations are optimized using path compression and union by rank. + +SPACE_COMPLEXITY: +The space complexity is O(n), where n is the number of elements, as each element needs to be stored in the data structure. + +USE_WHEN: +Use Union-Find when you need to manage a collection of elements that are divided into distinct groups and need to efficiently determine whether two elements are in the same group or merge groups. This is particularly useful in scenarios like network connectivity or social network analysis. + +AVOID_WHEN: +Avoid using Union-Find when dealing with complex relationships between elements that require more sophisticated data structures, such as graphs with weighted edges, and instead opt for a data structure like an adjacency list or matrix. + +EXAMPLE: +Suppose we have a set of friends [A, B, C, D] and we want to track their social connections: + Initially, each friend is in their own set: [A], [B], [C], [D] + If A and B are friends, we union them: [A, B], [C], [D] + If C and D are friends, we union them: [A, B], [C, D] + If B and C are friends, we union [A, B] and [C, D]: [A, B, C, D] + Now, if we check if A and D are in the same set, the answer is yes + Result: All friends are now in one set, indicating they are all connected + +REAL_WORLD_ANALOGY: +The Union-Find data structure is similar to how people form and merge social cliques, where initially separate groups of friends may eventually become connected through mutual acquaintances. + +SOURCE_NOTE: +Concepts referenced from general knowledge of data structures and algorithms, synthesized for clarity and simplicity. \ No newline at end of file diff --git a/rag/knowledge_base/xor_tricks.txt b/rag/knowledge_base/xor_tricks.txt new file mode 100644 index 0000000000000000000000000000000000000000..635a4099af7000dcf579ff52d2a38fea8ec2c4e3 --- /dev/null +++ b/rag/knowledge_base/xor_tricks.txt @@ -0,0 +1,25 @@ +TOPIC: XOR Tricks + +DEFINITION: XOR tricks involve using the bitwise XOR operation to solve problems that require finding a single element in an array or detecting a cycle in a linked list. This technique takes advantage of the properties of XOR, which returns 1 if the two bits are different and 0 if they are the same, allowing for efficient and clever solutions. By applying XOR operations to elements in a specific order, we can eliminate duplicate values and find the desired result. + +TIME_COMPLEXITY: O(n), where n is the number of elements in the array or linked list, because we typically need to iterate through all elements once to apply the XOR operations. + +SPACE_COMPLEXITY: O(1), as we usually only need a constant amount of space to store the result of the XOR operations. + +USE_WHEN: XOR tricks are particularly useful when dealing with arrays or linked lists where we need to find a single element that appears only once, or when detecting a cycle in a linked list. They offer a clever and efficient solution to these problems. + +AVOID_WHEN: XOR tricks may not be the best choice when dealing with floating-point numbers or non-integer values, as the bitwise XOR operation is not well-defined for these types; instead, consider using other techniques like hashing or sorting. + +EXAMPLE: + Start with an array: [2, 3, 5, 2, 3] + Apply XOR to all elements: + 2 XOR 3 = 1 + 1 XOR 5 = 4 + 4 XOR 2 = 6 + 6 XOR 3 = 5 + Result: 5 + Checkmark: The single element that appears only once in the array is indeed 5. + +REAL_WORLD_ANALOGY: XOR tricks can be thought of like a game of "musical chairs" where all the pairs of identical elements "cancel each other out", leaving only the single unique element standing. + +SOURCE_NOTE: Concepts referenced from general knowledge of bitwise operations and algorithm design principles. \ No newline at end of file diff --git a/rag/retriever.py b/rag/retriever.py new file mode 100644 index 0000000000000000000000000000000000000000..f9d48ba10c319d093780101c59002355d5572596 --- /dev/null +++ b/rag/retriever.py @@ -0,0 +1,164 @@ +""" +Retrieval pipeline. + +Query -> Embedding -> Similarity Search (top INITIAL_RETRIEVAL_K) + -> Similarity Threshold Filter -> top TOP_K chunks + +No keyword-based topic detection. Retrieval is purely embedding-based. +If nothing clears the similarity threshold, the caller gets an empty list +and is expected to respond that the knowledge base doesn't have sufficient +information, rather than hallucinating. +""" + +import os +import sys + +import chromadb +from sentence_transformers import SentenceTransformer + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +import config +from logs.logger import get_logger + +logger = get_logger(__name__) + + +class Retriever: + """ + Thin wrapper around a persistent ChromaDB collection + a shared + sentence-transformers embedder. Instantiate once (e.g. as a module-level + singleton) so the embedding model isn't reloaded per request. + """ + + def __init__(self): + self.embedder = SentenceTransformer(config.EMBEDDING_MODEL_NAME) + self.client = chromadb.PersistentClient(path=config.CHROMA_DB_DIR) + self.collection = self.client.get_or_create_collection( + name=config.CHROMA_COLLECTION_NAME + ) + + @staticmethod + def _distance_to_similarity(distance: float) -> float: + """ + ChromaDB's default distance metric is cosine distance (0 = identical, + 2 = opposite). Convert to a similarity score in roughly [0, 1] so + SIMILARITY_THRESHOLD in config.py is intuitive to tune. + """ + return 1 - (distance / 2) + + def retrieve( + self, + query: str, + top_k: int = None, + initial_k: int = None, + similarity_threshold: float = None, + topic_filter: list = None, + bypass_threshold_if_single_topic: bool = False, + ) -> list: + """ + Args: + query: the user's natural-language question. + top_k: final number of chunks to return (defaults to config.TOP_K). + initial_k: candidate pool size before filtering + (defaults to config.INITIAL_RETRIEVAL_K). + similarity_threshold: minimum similarity to keep a chunk + (defaults to config.SIMILARITY_THRESHOLD). + topic_filter: optional list of topic names to restrict the + search to (used by the comparison agent to pull + chunks for specific topics). + + Returns: + A list of dicts, each: + { + "text": "...", + "metadata": {...}, + "similarity": 0.83 + } + sorted by descending similarity. Empty if nothing clears the + threshold. + """ + top_k = top_k or config.TOP_K + initial_k = initial_k or config.INITIAL_RETRIEVAL_K + similarity_threshold = ( + similarity_threshold + if similarity_threshold is not None + else config.SIMILARITY_THRESHOLD + ) + + # A confirmed single-topic lookup (router already matched the topic + # name with high confidence) doesn't need per-chunk similarity + # filtering — every section chunk for that topic is relevant, even if + # its wording (e.g. TIME_COMPLEXITY) doesn't closely match the user's + # phrasing. Thresholding here only causes false negatives. + if bypass_threshold_if_single_topic and topic_filter and len(topic_filter) == 1: + similarity_threshold = -1.0 # effectively disables the filter + initial_k = max(initial_k, 10) # make sure all sections fit in the pool + + query_embedding = self.embedder.encode([query]).tolist() + + where_clause = None + if topic_filter: + if len(topic_filter) == 1: + where_clause = {"topic": topic_filter[0]} + else: + where_clause = {"topic": {"$in": topic_filter}} + + results = self.collection.query( + query_embeddings=query_embedding, + n_results=initial_k, + where=where_clause, + ) + + documents = results.get("documents", [[]])[0] + metadatas = results.get("metadatas", [[]])[0] + distances = results.get("distances", [[]])[0] + + candidates = [] + for doc, meta, dist in zip(documents, metadatas, distances): + similarity = self._distance_to_similarity(dist) + candidates.append({"text": doc, "metadata": meta, "similarity": similarity}) + + # Similarity threshold filter + filtered = [c for c in candidates if c["similarity"] >= similarity_threshold] + + logger.info( + "Query=%r | candidates=%d | passed_threshold=%d | threshold=%.2f", + query, + len(candidates), + len(filtered), + similarity_threshold, + ) + + # Top-K after filtering (candidates already come back sorted by + # distance/similarity from ChromaDB, but sort defensively) + filtered.sort(key=lambda c: c["similarity"], reverse=True) + return filtered[:top_k] + + def retrieve_topics(self, topics: list, top_k_per_topic: int = None) -> dict: + """ + Convenience method for the comparison agent: retrieve top chunks + per topic rather than pooled across all topics. + + Returns: {topic_name: [chunk, chunk, ...]} + """ + top_k_per_topic = top_k_per_topic or config.TOP_K + result = {} + for topic in topics: + result[topic] = self.retrieve( + query=topic, + top_k=top_k_per_topic, + topic_filter=[topic], + ) + return result + + +_retriever_singleton = None + + +def get_retriever() -> Retriever: + """Module-level singleton accessor so the embedding model loads once.""" + global _retriever_singleton + if _retriever_singleton is None: + _retriever_singleton = Retriever() + return _retriever_singleton diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..b894942af6bc879bf51fe6a5006e84c9504ecb9b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,13 @@ +Flask==3.0.3 +Flask-SQLAlchemy==3.1.1 +python-dotenv==1.0.1 +Werkzeug==3.0.3 +gradio==5.9.1 +chromadb>=1.0.0 +sentence-transformers==3.3.1 +mem0ai==0.1.29 +groq==0.15.0 +torch==2.6.0 +transformers==4.47.1 +accelerate==1.2.1 +bitsandbytes==0.45.0 \ No newline at end of file diff --git a/ui/__init__.py b/ui/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/ui/callbacks.py b/ui/callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..6a909b75e914dd665d972e9ecfa40f5de3558440 --- /dev/null +++ b/ui/callbacks.py @@ -0,0 +1,198 @@ +""" +Gradio callback functions. + +These call chat_service / auth / history directly (in-process) rather than +going over HTTP, since the Gradio Blocks app is mounted into the same Flask +process (see app.py). Because Gradio callbacks run outside of a normal +Flask request context, every function that touches the database wraps its +body in `with _flask_app.app_context():`. + +Auth state for the UI is NOT the Flask session cookie (that's only for the +REST API in auth/routes.py) — it's tracked per-browser-tab in a gr.State +dict: {"user_id": int, "email": str} or None when logged out. +""" + +import time + +from auth.db import db +from auth.models import User +from chat_service import handle_chat_message +from history import service as history_service +from logs.logger import get_logger + +logger = get_logger(__name__) + +_flask_app = None + + +def init_app(flask_app): + """Called once from app.py so callbacks can open an app context.""" + global _flask_app + _flask_app = flask_app + + +# --------------------------------------------------------------------------- +# Auth +# --------------------------------------------------------------------------- + +def do_signup(email: str, password: str): + """Returns (auth_state, status_message, is_error).""" + email = (email or "").strip().lower() + password = password or "" + + if not email or "@" not in email: + return None, "Please enter a valid email address.", True + if len(password) < 8: + return None, "Password must be at least 8 characters.", True + + with _flask_app.app_context(): + if User.query.filter_by(email=email).first() is not None: + return None, "An account with this email already exists.", True + + user = User(email=email) + user.set_password(password) + db.session.add(user) + db.session.commit() + auth_state = {"user_id": user.id, "email": user.email} + + return auth_state, f"Account created. Welcome, {email}!", False + + +def do_login(email: str, password: str): + """Returns (auth_state, status_message, is_error).""" + email = (email or "").strip().lower() + password = password or "" + + with _flask_app.app_context(): + user = User.query.filter_by(email=email).first() + if user is None or not user.check_password(password): + return None, "Invalid email or password.", True + auth_state = {"user_id": user.id, "email": user.email} + + return auth_state, f"Logged in as {email}.", False + + +def do_logout(): + return None, "Logged out.", False + + +# --------------------------------------------------------------------------- +# Chat +# --------------------------------------------------------------------------- + +def send_message(user_message: str, chat_display: list, auth_state, session_state, anon_history: list): + """ + Args: + user_message: raw text from the input box. + chat_display: current [{"role": ..., "content": ...}, ...] shown in + gr.Chatbot(type="messages"). + auth_state: {"user_id", "email"} or None. + session_state: current chat_session id (logged-in only) or None. + anon_history: client-side rolling history used only when logged out. + + Returns: + (chat_display, session_state, anon_history, cleared_textbox) + """ + user_message = (user_message or "").strip() + if not user_message: + return chat_display, session_state, anon_history, "" + + chat_display = chat_display + [{"role": "user", "content": user_message}] + + is_logged_in = bool(auth_state) + + with _flask_app.app_context(): + if is_logged_in: + user = User.query.get(auth_state["user_id"]) + result = handle_chat_message( + query=user_message, + user=user, + session_id=session_state, + ) + session_state = result["session_id"] + else: + result = handle_chat_message( + query=user_message, + user=None, + anonymous_history=anon_history, + ) + anon_history = (anon_history or []) + [ + {"role": "user", "content": user_message}, + {"role": "assistant", "content": result["response"]}, + ] + + chat_display = chat_display + [{"role": "assistant", "content": result["response"]}] + return chat_display, session_state, anon_history, "" + + +def stream_last_response(chat_display: list): + """ + Fakes token-by-token streaming for the assistant's last message so the + UI feels responsive even though llm.generate() currently returns the + full string at once. Yields progressively longer chat_display copies. + """ + if not chat_display or chat_display[-1]["role"] != "assistant": + yield chat_display + return + + full_text = chat_display[-1]["content"] + prefix_display = chat_display[:-1] + + step = max(1, len(full_text) // 60) + for i in range(0, len(full_text) + step, step): + partial = full_text[:i] + yield prefix_display + [{"role": "assistant", "content": partial}] + time.sleep(0.012) + + yield chat_display + + +def new_chat(): + """Resets the chat window + clears the active session id.""" + return [], None, [] + + +# --------------------------------------------------------------------------- +# History panel +# --------------------------------------------------------------------------- + +def refresh_sessions(auth_state): + """Returns gr.update(...) choices for the session list, newest first.""" + import gradio as gr + + if not auth_state: + return gr.update(choices=[], value=None) + + with _flask_app.app_context(): + sessions = history_service.list_sessions(auth_state["user_id"]) + + choices = [(s["title"] or f"Session {s['id']}", s["id"]) for s in sessions] + return gr.update(choices=choices, value=None) + + +def open_session(session_id, auth_state): + """Loads a past session's full transcript into the chat window.""" + if not auth_state or session_id is None: + return [], None + + with _flask_app.app_context(): + full_session = history_service.get_full_session(session_id, auth_state["user_id"]) + + if full_session is None: + return [], None + + chat_display = [ + {"role": m["role"], "content": m["content"]} for m in full_session["messages"] + ] + return chat_display, session_id + + +def delete_session(session_id, auth_state): + """Deletes a session and returns a cleared chat window if it was active.""" + if not auth_state or session_id is None: + return [], None + + with _flask_app.app_context(): + history_service.delete_session(session_id, auth_state["user_id"]) + + return [], None diff --git a/ui/layout.py b/ui/layout.py new file mode 100644 index 0000000000000000000000000000000000000000..1f6e56b2ee3283024be471462713108b71b1b8be --- /dev/null +++ b/ui/layout.py @@ -0,0 +1,225 @@ +""" +Gradio Blocks layout for the DSA Tutor Chatbot. + +Custom layout only (per spec) — no gr.ChatInterface. Structure: + + Top bar: title, auth status, Login/Logout buttons + Sign-up dialog (modal-ish accordion, opened via a "Sign up" link) + Main row: + [ Toggleable history panel ] [ Chat window + input + send/new chat ] + +Streaming: the assistant's reply is revealed progressively via +ui.callbacks.stream_last_response (see that module's docstring for why +it's simulated rather than a true token stream). +""" + +import gradio as gr + +from ui import callbacks +from ui.theme import CUSTOM_CSS, dsa_theme + + +def build_ui(flask_app) -> gr.Blocks: + callbacks.init_app(flask_app) + + with gr.Blocks(theme=dsa_theme, css=CUSTOM_CSS, title="DSA Tutor Chatbot") as demo: + auth_state = gr.State(None) # {"user_id":.., "email":..} | None + session_state = gr.State(None) # active ChatSession.id | None + anon_history = gr.State([]) # rolling history for anonymous users + history_visible_state = gr.State(False) # tracks History panel visibility + + # --------------------------------------------------------------- + # Top bar + # --------------------------------------------------------------- + with gr.Row(elem_id="topbar"): + gr.Markdown("### 🧮 DSA Tutor Chatbot", elem_id="topbar-title") + with gr.Row(elem_id="topbar-actions"): + auth_status_md = gr.Markdown("Not logged in", elem_id="auth-status") + history_toggle_btn = gr.Button("📜 History", size="sm", scale=0, elem_classes=["small-btn"]) + login_btn = gr.Button("Log in", size="sm", scale=0, elem_classes=["small-btn"]) + signup_btn = gr.Button("Sign up", size="sm", scale=0, elem_classes=["small-btn"]) + logout_btn = gr.Button("Log out", size="sm", scale=0, visible=False, elem_classes=["small-btn"]) + + # --------------------------------------------------------------- + # Login popover + # --------------------------------------------------------------- + with gr.Group(visible=False) as login_panel: + gr.Markdown("#### Log in") + login_email = gr.Textbox(label="Email") + login_password = gr.Textbox(label="Password", type="password") + with gr.Row(): + login_submit = gr.Button("Log in", variant="primary", size="sm", elem_classes=["small-btn"]) + login_cancel = gr.Button("Cancel", size="sm", elem_classes=["small-btn"]) + login_status = gr.Markdown("") + + # --------------------------------------------------------------- + # Sign-up dialog + # --------------------------------------------------------------- + with gr.Group(visible=False) as signup_panel: + gr.Markdown("#### Create an account") + signup_email = gr.Textbox(label="Email") + signup_password = gr.Textbox(label="Password (min 8 characters)", type="password") + with gr.Row(): + signup_submit = gr.Button("Sign up", variant="primary", size="sm", elem_classes=["small-btn"]) + signup_cancel = gr.Button("Cancel", size="sm", elem_classes=["small-btn"]) + signup_status = gr.Markdown("") + + # --------------------------------------------------------------- + # Main area: history panel (toggleable) + chat + # --------------------------------------------------------------- + with gr.Row(elem_id="main-row"): + with gr.Column(scale=1, visible=False, elem_id="history-panel") as history_panel: + gr.Markdown("**Previous sessions**") + session_list = gr.Radio(choices=[], label="", elem_classes=["session-item"]) + with gr.Row(): + refresh_sessions_btn = gr.Button("Refresh", size="sm", elem_classes=["small-btn"]) + delete_session_btn = gr.Button("Delete selected", size="sm", elem_classes=["small-btn"]) + + with gr.Column(scale=4, elem_id="chat-column"): + chatbot = gr.Chatbot( + label="DSA Tutor", + type="messages", + height=360, + elem_id="chat-window", + avatar_images=(None, None), + ) + with gr.Row(elem_id="input-row"): + msg_box = gr.Textbox( + placeholder="Ask about any DSA topic (e.g. 'Explain merge sort')...", + show_label=False, + scale=8, + ) + send_btn = gr.Button("Send", variant="primary", scale=1, size="sm") + new_chat_btn = gr.Button("🆕 New chat", size="sm", elem_classes=["small-btn"]) + + # --------------------------------------------------------------- + # Wiring: top bar toggles + # --------------------------------------------------------------- + def _toggle_history(is_visible): + new_visible = not is_visible + return new_visible, gr.update(visible=new_visible) + + history_toggle_btn.click( + _toggle_history, + inputs=[history_visible_state], + outputs=[history_visible_state, history_panel], + ) + + login_btn.click(lambda: gr.update(visible=True), outputs=login_panel) + login_cancel.click(lambda: gr.update(visible=False), outputs=login_panel) + + signup_btn.click(lambda: gr.update(visible=True), outputs=signup_panel) + signup_cancel.click(lambda: gr.update(visible=False), outputs=signup_panel) + + def _auth_ui_update(auth_state_value): + if auth_state_value: + status = f"Logged in as **{auth_state_value['email']}**" + return ( + status, + gr.update(visible=False), # login_btn + gr.update(visible=False), # signup_btn + gr.update(visible=True), # logout_btn + ) + return ( + "Not logged in", + gr.update(visible=True), + gr.update(visible=True), + gr.update(visible=False), + ) + + def _on_login(email, password): + auth_state_value, message, is_error = callbacks.do_login(email, password) + status_md, login_show, signup_show, logout_show = _auth_ui_update(auth_state_value) + panel_visible = gr.update(visible=is_error) + return auth_state_value, status_md, login_show, signup_show, logout_show, panel_visible, message + + login_submit.click( + _on_login, + inputs=[login_email, login_password], + outputs=[auth_state, auth_status_md, login_btn, signup_btn, logout_btn, login_panel, login_status], + ) + + def _on_signup(email, password): + auth_state_value, message, is_error = callbacks.do_signup(email, password) + status_md, login_show, signup_show, logout_show = _auth_ui_update(auth_state_value) + panel_visible = gr.update(visible=is_error) + return auth_state_value, status_md, login_show, signup_show, logout_show, panel_visible, message + + signup_submit.click( + _on_signup, + inputs=[signup_email, signup_password], + outputs=[auth_state, auth_status_md, login_btn, signup_btn, logout_btn, signup_panel, signup_status], + ) + + def _on_logout(): + auth_state_value, message, _ = callbacks.do_logout() + status_md, login_show, signup_show, logout_show = _auth_ui_update(auth_state_value) + return auth_state_value, status_md, login_show, signup_show, logout_show, [], None, [] + + logout_btn.click( + _on_logout, + outputs=[ + auth_state, auth_status_md, login_btn, signup_btn, logout_btn, + chatbot, session_state, anon_history, + ], + ) + + # --------------------------------------------------------------- + # Wiring: chat + # --------------------------------------------------------------- + def _send_and_stream(user_message, chat_display, auth_state_value, session_state_value, anon_hist): + chat_display, session_state_value, anon_hist, cleared = callbacks.send_message( + user_message, chat_display, auth_state_value, session_state_value, anon_hist + ) + for partial in callbacks.stream_last_response(chat_display): + yield partial, session_state_value, anon_hist, cleared + + send_btn.click( + _send_and_stream, + inputs=[msg_box, chatbot, auth_state, session_state, anon_history], + outputs=[chatbot, session_state, anon_history, msg_box], + ) + msg_box.submit( + _send_and_stream, + inputs=[msg_box, chatbot, auth_state, session_state, anon_history], + outputs=[chatbot, session_state, anon_history, msg_box], + ) + + new_chat_btn.click( + callbacks.new_chat, + outputs=[chatbot, session_state, anon_history], + ) + + # --------------------------------------------------------------- + # Wiring: history panel + # --------------------------------------------------------------- + refresh_sessions_btn.click( + callbacks.refresh_sessions, + inputs=[auth_state], + outputs=[session_list], + ) + + session_list.change( + callbacks.open_session, + inputs=[session_list, auth_state], + outputs=[chatbot, session_state], + ) + + delete_session_btn.click( + callbacks.delete_session, + inputs=[session_list, auth_state], + outputs=[chatbot, session_state], + ).then( + callbacks.refresh_sessions, + inputs=[auth_state], + outputs=[session_list], + ) + + # Refresh the session list whenever a login/signup succeeds. + auth_state.change( + callbacks.refresh_sessions, + inputs=[auth_state], + outputs=[session_list], + ) + + return demo \ No newline at end of file diff --git a/ui/theme.py b/ui/theme.py new file mode 100644 index 0000000000000000000000000000000000000000..41992ecc34f73d100cfd502f9fff7da62d25df25 --- /dev/null +++ b/ui/theme.py @@ -0,0 +1,109 @@ +""" +Gradio theme + a few shared CSS tweaks. + +Blue / light-blue palette per spec. Kept intentionally simple/functional +rather than visually elaborate. +""" + +import gradio as gr + +dsa_theme = gr.themes.Soft( + primary_hue=gr.themes.colors.blue, + secondary_hue=gr.themes.colors.sky, + neutral_hue=gr.themes.colors.slate, +).set( + body_background_fill="*neutral_50", + body_background_fill_dark="*neutral_950", + block_background_fill="white", + block_border_width="1px", + block_shadow="*shadow_drop", + button_primary_background_fill="*primary_500", + button_primary_background_fill_hover="*primary_600", + button_primary_text_color="white", +) + +CUSTOM_CSS = """ +.gradio-container { + max-width: 90% !important; + margin: 0 auto !important; +} +#topbar { + display: flex; + align-items: center; + justify-content: space-between; + padding: 6px 4px; + border-bottom: 1px solid var(--border-color-primary); + margin-bottom: 6px; +} +#topbar-title { + font-size: 1.05rem; + font-weight: 700; + color: var(--primary-500); +} +#topbar-actions { + gap: 4px !important; +} +#auth-status { + font-size: 0.8rem; + color: var(--body-text-color-subdued); +} + +/* Shrink ONLY the buttons we explicitly tag - do not touch Gradio's + internal chatbot buttons (copy/like/dislike icons etc.), otherwise + their forced height squashes the surrounding message bubble and text + spills outside it. */ +.small-btn { + font-size: 0.78rem !important; + padding: 4px 10px !important; + min-height: unset !important; + height: 30px !important; + line-height: 1 !important; +} + +#history-panel { + border-right: 1px solid var(--border-color-primary); + padding-right: 8px; +} +.session-item { + cursor: pointer; +} + +/* Keep chat + input compact and always reachable without page scroll */ +#main-row { + max-height: 78vh; +} +#chat-column { + display: flex; + flex-direction: column; +} +#chat-window { + height: 360px !important; + min-height: 260px !important; + max-height: 55vh !important; + overflow-y: auto !important; +} + +/* Make sure message bubbles wrap and stay inside the chat window instead + of overflowing horizontally/vertically. */ +#chat-window .message, +#chat-window .message-wrap, +#chat-window .bot, +#chat-window .user { + max-width: 100% !important; + overflow-wrap: break-word !important; + word-break: break-word !important; + white-space: pre-wrap !important; + box-sizing: border-box !important; +} +#chat-window .message pre, +#chat-window .message code { + white-space: pre-wrap !important; + overflow-x: auto !important; +} + +#input-row { + margin-top: 4px; +} + +footer {display: none !important;} +""" \ No newline at end of file