Buckets:

tahamajs
/

DL_For_NLP

Files

xet

tahamajs/DL_For_NLP / learn.tex

tahamajs

about 1 month ago

download

raw

16 kB

\documentclass[conference]{IEEEtran}
\IEEEoverridecommandlockouts
\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amsfonts}
\usepackage{enumitem}
\usepackage{listings}
\usepackage{xcolor}
\usepackage{hyperref}

\lstdefinestyle{code}{
  basicstyle=\ttfamily\footnotesize,
  breaklines=true,
  frame=single,
  columns=fullflexible,
  backgroundcolor=\color[gray]{0.97},
  keywordstyle=\color{blue},
  commentstyle=\color{gray},
  showstringspaces=false
}

\title{Learning Guide: ReAct + Tool Use Agents (A05)}
\author{Course Mirror Assignment A05}
\date{}

\begin{document}
\maketitle

\begin{abstract}
This guide teaches the ReAct (reasoning+acting) paradigm, contrasts it with chain-of-thought (CoT), and walks through the provided code. It also introduces a Reflexion-style self-critique extension to improve ReAct. Use it as a teaching companion and checklist for students.
\end{abstract}

\begin{IEEEkeywords}
ReAct, tool use, Reflexion, chain-of-thought, HotpotQA, GSM8K
\end{IEEEkeywords}

\section{Motivation and Goals}
\begin{itemize}[leftmargin=*]
    \item Bridge LLM reasoning with external tools (search, calculator, python).
    \item Compare tool-augmented ReAct vs. CoT-only baselines.
    \item Study an improvement from recent work: Reflexion-style self-critique \cite{shinn2023reflexion}.
\end{itemize}

\section{Background}
\subsection{Chain-of-Thought (CoT)}
Stepwise natural-language reasoning \cite{wei2022chain}; strong but tool-less.

\subsection{ReAct}
Interleaves Thought $\rightarrow$ Action $\rightarrow$ Observation until Finish \cite{yao2023react}. Reduces hallucination and handles multi-hop reasoning.

\subsection{Tools}
\begin{itemize}[leftmargin=*]
    \item \textbf{Search}: BM25 over HotpotQA distractor paragraphs.
    \item \textbf{Calculator}: Safe AST arithmetic.
    \item \textbf{Python}: Sandboxed execution (PAL-style \cite{gao2023pal}); return value from \texttt{result}.
\end{itemize}

\section{Code Map}
\begin{itemize}[leftmargin=*]
    \item \texttt{src/agents/react.py}: ReAct loop and parsing.
    \item \texttt{src/agents/cot.py}: CoT baseline.
    \item \texttt{src/agents/reflexion.py}: ReAct + critic + revision (Reflexion-style).
    \item \texttt{src/tools/}: \texttt{search.py}, \texttt{calculator.py}, \texttt{python\_tool.py}.
    \item \texttt{src/eval.py}: evaluation harness and logging.
    \item \texttt{scripts/build\_index.py}: build BM25 corpus.
    \item \texttt{scripts/run\_eval.py}: CLI for react/cot/reflexion.
\end{itemize}

\section{Prompting Essentials}
\subsection{ReAct}
Specify available tools and enforce the loop:
\begin{lstlisting}[style=code]
Question: ...
Thought: ...
Action: search["query"]
Observation: ...
...
Finish: <answer>
\end{lstlisting}

\subsection{CoT}
Ask for step-by-step reasoning and end with \texttt{Final answer: <...>}.

\subsection{Reflexion Critique}
After a ReAct pass, a critic summarizes likely mistakes in 2--3 bullets; a revision pass rewrites with \texttt{Final answer: ...}.

\section{Agent Variants}
\subsection{ReAct}
Iterative tool calls; stops on \texttt{Finish}. Configurable max steps.

\subsection{CoT}
Single call; no tools.

\subsection{Reflexion Extension (New)}
1) Run ReAct. 2) Critic summarizes issues. 3) Revision produces a final answer using the critic (no extra tool calls for speed). Inspired by \cite{shinn2023reflexion}. Use to test if self-critique helps Hotpot multi-hop.

\section{Tools and Safety}
\begin{itemize}[leftmargin=*]
    \item Calculator/Python use restricted AST or limited builtins; Python has timeouts.
    \item Unknown tool names return explicit errors to avoid silent failure.
    \item Keep tool inputs short; avoid untrusted code.
\end{itemize}

\section{Data and Metrics}
\begin{itemize}[leftmargin=*]
    \item HotpotQA (distractor): metric = Exact Match (EM) after normalization.
    \item GSM8K: metric = accuracy on parsed numeric answers.
    \item Track avg tool calls, hallucinated tool names, runtime.
\end{itemize}

\section{Running}
\subsection{Setup}
\begin{lstlisting}[style=code]
pip install -r requirements.txt
export OPENAI_API_KEY=...
\end{lstlisting}

\subsection{Build Search Index}
\begin{lstlisting}[style=code]
python scripts/build_index.py --split train[:2000] --out data/hotpot_index.pkl
\end{lstlisting}

\subsection{Evaluate}
\begin{lstlisting}[style=code]
# ReAct on Hotpot
python scripts/run_eval.py --agent react --dataset hotpot --split validation[:25]

# CoT on GSM8K
python scripts/run_eval.py --agent cot --dataset gsm8k --split test[:25]

# Reflexion on Hotpot
python scripts/run_eval.py --agent reflexion --dataset hotpot --split validation[:25]
\end{lstlisting}

\section{Interpreting Logs}
JSONL logs (if enabled) contain question, gold answer, predicted answer, and trajectory (including critic/revision for Reflexion). Use them for error taxonomy: planning errors, tool misuse, retrieval gaps, arithmetic slips, hallucinated tools.

\section{Experiments and Assignments}
\begin{itemize}[leftmargin=*]
    \item \textbf{Baseline} ReAct vs. CoT on Hotpot/GSM8K.
    \item \textbf{Ablations} Remove search or calculator; quantify drop.
    \item \textbf{Reflexion study} Compare ReAct vs. Reflexion on Hotpot; report EM/tool-call stats.
    \item \textbf{Stability} Sweep temperature and max-steps.
    \item \textbf{Qualitative} Label 20 failure cases across error types.
\end{itemize}

\section{For Students: Extension Task (Must Read $\geq$2 Papers)}
\textbf{Requirement:} Read \emph{at least two} recent (2023--2024) papers and implement a non-trivial improvement inspired by them. Suggested directions:
\begin{itemize}[leftmargin=*]
    \item Tree/Graph-of-Thought planning \cite{yao2024tot,besta2024got} with branching/re-ranking of candidate thoughts.
    \item Structured tool calls with JSON schemas to cut parsing errors (see Toolformer-style ideas \cite{schick2023toolformer}).
    \item Passage reranking (cross-encoder) before feeding search results into ReAct/Reflexion.
    \item Self-consistency or sample-and-rerank over tool-augmented trajectories.
\end{itemize}
Deliverables: implement the enhancement as a new agent or tool module, run head-to-head against ReAct and Reflexion, and report metrics plus qualitative trajectories with citations to the two papers you used.

\section{Checklist}
\begin{itemize}[leftmargin=*]
    \item Understand ReAct vs. CoT prompts and stopping rules.
    \item Run Hotpot and GSM8K sanity checks; inspect trajectories.
    \item Try Reflexion and note any gains or regressions.
    \item Perform one ablation and one qualitative error study.
\end{itemize}

\section*{References}
\begin{thebibliography}{00}
\bibitem{yao2023react} S. Yao et al., ``ReAct: Synergizing reasoning and acting in language models,'' 2023.
\bibitem{wei2022chain} J. Wei et al., ``Chain-of-thought prompting elicits reasoning in large language models,'' 2022.
\bibitem{schick2023toolformer} T. Schick et al., ``Toolformer: Language models can teach themselves to use tools,'' 2023.
\bibitem{gao2023pal} L. Gao et al., ``PAL: Program-aided language models,'' 2023.
\bibitem{kojima2022zeroshot} T. Kojima et al., ``Large language models are zero-shot reasoners,'' 2022.
\bibitem{shinn2023reflexion} M. Shinn et al., ``Reflexion: Language agents with verbal reinforcement learning,'' 2023.
\bibitem{yao2024tot} S. Yao et al., ``Tree of Thoughts: Deliberate problem solving with large language models,'' 2024.
\bibitem{besta2024got} M. Besta et al., ``Graph of Thoughts: Solving elaborate problems with large language models,'' 2024.
\end{thebibliography}

\end{document}
\documentclass[conference]{IEEEtran}
\IEEEoverridecommandlockouts

\usepackage[T1]{fontenc}
\usepackage[utf8]{inputenc}
\usepackage{amsmath,amsfonts}
\usepackage{graphicx}
\usepackage{url}
\usepackage{enumitem}
\usepackage{listings}
\usepackage{xcolor}

\lstdefinestyle{code}{
  basicstyle=\ttfamily\footnotesize,
  breaklines=true,
  frame=single,
  columns=fullflexible,
  backgroundcolor=\color[gray]{0.97},
  keywordstyle=\color{blue},
  commentstyle=\color{gray},
  showstringspaces=false
}

\title{Learning Guide: ReAct + Tool-Use Agents for Multi-hop QA and Math}

\author{\IEEEauthorblockN{Your Name}
\IEEEauthorblockA{Course: DL for NLP --- Assignment A05}}

\begin{document}
\maketitle

\begin{abstract}
This note explains how to build and study a reasoning+acting (ReAct) agent with tool calls (search, calculator, sandboxed Python) and a chain-of-thought (CoT) baseline. It is intended as a teaching companion for students implementing the A05 assignment. We cover concepts, prompts, tools, safety, evaluation, and suggested experiments in a concise IEEE-style format.
\end{abstract}

\begin{IEEEkeywords}
ReAct, tool use, chain-of-thought, HotpotQA, GSM8K, evaluation, safety
\end{IEEEkeywords}

\section{Why ReAct?}
Large language models (LLMs) reason well but often hallucinate facts or struggle with arithmetic. ReAct~\cite{yao2023react} interleaves \emph{thought} (natural language reasoning) with \emph{actions} (tool calls) and \emph{observations} (tool outputs). This yields:
\begin{itemize}[leftmargin=*]
    \item Better factuality via retrieval tools (search).
    \item Reliable math via calculator or code execution (PAL~\cite{gao2023pal}).
    \item Transparent trajectories for debugging and safety analysis.
\end{itemize}

\section{Key Concepts}
\textbf{Chain-of-Thought (CoT)}~\cite{wei2022cot} asks the model to think step-by-step but without external tools.\\
\textbf{ReAct} augments CoT with tool-use steps: \texttt{Thought} $\rightarrow$ \texttt{Action[Input]} $\rightarrow$ \texttt{Observation}, repeated until \texttt{Final Answer}.\\
\textbf{Tools} in this assignment:
\begin{itemize}[leftmargin=*]
    \item \textbf{search}: BM25 over a local HotpotQA-derived corpus.
    \item \textbf{calculator}: safe arithmetic via AST evaluation.
    \item \textbf{python}: sandboxed Python executor (PAL-style).
\end{itemize}

\section{Prompting Templates}
\subsection{ReAct System Prompt}
The system message lists available tools and enforces the loop format:
\begin{lstlisting}[style=code]
Thought: <reasoning>
Action: <tool>[<input>]
Observation: <tool output>
...
Final Answer: <answer>
\end{lstlisting}
Guidelines include: use only provided tools, keep inputs concise, and stop with \texttt{Final Answer}.

\subsection{CoT Prompt}
The baseline prompt asks the model to think step-by-step and end with \texttt{Final Answer: <...>} without calling tools.

\section{Agent Algorithms}
\subsection{ReAct Loop (pseudo-code)}
\begin{lstlisting}[style=code]
messages = [system_prompt(tools), user_question]
for step in 1..max_steps:
    reply = LLM(messages)
    if "Final Answer" in reply:
        return answer, trajectory
    action, arg = parse_action(reply)
    if action not in tools:
        obs = "Tool <action> not available"
    else:
        obs = tools[action](arg)  # guarded execution
    append trajectory with (thought, action, obs)
    messages += [{role: assistant, reply},
                 {role: user, content: "Observation: "+obs}]
return failure ("max steps reached")
\end{lstlisting}

\subsection{CoT Baseline}
Single LLM call with the CoT prompt; extract the span after \texttt{Final Answer:}.

\section{Tool Implementations}
\textbf{Search}: BM25 (rank\_bm25) over a JSONL corpus built from HotpotQA contexts. Inputs are tokenized with a simple regex; outputs are top-$k$ passages.\\
\textbf{Calculator}: Parses arithmetic expressions via Python AST, allowing only safe operators and math functions (e.g., \texttt{+, -, *, /, pow, sqrt, log}). Blocks identifiers outside the allowlist.\\
\textbf{Python Executor}: Runs short Python snippets in a spawned process with a restricted \texttt{\_\_builtins\_\_}, math library, timeouts, and regex-based danger filtering (blocking \texttt{subprocess}, \texttt{os}, \texttt{socket}, etc.).

\section{Safety Guardrails}
\begin{itemize}[leftmargin=*]
    \item Block dangerous code patterns (file/network/process).
    \item Timeouts for execution; terminate if still alive.
    \item Log hallucinated tool calls (tools not in the allowlist).
    \item Keep prompts explicit about allowed tools and stopping.
\end{itemize}

\section{Data \& Metrics}
\textbf{HotpotQA (distractor)} for multi-hop reading. Metrics: EM and F1.\\
\textbf{GSM8K} for grade-school math. Metric: numeric accuracy (with string fallback).\\
Additional tracked stats: average tool calls per question, hallucinated tool calls, trajectory length, runtime.

\section{Running the Code}
\begin{enumerate}[leftmargin=*]
    \item Install deps: \texttt{pip install -r requirements.txt}.
    \item Set \texttt{OPENAI\_API\_KEY} (and optionally \texttt{OPENAI\_BASE\_URL}).
    \item Build Hotpot corpus: \texttt{python scripts/build\_hotpot\_corpus.py --limit 5000}.
    \item Evaluate:
\begin{lstlisting}[style=code]
# ReAct on Hotpot
python scripts/run_eval.py --dataset hotpot --agent react --limit 50
# CoT on GSM8K
python scripts/run_eval.py --dataset gsm8k --agent cot --limit 50
\end{lstlisting}
    \item Inspect outputs in \texttt{runs/<dataset>-<agent>-<timestamp>/}.
\end{enumerate}

\section{Suggested Experiments}
\begin{itemize}[leftmargin=*]
    \item \textbf{ReAct vs CoT}: Compare EM/F1 (Hotpot) and accuracy (GSM8K).
    \item \textbf{Ablations}: remove calculator or search; measure degradation.
    \item \textbf{Temperature}: sweep 0.0--0.7; watch stability vs creativity.
    \item \textbf{Max steps}: 4/6/8 to study planning depth vs latency.
    \item \textbf{Failure study}: manually label 20 errors (bad plan, tool misuse, hallucinated observation).
\end{itemize}

\section{Qualitative Analysis Tips}
\begin{itemize}[leftmargin=*]
    \item Look for circular tool calls (repeating search without new hypotheses).
    \item Check whether calculator is used for simple arithmetic vs offloaded to Python.
    \item Inspect hallucinated tools; tighten prompts if frequent.
    \item For math, ensure final answers are numeric and not full sentences.
\end{itemize}

\section{Limitations}
\begin{itemize}[leftmargin=*]
    \item BM25 over a small corpus may miss evidence; consider larger context windows.
    \item Regex safety is heuristic; not production-grade sandboxing.
    \item OpenAI-compatible API costs and rate limits apply.
\end{itemize}

\section{Files to Know (this repo)}
\begin{itemize}[leftmargin=*]
    \item \texttt{src/agents/react.py}: ReAct loop and parsing.
    \item \texttt{src/agents/cot.py}: CoT baseline.
    \item \texttt{src/tools/}: \texttt{search.py}, \texttt{calculator.py}, \texttt{code\_exec.py}.
    \item \texttt{scripts/run\_eval.py}: CLI for running evaluations.
    \item \texttt{scripts/build\_hotpot\_corpus.py}: builds BM25 corpus from HotpotQA.
    \item \texttt{report\_template.md}: structure for the 4--6 page writeup.
\end{itemize}

\section{Study Checklist}
\begin{itemize}[leftmargin=*]
    \item Understand ReAct vs CoT prompting and stopping criteria.
    \item Be able to trace a trajectory and explain each tool call.
    \item Know how safety guards block dangerous code.
    \item Run at least one Hotpot and one GSM8K eval; read the trajectories.
    \item Perform one ablation and one qualitative failure analysis.
\end{itemize}

\section*{Acknowledgments}
Key inspirations: ReAct~\cite{yao2023react}, Toolformer~\cite{schick2023toolformer}, PAL~\cite{gao2023pal}, and CoT~\cite{wei2022cot}.

\begin{thebibliography}{00}
\bibitem{yao2023react} S. Yao \emph{et al.}, ``ReAct: Synergizing reasoning and acting in language models,'' 2023.
\bibitem{schick2023toolformer} T. Schick \emph{et al.}, ``Toolformer: Language models can teach themselves to use tools,'' 2023.
\bibitem{gao2023pal} L. Gao \emph{et al.}, ``PAL: Program-aided language models,'' 2023.
\bibitem{wei2022cot} J. Wei \emph{et al.}, ``Chain-of-thought prompting elicits reasoning in large language models,'' 2022.
\end{thebibliography}

\end{document}

Xet Storage Details

Size:: 16 kB
Xet hash:: 823946ceb6f11a37b5de68622e18beb07c7ee6a3d7f2dae6827d9c80421ce913

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.

	\documentclass[conference]{IEEEtran}
	\IEEEoverridecommandlockouts
	\usepackage[T1]{fontenc}
	\usepackage[utf8]{inputenc}
	\usepackage{amsmath,amsfonts}
	\usepackage{enumitem}
	\usepackage{listings}
	\usepackage{xcolor}
	\usepackage{hyperref}

	\lstdefinestyle{code}{
	basicstyle=\ttfamily\footnotesize,
	breaklines=true,
	frame=single,
	columns=fullflexible,
	backgroundcolor=\color[gray]{0.97},
	keywordstyle=\color{blue},
	commentstyle=\color{gray},
	showstringspaces=false
	}

	\title{Learning Guide: ReAct + Tool Use Agents (A05)}
	\author{Course Mirror Assignment A05}
	\date{}

	\begin{document}
	\maketitle

	\begin{abstract}
	This guide teaches the ReAct (reasoning+acting) paradigm, contrasts it with chain-of-thought (CoT), and walks through the provided code. It also introduces a Reflexion-style self-critique extension to improve ReAct. Use it as a teaching companion and checklist for students.
	\end{abstract}

	\begin{IEEEkeywords}
	ReAct, tool use, Reflexion, chain-of-thought, HotpotQA, GSM8K
	\end{IEEEkeywords}

	\section{Motivation and Goals}
	\begin{itemize}[leftmargin=*]
	\item Bridge LLM reasoning with external tools (search, calculator, python).
	\item Compare tool-augmented ReAct vs. CoT-only baselines.
	\item Study an improvement from recent work: Reflexion-style self-critique \cite{shinn2023reflexion}.
	\end{itemize}

	\section{Background}
	\subsection{Chain-of-Thought (CoT)}
	Stepwise natural-language reasoning \cite{wei2022chain}; strong but tool-less.

	\subsection{ReAct}
	Interleaves Thought $\rightarrow$ Action $\rightarrow$ Observation until Finish \cite{yao2023react}. Reduces hallucination and handles multi-hop reasoning.

	\subsection{Tools}
	\begin{itemize}[leftmargin=*]
	\item \textbf{Search}: BM25 over HotpotQA distractor paragraphs.
	\item \textbf{Calculator}: Safe AST arithmetic.
	\item \textbf{Python}: Sandboxed execution (PAL-style \cite{gao2023pal}); return value from \texttt{result}.
	\end{itemize}

	\section{Code Map}
	\begin{itemize}[leftmargin=*]
	\item \texttt{src/agents/react.py}: ReAct loop and parsing.
	\item \texttt{src/agents/cot.py}: CoT baseline.
	\item \texttt{src/agents/reflexion.py}: ReAct + critic + revision (Reflexion-style).
	\item \texttt{src/tools/}: \texttt{search.py}, \texttt{calculator.py}, \texttt{python\_tool.py}.
	\item \texttt{src/eval.py}: evaluation harness and logging.
	\item \texttt{scripts/build\_index.py}: build BM25 corpus.
	\item \texttt{scripts/run\_eval.py}: CLI for react/cot/reflexion.
	\end{itemize}

	\section{Prompting Essentials}
	\subsection{ReAct}
	Specify available tools and enforce the loop:
	\begin{lstlisting}[style=code]
	Question: ...
	Thought: ...
	Action: search["query"]
	Observation: ...
	...
	Finish: <answer>
	\end{lstlisting}

	\subsection{CoT}
	Ask for step-by-step reasoning and end with \texttt{Final answer: <...>}.

	\subsection{Reflexion Critique}
	After a ReAct pass, a critic summarizes likely mistakes in 2--3 bullets; a revision pass rewrites with \texttt{Final answer: ...}.

	\section{Agent Variants}
	\subsection{ReAct}
	Iterative tool calls; stops on \texttt{Finish}. Configurable max steps.

	\subsection{CoT}
	Single call; no tools.

	\subsection{Reflexion Extension (New)}
	1) Run ReAct. 2) Critic summarizes issues. 3) Revision produces a final answer using the critic (no extra tool calls for speed). Inspired by \cite{shinn2023reflexion}. Use to test if self-critique helps Hotpot multi-hop.

	\section{Tools and Safety}
	\begin{itemize}[leftmargin=*]
	\item Calculator/Python use restricted AST or limited builtins; Python has timeouts.
	\item Unknown tool names return explicit errors to avoid silent failure.
	\item Keep tool inputs short; avoid untrusted code.
	\end{itemize}

	\section{Data and Metrics}
	\begin{itemize}[leftmargin=*]
	\item HotpotQA (distractor): metric = Exact Match (EM) after normalization.
	\item GSM8K: metric = accuracy on parsed numeric answers.
	\item Track avg tool calls, hallucinated tool names, runtime.
	\end{itemize}

	\section{Running}
	\subsection{Setup}
	\begin{lstlisting}[style=code]
	pip install -r requirements.txt
	export OPENAI_API_KEY=...
	\end{lstlisting}

	\subsection{Build Search Index}
	\begin{lstlisting}[style=code]
	python scripts/build_index.py --split train[:2000] --out data/hotpot_index.pkl
	\end{lstlisting}

	\subsection{Evaluate}
	\begin{lstlisting}[style=code]
	# ReAct on Hotpot
	python scripts/run_eval.py --agent react --dataset hotpot --split validation[:25]

	# CoT on GSM8K
	python scripts/run_eval.py --agent cot --dataset gsm8k --split test[:25]

	# Reflexion on Hotpot
	python scripts/run_eval.py --agent reflexion --dataset hotpot --split validation[:25]
	\end{lstlisting}

	\section{Interpreting Logs}
	JSONL logs (if enabled) contain question, gold answer, predicted answer, and trajectory (including critic/revision for Reflexion). Use them for error taxonomy: planning errors, tool misuse, retrieval gaps, arithmetic slips, hallucinated tools.

	\section{Experiments and Assignments}
	\begin{itemize}[leftmargin=*]
	\item \textbf{Baseline} ReAct vs. CoT on Hotpot/GSM8K.
	\item \textbf{Ablations} Remove search or calculator; quantify drop.
	\item \textbf{Reflexion study} Compare ReAct vs. Reflexion on Hotpot; report EM/tool-call stats.
	\item \textbf{Stability} Sweep temperature and max-steps.
	\item \textbf{Qualitative} Label 20 failure cases across error types.
	\end{itemize}

	\section{For Students: Extension Task (Must Read $\geq$2 Papers)}
	\textbf{Requirement:} Read \emph{at least two} recent (2023--2024) papers and implement a non-trivial improvement inspired by them. Suggested directions:
	\begin{itemize}[leftmargin=*]
	\item Tree/Graph-of-Thought planning \cite{yao2024tot,besta2024got} with branching/re-ranking of candidate thoughts.
	\item Structured tool calls with JSON schemas to cut parsing errors (see Toolformer-style ideas \cite{schick2023toolformer}).
	\item Passage reranking (cross-encoder) before feeding search results into ReAct/Reflexion.
	\item Self-consistency or sample-and-rerank over tool-augmented trajectories.
	\end{itemize}
	Deliverables: implement the enhancement as a new agent or tool module, run head-to-head against ReAct and Reflexion, and report metrics plus qualitative trajectories with citations to the two papers you used.

	\section{Checklist}
	\begin{itemize}[leftmargin=*]
	\item Understand ReAct vs. CoT prompts and stopping rules.
	\item Run Hotpot and GSM8K sanity checks; inspect trajectories.
	\item Try Reflexion and note any gains or regressions.
	\item Perform one ablation and one qualitative error study.
	\end{itemize}

	\section*{References}
	\begin{thebibliography}{00}
	\bibitem{yao2023react} S. Yao et al., ``ReAct: Synergizing reasoning and acting in language models,'' 2023.
	\bibitem{wei2022chain} J. Wei et al., ``Chain-of-thought prompting elicits reasoning in large language models,'' 2022.
	\bibitem{schick2023toolformer} T. Schick et al., ``Toolformer: Language models can teach themselves to use tools,'' 2023.
	\bibitem{gao2023pal} L. Gao et al., ``PAL: Program-aided language models,'' 2023.
	\bibitem{kojima2022zeroshot} T. Kojima et al., ``Large language models are zero-shot reasoners,'' 2022.
	\bibitem{shinn2023reflexion} M. Shinn et al., ``Reflexion: Language agents with verbal reinforcement learning,'' 2023.
	\bibitem{yao2024tot} S. Yao et al., ``Tree of Thoughts: Deliberate problem solving with large language models,'' 2024.
	\bibitem{besta2024got} M. Besta et al., ``Graph of Thoughts: Solving elaborate problems with large language models,'' 2024.
	\end{thebibliography}

	\end{document}
	\documentclass[conference]{IEEEtran}
	\IEEEoverridecommandlockouts

	\usepackage[T1]{fontenc}
	\usepackage[utf8]{inputenc}
	\usepackage{amsmath,amsfonts}
	\usepackage{graphicx}
	\usepackage{url}
	\usepackage{enumitem}
	\usepackage{listings}
	\usepackage{xcolor}

	\lstdefinestyle{code}{
	basicstyle=\ttfamily\footnotesize,
	breaklines=true,
	frame=single,
	columns=fullflexible,
	backgroundcolor=\color[gray]{0.97},
	keywordstyle=\color{blue},
	commentstyle=\color{gray},
	showstringspaces=false
	}

	\title{Learning Guide: ReAct + Tool-Use Agents for Multi-hop QA and Math}

	\author{\IEEEauthorblockN{Your Name}
	\IEEEauthorblockA{Course: DL for NLP --- Assignment A05}}

	\begin{document}
	\maketitle

	\begin{abstract}
	This note explains how to build and study a reasoning+acting (ReAct) agent with tool calls (search, calculator, sandboxed Python) and a chain-of-thought (CoT) baseline. It is intended as a teaching companion for students implementing the A05 assignment. We cover concepts, prompts, tools, safety, evaluation, and suggested experiments in a concise IEEE-style format.
	\end{abstract}

	\begin{IEEEkeywords}
	ReAct, tool use, chain-of-thought, HotpotQA, GSM8K, evaluation, safety
	\end{IEEEkeywords}

	\section{Why ReAct?}
	Large language models (LLMs) reason well but often hallucinate facts or struggle with arithmetic. ReAct~\cite{yao2023react} interleaves \emph{thought} (natural language reasoning) with \emph{actions} (tool calls) and \emph{observations} (tool outputs). This yields:
	\begin{itemize}[leftmargin=*]
	\item Better factuality via retrieval tools (search).
	\item Reliable math via calculator or code execution (PAL~\cite{gao2023pal}).
	\item Transparent trajectories for debugging and safety analysis.
	\end{itemize}

	\section{Key Concepts}
	\textbf{Chain-of-Thought (CoT)}~\cite{wei2022cot} asks the model to think step-by-step but without external tools.\\
	\textbf{ReAct} augments CoT with tool-use steps: \texttt{Thought} $\rightarrow$ \texttt{Action[Input]} $\rightarrow$ \texttt{Observation}, repeated until \texttt{Final Answer}.\\
	\textbf{Tools} in this assignment:
	\begin{itemize}[leftmargin=*]
	\item \textbf{search}: BM25 over a local HotpotQA-derived corpus.
	\item \textbf{calculator}: safe arithmetic via AST evaluation.
	\item \textbf{python}: sandboxed Python executor (PAL-style).
	\end{itemize}

	\section{Prompting Templates}
	\subsection{ReAct System Prompt}
	The system message lists available tools and enforces the loop format:
	\begin{lstlisting}[style=code]
	Thought: <reasoning>
	Action: <tool>[<input>]
	Observation: <tool output>
	...
	Final Answer: <answer>
	\end{lstlisting}
	Guidelines include: use only provided tools, keep inputs concise, and stop with \texttt{Final Answer}.

	\subsection{CoT Prompt}
	The baseline prompt asks the model to think step-by-step and end with \texttt{Final Answer: <...>} without calling tools.

	\section{Agent Algorithms}
	\subsection{ReAct Loop (pseudo-code)}
	\begin{lstlisting}[style=code]
	messages = [system_prompt(tools), user_question]
	for step in 1..max_steps:
	reply = LLM(messages)
	if "Final Answer" in reply:
	return answer, trajectory
	action, arg = parse_action(reply)
	if action not in tools:
	obs = "Tool <action> not available"
	else:
	obs = tools[action](arg) # guarded execution
	append trajectory with (thought, action, obs)
	messages += [{role: assistant, reply},
	{role: user, content: "Observation: "+obs}]
	return failure ("max steps reached")
	\end{lstlisting}

	\subsection{CoT Baseline}
	Single LLM call with the CoT prompt; extract the span after \texttt{Final Answer:}.

	\section{Tool Implementations}
	\textbf{Search}: BM25 (rank\_bm25) over a JSONL corpus built from HotpotQA contexts. Inputs are tokenized with a simple regex; outputs are top-$k$ passages.\\
	\textbf{Calculator}: Parses arithmetic expressions via Python AST, allowing only safe operators and math functions (e.g., \texttt{+, -, *, /, pow, sqrt, log}). Blocks identifiers outside the allowlist.\\
	\textbf{Python Executor}: Runs short Python snippets in a spawned process with a restricted \texttt{\_\_builtins\_\_}, math library, timeouts, and regex-based danger filtering (blocking \texttt{subprocess}, \texttt{os}, \texttt{socket}, etc.).

	\section{Safety Guardrails}
	\begin{itemize}[leftmargin=*]
	\item Block dangerous code patterns (file/network/process).
	\item Timeouts for execution; terminate if still alive.
	\item Log hallucinated tool calls (tools not in the allowlist).
	\item Keep prompts explicit about allowed tools and stopping.
	\end{itemize}

	\section{Data \& Metrics}
	\textbf{HotpotQA (distractor)} for multi-hop reading. Metrics: EM and F1.\\
	\textbf{GSM8K} for grade-school math. Metric: numeric accuracy (with string fallback).\\
	Additional tracked stats: average tool calls per question, hallucinated tool calls, trajectory length, runtime.

	\section{Running the Code}
	\begin{enumerate}[leftmargin=*]
	\item Install deps: \texttt{pip install -r requirements.txt}.
	\item Set \texttt{OPENAI\_API\_KEY} (and optionally \texttt{OPENAI\_BASE\_URL}).
	\item Build Hotpot corpus: \texttt{python scripts/build\_hotpot\_corpus.py --limit 5000}.
	\item Evaluate:
	\begin{lstlisting}[style=code]
	# ReAct on Hotpot
	python scripts/run_eval.py --dataset hotpot --agent react --limit 50
	# CoT on GSM8K
	python scripts/run_eval.py --dataset gsm8k --agent cot --limit 50
	\end{lstlisting}
	\item Inspect outputs in \texttt{runs/<dataset>-<agent>-<timestamp>/}.
	\end{enumerate}

	\section{Suggested Experiments}
	\begin{itemize}[leftmargin=*]
	\item \textbf{ReAct vs CoT}: Compare EM/F1 (Hotpot) and accuracy (GSM8K).
	\item \textbf{Ablations}: remove calculator or search; measure degradation.
	\item \textbf{Temperature}: sweep 0.0--0.7; watch stability vs creativity.
	\item \textbf{Max steps}: 4/6/8 to study planning depth vs latency.
	\item \textbf{Failure study}: manually label 20 errors (bad plan, tool misuse, hallucinated observation).
	\end{itemize}

	\section{Qualitative Analysis Tips}
	\begin{itemize}[leftmargin=*]
	\item Look for circular tool calls (repeating search without new hypotheses).
	\item Check whether calculator is used for simple arithmetic vs offloaded to Python.
	\item Inspect hallucinated tools; tighten prompts if frequent.
	\item For math, ensure final answers are numeric and not full sentences.
	\end{itemize}

	\section{Limitations}
	\begin{itemize}[leftmargin=*]
	\item BM25 over a small corpus may miss evidence; consider larger context windows.
	\item Regex safety is heuristic; not production-grade sandboxing.
	\item OpenAI-compatible API costs and rate limits apply.
	\end{itemize}

	\section{Files to Know (this repo)}
	\begin{itemize}[leftmargin=*]
	\item \texttt{src/agents/react.py}: ReAct loop and parsing.
	\item \texttt{src/agents/cot.py}: CoT baseline.
	\item \texttt{src/tools/}: \texttt{search.py}, \texttt{calculator.py}, \texttt{code\_exec.py}.
	\item \texttt{scripts/run\_eval.py}: CLI for running evaluations.
	\item \texttt{scripts/build\_hotpot\_corpus.py}: builds BM25 corpus from HotpotQA.
	\item \texttt{report\_template.md}: structure for the 4--6 page writeup.
	\end{itemize}

	\section{Study Checklist}
	\begin{itemize}[leftmargin=*]
	\item Understand ReAct vs CoT prompting and stopping criteria.
	\item Be able to trace a trajectory and explain each tool call.
	\item Know how safety guards block dangerous code.
	\item Run at least one Hotpot and one GSM8K eval; read the trajectories.
	\item Perform one ablation and one qualitative failure analysis.
	\end{itemize}

	\section*{Acknowledgments}
	Key inspirations: ReAct~\cite{yao2023react}, Toolformer~\cite{schick2023toolformer}, PAL~\cite{gao2023pal}, and CoT~\cite{wei2022cot}.

	\begin{thebibliography}{00}
	\bibitem{yao2023react} S. Yao \emph{et al.}, ``ReAct: Synergizing reasoning and acting in language models,'' 2023.
	\bibitem{schick2023toolformer} T. Schick \emph{et al.}, ``Toolformer: Language models can teach themselves to use tools,'' 2023.
	\bibitem{gao2023pal} L. Gao \emph{et al.}, ``PAL: Program-aided language models,'' 2023.
	\bibitem{wei2022cot} J. Wei \emph{et al.}, ``Chain-of-thought prompting elicits reasoning in large language models,'' 2022.
	\end{thebibliography}

	\end{document}