-  a mapping of criterion names to descriptions
                -  a sequence of criterion names
                -  a single criterion name present in one of the default criteria
                -  a sequence of `ConstitutionalPrinciple` instances
                -  a single `ConstitutionalPrinciple` instance
        Returns
        -------
        Dict[str, str]
            A dictionary mapping criterion names to descriptions.
        Examples
        --------
        >>> criteria = ["relevance", "coherence"]
        >>> CriteriaEvalChain.resolve_criteria(criteria)
        {'relevance': 'Is the submission referring to a real quote from the text?',
         'coherence': 'Is the submission coherent, well-structured, and organized?'}
        """  # noqa: E501
        if isinstance(criteria, str):
            criteria_ = {criteria: _SUPPORTED_CRITERIA[criteria]}
        elif isinstance(criteria, ConstitutionalPrinciple):
            criteria_ = {criteria.name: criteria.critique_request}
        elif isinstance(criteria, Sequence):
            criteria_ = {}
            for criterion in criteria:
                if isinstance(criterion, str):
                    criteria_[criterion] = _SUPPORTED_CRITERIA[criterion]
                elif isinstance(criterion, ConstitutionalPrinciple):
                    criteria_[criterion.name] = criterion.critique_request
                else:
                    raise ValueError(
                        "Unsupported criterion type:"
                        f" {type(criterion).__name__}, {criterion}"
                    )
        else:
            criteria_ = dict(criteria)
        return criteria_
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,
        criteria: CRITERIA_TYPE,
        *,
        prompt: Optional[BasePromptTemplate] = None,
        requires_reference: bool = False,


requires_reference: bool = False,
        **kwargs: Any,
    ) -> CriteriaEvalChain:
        """Create a `CriteriaEvalChain` instance from an llm and criteria.
        Parameters
        ----------
        llm : BaseLanguageModel
            The language model to use for evaluation.
        criteria : CRITERIA_TYPE
            The criteria to evaluate the runs against. It can be:
                -  a mapping of criterion names to descriptions
                -  a sequence of criterion names
                -  a single criterion name present in one of the default criteria
                -  a sequence of `ConstitutionalPrinciple` instances
                -  a single `ConstitutionalPrinciple` instance
        prompt : Optional[BasePromptTemplate], default=None
            The prompt template to use for generating prompts. If not provided,
            a default prompt template will be used based on the value of
            `requires_reference`.
        requires_reference : bool, default=False
            Whether the evaluation requires a reference text. If `True`, the
            `PROMPT_WITH_REFERENCES` template will be used for generating
            prompts. If `False`, the `PROMPT` template will be used.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain`
            constructor.
        Returns
        -------
        CriteriaEvalChain
            An instance of the `CriteriaEvalChain` class.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import CriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = {
                "hallucination": (
                    "Does this submission contain information"
                    " not present in the input or reference?"
                ),
            }
        >>> chain = CriteriaEvalChain.from_llm(


),
            }
        >>> chain = CriteriaEvalChain.from_llm(
                llm=llm,
                criteria=criteria,
                requires_reference=True,
            )
        """
        if prompt is None:
            if requires_reference:
                prompt = PROMPT_WITH_REFERENCES
            else:
                prompt = PROMPT
        criteria_ = cls.resolve_criteria(criteria)
        criteria_str = " ".join(f"{k}: {v}" for k, v in criteria_.items())
        prompt_ = prompt.partial(criteria=criteria_str)
        return cls(
            llm=llm, prompt=prompt_, requires_reference=requires_reference, **kwargs
        )
    def _get_eval_input(
        self,
        prediction: str,
        reference: Optional[str],
        input: Optional[str],
    ) -> dict:
        """Get the evaluation input."""
        input_ = {
            "input": input,
            "output": prediction,
        }
        if self.requires_reference:
            input_["reference"] = reference
        return input_
[docs]    def evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        **kwargs: Any,
    ) -> dict:
        """Evaluate a prediction against the criteria.
        Parameters
        ----------
        prediction : str
            The predicted text to evaluate.
        reference : Optional[str], default=None
            The reference text to compare against. This is required if
            `requires_reference` is `True`.
        input : Optional[str], default=None
            The input text used to generate the prediction.
        **kwargs : Any


The input text used to generate the prediction.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain` `__call__`
            method.
        Returns
        -------
        dict
            The evaluation results.
        Examples
        --------
        >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import CriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = "conciseness"
        >>> chain = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
        >>> chain.evaluate_strings(
                prediction="The answer is 42.",
                reference="42",
                input="What is the answer to life, the universe, and everything?",
            )
        """
        input_ = self._get_eval_input(prediction, reference, input)
        return self(input_, **kwargs)["text"]
[docs]    async def aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously evaluate a prediction against the criteria.
        Parameters
        ----------
        prediction : str
            The predicted text to evaluate.
        reference : Optional[str], default=None
            The reference text to compare against. This is required if
            `requires_reference` is `True`.
        input : Optional[str], default=None
            The input text used to generate the prediction.
        **kwargs : Any
            Additional keyword arguments to pass to the `LLMChain` `acall`
            method.
        Returns
        -------
        dict
            The evaluation results.
        Examples
        --------
         >>> from langchain.llms import OpenAI


Examples
        --------
         >>> from langchain.llms import OpenAI
        >>> from langchain.evaluation.criteria import CriteriaEvalChain
        >>> llm = OpenAI()
        >>> criteria = "conciseness"
        >>> chain = CriteriaEvalChain.from_llm(llm=llm, criteria=criteria)
        >>> await chain.aevaluate_strings(
                prediction="The answer is 42.",
                reference="42",
                input="What is the answer to life, the universe, and everything?",
            )
        """
        input_ = self._get_eval_input(prediction, reference, input)
        result = await self.acall(input_, **kwargs)
        return result["text"]


Source code for langchain.evaluation.agents.trajectory_eval_chain
"""A chain for evaluating ReAct style agents.
This chain is used to evaluate ReAct style agents by reasoning about
the sequence of actions taken and their outcomes. It uses a language model
chain (LLMChain) to generate the reasoning and scores.
"""
from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, Union
from pydantic import Field
from langchain.callbacks.manager import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
    Callbacks,
)
from langchain.chains.base import Chain
from langchain.chains.llm import LLMChain
from langchain.chat_models.base import BaseChatModel
from langchain.evaluation.agents.trajectory_eval_prompt import (
    EVAL_CHAT_PROMPT,
    TOOL_FREE_EVAL_CHAT_PROMPT,
)
from langchain.schema import AgentAction, BaseOutputParser, OutputParserException
from langchain.tools.base import BaseTool
[docs]class TrajectoryEval(NamedTuple):
    score: int
    reasoning: str
[docs]class TrajectoryOutputParser(BaseOutputParser):
    @property
    def _type(self) -> str:
        return "agent_trajectory"
[docs]    def parse(self, text: str) -> TrajectoryEval:
        """Parse the output text and extract the score and reasoning.
        Args:
            text (str): The output text to parse.
        Returns:
            TrajectoryEval: A named tuple containing the score and reasoning.
        Raises:
            OutputParserException: If the score is not found in the output text or
                if the score is not a digit in the range 1-5.
        """
        if "Score:" not in text:
            raise OutputParserException(


"""
        if "Score:" not in text:
            raise OutputParserException(
                f"Could not find score in model eval output: {text}"
            )
        reasoning, score_str = text.split("Score: ")
        reasoning, score_str = reasoning.strip(), score_str.strip()
        score_str = next(
            (char for char in score_str if char.isdigit()), "0"
        )  # Scan for first digit
        if not 1 <= int(score_str) <= 5:
            raise OutputParserException(
                f"Score is not a digit in the range 1-5: {text}"
            )
        return TrajectoryEval(score=int(score_str), reasoning=reasoning)
[docs]class TrajectoryEvalChain(Chain):
    """A chain for evaluating ReAct style agents.
    This chain is used to evaluate ReAct style agents by reasoning about
    the sequence of actions taken and their outcomes.
    Example:
        .. code-block:: python
            from langchain.agents import AgentType, initialize_agent
            from langchain.chat_models import ChatOpenAI
            from langchain.evaluation import TrajectoryEvalChain
            from langchain.tools import tool
            @tool
            def geography_answers(country: str, question: str) -> str:
                \"\"\"Very helpful answers to geography questions.\"\"\"
                return f"{country}? IDK - We may never know {question}."
            llm = ChatOpenAI(model="gpt-3.5-turbo-0613", temperature=0)
            agent = initialize_agent(
                tools=[geography_answers],
                llm=llm,
                agent=AgentType.OPENAI_FUNCTIONS,
                return_intermediate_steps=True,
            )


return_intermediate_steps=True,
            )
            question = "How many dwell in the largest minor region in Argentina?"
            response = agent(question)
            eval_chain = TrajectoryEvalChain.from_llm(
                llm=llm, agent_tools=[geography_answers], return_reasoning=True
            )
            result = eval_chain.evaluate_agent_trajectory(
                input=question,
                agent_trajectory=response["intermediate_steps"],
                prediction=response["output"],
                reference="Paris",
            )
            print(result["score"])
            # 0
    """  # noqa: E501
    agent_tools: Optional[List[BaseTool]] = None
    """A list of tools available to the agent."""
    eval_chain: LLMChain
    """The language model chain used for evaluation."""
    output_parser: TrajectoryOutputParser = Field(
        default_factory=TrajectoryOutputParser
    )
    """The output parser used to parse the output."""
    return_reasoning: bool = False
    """Whether to return the reasoning along with the score."""
    @property
    def _tools_description(self) -> str:
        """Get the description of the agent tools.
        Returns:
            str: The description of the agent tools.
        """
        if self.agent_tools is None:
            return ""
        return "\n\n".join(
            [
                f"""Tool {i}: {tool.name}
Description: {tool.description}"""
                for i, tool in enumerate(self.agent_tools, 1)
            ]
        )
[docs]    @staticmethod
    def get_agent_trajectory(steps: Union[str, List[Tuple[AgentAction, str]]]) -> str:
        """Get the agent trajectory as a formatted string.
        Args:


"""Get the agent trajectory as a formatted string.
        Args:
            steps (Union[str, List[Tuple[AgentAction, str]]]): The agent trajectory.
        Returns:
            str: The formatted agent trajectory.
        """
        if isinstance(steps, str):
            return steps
        return "\n\n".join(
            [
                f"""Step {i}:
Tool used: {action.tool}
Tool input: {action.tool_input}
Tool output: {output}"""
                for i, (action, output) in enumerate(steps, 1)
            ]
        )
    @staticmethod
    def _format_reference(reference: Optional[str]) -> str:
        """Format the reference text.
        Args:
            reference (str): The reference text.
        Returns:
            str: The formatted reference text.
        """
        if not reference:
            return ""
        return f"""
The following is the expected answer. Use this to measure correctness:
[GROUND_TRUTH]
{reference}
[END_GROUND_TRUTH]
"""
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseChatModel,
        agent_tools: Optional[Sequence[BaseTool]] = None,
        output_parser: Optional[TrajectoryOutputParser] = None,
        return_reasoning: bool = False,
    ) -> "TrajectoryEvalChain":
        """Create a TrajectoryEvalChain object from a language model chain.
        Args:
            llm (BaseChatModel): The language model chain.
            agent_tools (Optional[Sequence[BaseTool]]): A list of tools
                available tothe agent.
            output_parser (Optional[TrajectoryOutputParser]): The output parser
                used to parse the chain output into a score.


used to parse the chain output into a score.
            return_reasoning (bool): Whether to return the
                reasoning along with the score.
        Returns:
            TrajectoryEvalChain: The TrajectoryEvalChain object.
        """
        if agent_tools:
            prompt = EVAL_CHAT_PROMPT
        else:
            prompt = TOOL_FREE_EVAL_CHAT_PROMPT
        eval_chain = LLMChain(llm=llm, prompt=prompt)
        return cls(
            agent_tools=agent_tools,
            return_reasoning=return_reasoning,
            eval_chain=eval_chain,
            output_parser=output_parser or TrajectoryOutputParser(),
        )
    @property
    def input_keys(self) -> List[str]:
        """Get the input keys for the chain.
        Returns:
            List[str]: The input keys.
        """
        return ["question", "agent_trajectory", "answer", "reference"]
    @property
    def output_keys(self) -> List[str]:
        """Get the output keys for the chain.
        Returns:
            List[str]: The output keys.
        """
        if self.return_reasoning:
            return ["score", "reasoning"]
        return ["score"]
[docs]    def __call__(
        self,
        inputs: Union[Dict[str, Any], Any],
        return_only_outputs: bool = False,
        callbacks: Callbacks = None,
        *,
        tags: Optional[List[str]] = None,
        include_run_info: bool = False,
    ) -> Dict[str, Any]:
        """Run the logic of this chain and add to output if desired.
        Args:
            inputs: Dictionary of inputs, or single input if chain expects
                only one param.


inputs: Dictionary of inputs, or single input if chain expects
                only one param.
            return_only_outputs: boolean for whether to return only outputs in the
                response. If True, only new keys generated by this chain will be
                returned. If False, both input keys and new keys generated by this
                chain will be returned. Defaults to False.
            callbacks: Callbacks to use for this chain run. If not provided, will
                use the callbacks provided to the chain.
            include_run_info: Whether to include run info in the response. Defaults
                to False.
        """
        if "reference" not in inputs:
            inputs["reference"] = ""
        return super().__call__(
            inputs=inputs,
            return_only_outputs=return_only_outputs,
            callbacks=callbacks,
            tags=tags,
            include_run_info=include_run_info,
        )
    def _call(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Run the chain and generate the output.
        Args:
            inputs (Dict[str, str]): The input values for the chain.
            run_manager (Optional[CallbackManagerForChainRun]): The callback
                manager for the chain run.
        Returns:
            Dict[str, Any]: The output values of the chain.
        """
        chain_input = {**inputs}
        if self.agent_tools:
            chain_input["tool_descriptions"] = self._tools_description
        raw_output = self.eval_chain.run(chain_input)
        parsed_output = self.output_parser.parse(raw_output)
        if self.return_reasoning:
            return {"score": parsed_output.score, "reasoning": parsed_output.reasoning}


return {"score": parsed_output.score, "reasoning": parsed_output.reasoning}
        return {"score": parsed_output.score}
    async def _acall(
        self,
        inputs: Dict[str, str],
        run_manager: Optional[AsyncCallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Run the chain and generate the output.
        Args:
            inputs (Dict[str, str]): The input values for the chain.
            run_manager (Optional[CallbackManagerForChainRun]): The callback
                manager for the chain run.
        Returns:
            Dict[str, Any]: The output values of the chain.
        """
        chain_input = {**inputs}
        if self.agent_tools:
            chain_input["tool_descriptions"] = self._tools_description
        raw_output = await self.eval_chain.arun(chain_input)
        parsed_output = self.output_parser.parse(raw_output)
        if self.return_reasoning:
            return {"score": parsed_output.score, "reasoning": parsed_output.reasoning}
        return {"score": parsed_output.score}
[docs]    def evaluate_agent_trajectory(
        self,
        *,
        prediction: str,
        input: str,
        agent_trajectory: Union[str, List[Tuple[AgentAction, str]]],
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        **kwargs: Any,
    ) -> dict:
        """Evaluate a trajectory.
        Args:
            input (str): The input question.
            agent_trajectory (Union[str, List[Tuple[AgentAction, str]]]):
                The intermediate steps forming the agent trajectory.
            prediction (str): The expected prediction.
            reference (Optional[str]): The reference answer.
        Returns:


reference (Optional[str]): The reference answer.
        Returns:
            dict: The evaluation result.
        """
        inputs = {
            "question": input,
            "agent_trajectory": self.get_agent_trajectory(agent_trajectory),
            "answer": prediction,
            "reference": self._format_reference(reference),
        }
        return self(inputs=inputs, callbacks=callbacks, **kwargs)
[docs]    async def aevaluate_agent_trajectory(
        self,
        *,
        prediction: str,
        input: str,
        agent_trajectory: Union[str, List[Tuple[AgentAction, str]]],
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously evaluate a trajectory.
        Args:
            input (str): The input question.
            agent_trajectory (Union[str, List[Tuple[AgentAction, str]]]):
                The intermediate steps forming the agent trajectory.
            prediction (str): The expected prediction.
            reference (Optional[str]): The reference answer.
        Returns:
            dict: The evaluation result.
        """
        inputs = {
            "question": input,
            "agent_trajectory": self.get_agent_trajectory(agent_trajectory),
            "answer": prediction,
            "reference": self._format_reference(reference),
        }
        return await self.acall(
            inputs=inputs,
            callbacks=callbacks,
            **kwargs,
        )


Source code for langchain.evaluation.qa.generate_chain
"""LLM Chain specifically for generating examples for question answering."""
from __future__ import annotations
from typing import Any
from langchain.base_language import BaseLanguageModel
from langchain.chains.llm import LLMChain
from langchain.evaluation.qa.generate_prompt import PROMPT
[docs]class QAGenerateChain(LLMChain):
    """LLM Chain specifically for generating examples for question answering."""
[docs]    @classmethod
    def from_llm(cls, llm: BaseLanguageModel, **kwargs: Any) -> QAGenerateChain:
        """Load QA Generate Chain from LLM."""
        return cls(llm=llm, prompt=PROMPT, **kwargs)


Source code for langchain.evaluation.qa.eval_chain
"""LLM Chain specifically for evaluating question answering."""
from __future__ import annotations
from typing import Any, List, Optional, Sequence
from langchain import PromptTemplate
from langchain.base_language import BaseLanguageModel
from langchain.callbacks.manager import Callbacks
from langchain.chains.llm import LLMChain
from langchain.evaluation.qa.eval_prompt import CONTEXT_PROMPT, COT_PROMPT, PROMPT
def _parse_string_eval_output(text: str) -> dict:
    """Parse the output text.
    Args:
        text (str): The output text to parse.
    Returns:
        Any: The parsed output.
    """
    reasoning, verdict = text.strip().rsplit("\n", maxsplit=1)
    score = (
        1
        if verdict.upper() == "CORRECT"
        else (0 if verdict.upper() == "INCORRECT" else None)
    )
    return {
        "reasoning": reasoning.strip(),
        "value": verdict,
        "score": score,
    }
[docs]class QAEvalChain(LLMChain):
    """LLM Chain specifically for evaluating question answering."""
[docs]    @classmethod
    def from_llm(
        cls, llm: BaseLanguageModel, prompt: PromptTemplate = PROMPT, **kwargs: Any
    ) -> QAEvalChain:
        """Load QA Eval Chain from LLM.
        Args:
            llm (BaseLanguageModel): the base language model to use.
            prompt (PromptTemplate): A prompt template containing the input_variables:
            'input', 'answer' and 'result' that will be used as the prompt
            for evaluation.
            Defaults to PROMPT.
            **kwargs: additional keyword arguments.


Defaults to PROMPT.
            **kwargs: additional keyword arguments.
        Returns:
            QAEvalChain: the loaded QA eval chain.
        """
        expected_input_vars = {"query", "answer", "result"}
        if expected_input_vars != set(prompt.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt.input_variables}"
            )
        return cls(llm=llm, prompt=prompt, **kwargs)
[docs]    def evaluate(
        self,
        examples: Sequence[dict],
        predictions: Sequence[dict],
        question_key: str = "query",
        answer_key: str = "answer",
        prediction_key: str = "result",
        *,
        callbacks: Callbacks = None,
    ) -> List[dict]:
        """Evaluate question answering examples and predictions."""
        inputs = [
            {
                "query": example[question_key],
                "answer": example[answer_key],
                "result": predictions[i][prediction_key],
            }
            for i, example in enumerate(examples)
        ]
        return self.apply(inputs, callbacks=callbacks)
[docs]    def evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        **kwargs: Any,
    ) -> dict:
        """Evaluate Chain or LLM output, based on optional input and label.
        Args:
            prediction (str): the LLM or chain prediction to evaluate.
            reference (Optional[str], optional): the reference label
                to evaluate against.


reference (Optional[str], optional): the reference label
                to evaluate against.
            input (Optional[str], optional): the input to consider during evaluation
            callbacks (Callbacks, optional): the callbacks to use for tracing.
            **kwargs: additional keyword arguments, including callbacks, tags, etc.
        Returns:
            dict: The evaluation results containing the score or value.
        """
        result = self.evaluate(
            examples=[{"query": input, "answer": reference}],
            predictions=[{"result": prediction}],
            callbacks=callbacks,
        )[0]
        return _parse_string_eval_output(result["text"])
[docs]    async def aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        callbacks: Callbacks = None,
        **kwargs: Any,
    ) -> dict:
        result = await self.acall(
            inputs={"query": input, "answer": reference, "result": prediction},
            callbacks=callbacks,
        )
        return _parse_string_eval_output(result["text"])
[docs]class ContextQAEvalChain(LLMChain):
    """LLM Chain specifically for evaluating QA w/o GT based on context"""
    @classmethod
    def _validate_input_vars(cls, prompt: PromptTemplate) -> None:
        expected_input_vars = {"query", "context", "result"}
        if expected_input_vars != set(prompt.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt.input_variables}"
            )
[docs]    @classmethod
    def from_llm(
        cls,
        llm: BaseLanguageModel,


def from_llm(
        cls,
        llm: BaseLanguageModel,
        prompt: PromptTemplate = CONTEXT_PROMPT,
        **kwargs: Any,
    ) -> ContextQAEvalChain:
        """Load QA Eval Chain from LLM.
        Args:
            llm (BaseLanguageModel): the base language model to use.
            prompt (PromptTemplate): A prompt template containing the input_variables:
            'query', 'context' and 'result' that will be used as the prompt
            for evaluation.
            Defaults to PROMPT.
            **kwargs: additional keyword arguments.
        Returns:
            ContextQAEvalChain: the loaded QA eval chain.
        """
        cls._validate_input_vars(prompt)
        return cls(llm=llm, prompt=prompt, **kwargs)
[docs]    def evaluate(
        self,
        examples: List[dict],
        predictions: List[dict],
        question_key: str = "query",
        context_key: str = "context",
        prediction_key: str = "result",
        *,
        callbacks: Callbacks = None,
    ) -> List[dict]:
        """Evaluate question answering examples and predictions."""
        inputs = [
            {
                "query": example[question_key],
                "context": example[context_key],
                "result": predictions[i][prediction_key],
            }
            for i, example in enumerate(examples)
        ]
        return self.apply(inputs, callbacks=callbacks)
[docs]    def evaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        **kwargs: Any,
    ) -> dict:
        result = self.evaluate(


**kwargs: Any,
    ) -> dict:
        result = self.evaluate(
            examples=[{"query": input, "context": reference}],
            predictions=[{"result": prediction}],
            callbacks=kwargs.get("callbacks"),
        )[0]
        return _parse_string_eval_output(result["text"])
[docs]    async def aevaluate_strings(
        self,
        *,
        prediction: str,
        reference: Optional[str] = None,
        input: Optional[str] = None,
        **kwargs: Any,
    ) -> dict:
        result = await self.acall(
            inputs={"query": input, "context": reference, "result": prediction},
            callbacks=kwargs.get("callbacks"),
        )
        return _parse_string_eval_output(result["text"])
[docs]class CotQAEvalChain(ContextQAEvalChain):
    """LLM Chain specifically for evaluating QA using chain of thought reasoning."""
[docs]    @classmethod
    def from_llm(
        cls, llm: BaseLanguageModel, prompt: PromptTemplate = COT_PROMPT, **kwargs: Any
    ) -> CotQAEvalChain:
        cls._validate_input_vars(prompt)
        return cls(llm=llm, prompt=prompt, **kwargs)


Source code for langchain.evaluation.comparison.eval_chain
"""Base classes for comparing the output of two models."""
from __future__ import annotations
from typing import Any, Optional
from pydantic import Field
from langchain.base_language import BaseLanguageModel
from langchain.callbacks.manager import Callbacks
from langchain.chains.llm import LLMChain
from langchain.evaluation.comparison.prompt import PROMPT, PROMPT_WITH_REFERENCE
from langchain.prompts.prompt import PromptTemplate
from langchain.schema import BaseOutputParser
[docs]class PairwiseStringResultOutputParser(BaseOutputParser[dict]):
    """A parser for the output of the PairwiseStringEvalChain."""
    @property
    def _type(self) -> str:
        return "pairwise_string_result"
[docs]    def parse(self, text: str) -> Any:
        """Parse the output text.
        Args:
            text (str): The output text to parse.
        Returns:
            Any: The parsed output.
        """
        reasoning, verdict = text.strip().rsplit("\n", maxsplit=1)
        verdict = verdict.strip("[").strip("]")
        if verdict not in {"A", "B", "C"}:
            raise ValueError(
                f"Invalid verdict: {verdict}. "
                "Verdict must be one of 'A', 'B', or 'C'."
            )
        # C means the models are tied. Return 'None' meaning no preference
        verdict_ = None if verdict == "C" else verdict
        score = {
            "A": 1,
            "B": 0,
            None: 0.5,
        }.get(verdict_)
        return {
            "reasoning": reasoning,
            "value": verdict_,


return {
            "reasoning": reasoning,
            "value": verdict_,
            "score": score,
        }
[docs]class PairwiseStringEvalChain(LLMChain):
    """A chain for comparing the output of two models.
    Example:
    >>> from langchain.chat_models import ChatOpenAI
    >>> from langchain.evaluation.comparison import PairwiseStringEvalChain
    >>> llm = ChatOpenAI(temperature=0)
    >>> chain = PairwiseStringEvalChain.from_llm(llm=llm)
    >>> result = chain.evaluate_string_pairs(
    ...     input = "What is the chemical formula for water?",
    ...     prediction = "H2O",
    ...     prediction_b = (
    ...        "The chemical formula for water is H2O, which means"
    ...        " there are two hydrogen atoms and one oxygen atom."
    ...     referenc = "The chemical formula for water is H2O.",
    ... )
    >>> print(result["text"])
    # {
    #    "value": "B",
    #    "comment": "Both responses accurately state"
    #       " that the chemical formula for water is H2O."
    #       " However, Response B provides additional information"
    # .     " by explaining what the formula means.\n[[B]]"
    # }
    """
    output_parser: BaseOutputParser = Field(
        default_factory=PairwiseStringResultOutputParser
    )
[docs]    @classmethod
    def from_llm(
        cls,
        *,
        llm: BaseLanguageModel,
        prompt: Optional[PromptTemplate] = None,
        require_reference: bool = False,
        **kwargs: Any,


require_reference: bool = False,
        **kwargs: Any,
    ) -> PairwiseStringEvalChain:
        """Initialize the PairwiseStringEvalChain from an LLM.
        Args:
            llm (BaseLanguageModel): The LLM to use.
            prompt (PromptTemplate, optional): The prompt to use.
            require_reference (bool, optional): Whether to require a reference
                string. Defaults to False.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            PairwiseStringEvalChain: The initialized PairwiseStringEvalChain.
        """
        expected_input_vars = {"prediction", "prediction_b", "input"}
        if prompt is None:
            if require_reference:
                expected_input_vars.add("reference")
                prompt_ = PROMPT_WITH_REFERENCE
            else:
                prompt_ = PROMPT
        else:
            if require_reference:
                expected_input_vars.add("reference")
            prompt_ = prompt
        if expected_input_vars != set(prompt_.input_variables):
            raise ValueError(
                f"Input variables should be {expected_input_vars}, "
                f"but got {prompt_.input_variables}"
            )
        return cls(llm=llm, prompt=prompt_, **kwargs)
    def _prepare_input(
        self, prediction: str, prediction_b: str, input: str, reference: Optional[str]
    ) -> dict:
        input_ = {
            "prediction": prediction,
            "prediction_b": prediction_b,
            "input": input,
        }
        if reference is not None and "reference" in self.prompt.input_variables:
            input_["reference"] = reference
        return input_
[docs]    def evaluate_string_pairs(
        self,
        *,
        prediction: str,


self,
        *,
        prediction: str,
        prediction_b: str,
        input: str,
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        **kwargs: Any,
    ) -> dict:
        """Evaluate whether output A is preferred to output B.
        Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            input (str): The input or task string.
            callbacks (Callbacks, optional): The callbacks to use.
            reference (str, optional): The reference string, if any.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - reasoning: The reasoning for the preference.
                - value: The preference value, which is either 'A', 'B', or None
                    for no preference.
                - score: The preference score, which is 1 for 'A', 0 for 'B',
                    and 0.5 for None.
        """
        input_ = self._prepare_input(prediction, prediction_b, input, reference)
        result = self(
            inputs=input_,
            callbacks=callbacks,
            **kwargs,
        )
        return result["text"]
[docs]    async def aevaluate_string_pairs(
        self,
        *,
        prediction: str,
        prediction_b: str,
        input: str,
        reference: Optional[str] = None,
        callbacks: Callbacks = None,
        **kwargs: Any,
    ) -> dict:
        """Asynchronously evaluate whether output A is preferred to output B.
        Args:
            prediction (str): The output string from the first model.


Args:
            prediction (str): The output string from the first model.
            prediction_b (str): The output string from the second model.
            input (str): The input or task string.
            callbacks (Callbacks, optional): The callbacks to use.
            reference (str, optional): The reference string, if any.
            **kwargs (Any): Additional keyword arguments.
        Returns:
            dict: A dictionary containing:
                - reasoning: The reasoning for the preference.
                - value: The preference value, which is either 'A', 'B', or None
                    for no preference.
                - score: The preference score, which is 1 for 'A', 0 for 'B',
                    and 0.5 for None.
        """
        input_ = self._prepare_input(prediction, prediction_b, input, reference)
        result = await self.acall(
            inputs=input_,
            callbacks=callbacks,
            **kwargs,
        )
        return result["text"]


Source code for langchain.evaluation.run_evaluators.implementations
from typing import Any, Dict, List, Mapping, Optional, Sequence, Union
from langchainplus_sdk.evaluation import EvaluationResult
from langchainplus_sdk.schemas import Example, Run, RunTypeEnum
from pydantic import BaseModel, Field
from langchain.base_language import BaseLanguageModel
from langchain.chains.llm import LLMChain
from langchain.chat_models.base import BaseChatModel
from langchain.evaluation.agents.trajectory_eval_prompt import (
    EVAL_CHAT_PROMPT as TRAJECTORY_PROMPT,
)
from langchain.evaluation.criteria.eval_chain import (
    CriteriaEvalChain,
    CriteriaResultOutputParser,
)
from langchain.evaluation.criteria.prompt import PROMPT as CRITERIA_PROMPT
from langchain.evaluation.qa.eval_chain import QAEvalChain
from langchain.evaluation.qa.eval_prompt import PROMPT as QA_DEFAULT_PROMPT
from langchain.evaluation.qa.eval_prompt import SQL_PROMPT
from langchain.evaluation.run_evaluators.base import (
    RunEvaluatorChain,
    RunEvaluatorInputMapper,
    RunEvaluatorOutputParser,
)
from langchain.prompts.base import BasePromptTemplate
from langchain.prompts.prompt import PromptTemplate
from langchain.schema import OutputParserException
from langchain.tools.base import BaseTool
_QA_PROMPTS = {
    "qa": QA_DEFAULT_PROMPT,
    "sql": SQL_PROMPT,
}
[docs]class StringRunEvaluatorInputMapper(RunEvaluatorInputMapper, BaseModel):
    """Maps the Run and Optional[Example] to a dictionary."""
    prediction_map: Dict[str, str]
    """Map from run outputs to the evaluation inputs."""
    input_map: Dict[str, str]
    """Map from run inputs to the evaluation inputs."""


"""Map from run inputs to the evaluation inputs."""
    answer_map: Optional[Dict[str, str]] = None
    """Map from example outputs to the evaluation inputs."""
[docs]    def map(self, run: Run, example: Optional[Example] = None) -> Dict[str, Any]:
        """Maps the Run and Optional[Example] to a dictionary"""
        if run.outputs is None and self.prediction_map:
            raise ValueError(f"Run {run.id} has no outputs.")
        if self.answer_map and (not example or not example.outputs):
            raise ValueError("This evaluator requires references, but none were given.")
        outputs = run.outputs or {}
        data = {value: outputs[key] for key, value in self.prediction_map.items()}
        data.update({value: run.inputs[key] for key, value in self.input_map.items()})
        if self.answer_map and example and example.outputs:
            data.update(
                {value: example.outputs[key] for key, value in self.answer_map.items()}
            )
        return data
[docs]class ChoicesOutputParser(RunEvaluatorOutputParser):
    """Parse a feedback run with optional choices."""
    evaluation_name: str
    choices_map: Optional[Dict[str, int]] = None
    @property
    def _type(self) -> str:
        return "choices_run_eval"
[docs]    def parse(self, text: str) -> EvaluationResult:
        """Parse the last line of the text and return an evaluation result."""
        lines = text.strip().split()
        value = lines[-1].strip()
        score = self.choices_map.get(value) if self.choices_map else None
        comment = " ".join(lines[:-1]) if len(lines) > 1 else None
        return EvaluationResult(


return EvaluationResult(
            key=self.evaluation_name,
            score=score,
            value=value,
            comment=comment,
        )
[docs]def get_qa_evaluator(
    llm: BaseLanguageModel,
    *,
    prompt: Union[PromptTemplate, str] = QA_DEFAULT_PROMPT,
    input_key: str = "input",
    prediction_key: str = "output",
    answer_key: str = "output",
    evaluation_name: Optional[str] = None,
    **kwargs: Any,
) -> RunEvaluatorChain:
    """Get an eval chain that compares response against ground truth."""
    if isinstance(prompt, str):
        prompt = _QA_PROMPTS[prompt]
    eval_chain = QAEvalChain.from_llm(llm=llm, prompt=prompt, **kwargs)
    input_mapper = kwargs.pop(
        "input_mapper",
        StringRunEvaluatorInputMapper(
            input_map={input_key: "query"},
            prediction_map={prediction_key: "result"},
            answer_map={answer_key: "answer"},
        ),
    )
    evaluation_name = evaluation_name or "Correctness"
    output_parser = kwargs.pop(
        "output_parser",
        ChoicesOutputParser(
            evaluation_name=evaluation_name,
            choices_map={"CORRECT": 1, "INCORRECT": 0},
        ),
    )
    tags = kwargs.pop("tags", [])
    return RunEvaluatorChain(
        eval_chain=eval_chain,
        input_mapper=input_mapper,
        output_parser=output_parser,
        tags=tags + [evaluation_name],
        **kwargs,
    )
[docs]class CriteriaOutputParser(RunEvaluatorOutputParser):
    """Parse a criteria results into an evaluation result."""


"""Parse a criteria results into an evaluation result."""
    evaluation_name: str
    @property
    def _type(self) -> str:
        return "criteria"
[docs]    def parse(self, parsed_output: Union[str, dict]) -> EvaluationResult:
        """Parse the last line of the text and return an evaluation result."""
        if isinstance(parsed_output, str):
            parsed_output_ = CriteriaResultOutputParser().parse(parsed_output)
        else:
            parsed_output_ = parsed_output
        return EvaluationResult(
            key=self.evaluation_name,
            score=parsed_output_["score"],
            value=parsed_output_["value"],
            comment=parsed_output_["reasoning"],
        )
[docs]def get_criteria_evaluator(
    llm: BaseLanguageModel,
    criteria: Union[Mapping[str, str], Sequence[str], str],
    *,
    input_key: str = "input",
    prediction_key: str = "output",
    prompt: BasePromptTemplate = CRITERIA_PROMPT,
    evaluation_name: Optional[str] = None,
    **kwargs: Any,
) -> RunEvaluatorChain:
    """Get an eval chain for grading a model's response against a map of criteria."""
    input_mapper = kwargs.pop(
        "input_mapper",
        StringRunEvaluatorInputMapper(
            input_map={input_key: "input"},
            prediction_map={prediction_key: "output"},
        ),
    )
    criteria_ = CriteriaEvalChain.resolve_criteria(criteria)
    evaluation_name = evaluation_name or " ".join(criteria_.keys())
    parser = kwargs.pop(
        "output_parser",
        CriteriaOutputParser(
            choices_map={"Y": 1, "N": 0}, evaluation_name=evaluation_name
        ),
    )


),
    )
    tags = kwargs.pop("tags", [])
    eval_chain = CriteriaEvalChain.from_llm(
        llm=llm, criteria=criteria_, prompt=prompt, **kwargs
    )
    return RunEvaluatorChain(
        eval_chain=eval_chain,
        input_mapper=input_mapper,
        output_parser=parser,
        tags=tags + [evaluation_name],
        **kwargs,
    )
[docs]class TrajectoryEvalOutputParser(RunEvaluatorOutputParser):
    evaluation_name: str = "Agent Trajectory"
    """The name assigned to the evaluation feedback."""
    evaluator_info: dict = Field(default_factory=dict)
    """Additional information to log as feedback metadata."""
    @property
    def _type(self) -> str:
        return "agent_trajectory_run_eval"
[docs]    def parse(self, text: str) -> EvaluationResult:
        if "Score:" not in text:
            raise OutputParserException(
                f"Could not find score in model eval output: {text}"
            )
        reasoning, score_str = text.split("Score: ")
        reasoning, score_str = reasoning.strip(), score_str.strip()
        score_str = next(
            (char for char in score_str if char.isdigit()), "0"
        )  # Scan for first digit
        if not 1 <= int(score_str) <= 5:
            raise OutputParserException(
                f"Score is not a digit in the range 1-5: {text}"
            )
        return EvaluationResult(
            key=self.evaluation_name,
            score=int(score_str),
            comment=reasoning,
            evaluator_info=self.evaluator_info,
        )
[docs]class TrajectoryInputMapper(RunEvaluatorInputMapper, BaseModel):


)
[docs]class TrajectoryInputMapper(RunEvaluatorInputMapper, BaseModel):
    """Maps the Run and Optional[Example] to a dictionary."""
    tool_descriptions: List[str]
    """The descriptions for each of the tools available to the agent."""
    agent_input_key: str = "input"
    """The key to load from the agent executor's run input dictionary."""
    agent_output_key: str = "output"
    """The key to load from the agent executor's run output dictionary."""
    tool_input_key: str = "input"
    """The key to load from the tool executor's run input dictionary."""
    tool_output_key: str = "output"
    """The key to load from the tool executor's run output dictionary."""
[docs]    def map(self, run: Run, example: Optional[Example] = None) -> Dict[str, str]:
        """Maps the Run and Optional[Example] to a dictionary"""
        if run.child_runs is None:
            raise ValueError("Run must have child runs to be evaluated.")
        if run.outputs is None:
            raise ValueError("Run must have outputs to be evaluated.")
        question = run.inputs[self.agent_input_key]
        tool_runs = [
            run_ for run_ in run.child_runs if run_.run_type == RunTypeEnum.tool
        ]
        agent_steps = []
        for i, run_ in enumerate(tool_runs, 1):
            tool_output = (
                f"Tool output: {run_.outputs.get(self.tool_output_key, run_.outputs)}"
                if run_.outputs
                else (f"Tool error: {run_.error}" if run_.error else "No output")
            )
            agent_steps.append(
                f"""Step {i}:
Tool used: {run_.name}


f"""Step {i}:
Tool used: {run_.name}
Tool input: {run_.inputs.get(self.tool_input_key, run_.inputs)}
Tool output: {tool_output}"""
            )
        return {
            "tool_descriptions": "\n\n".join(self.tool_descriptions),
            "question": question,
            "agent_trajectory": "\n\n".join(agent_steps),
            "answer": run.outputs[self.agent_output_key],
        }
[docs]def get_trajectory_evaluator(
    llm: BaseChatModel,
    agent_tools: Union[Sequence[str], Sequence[BaseTool]],
    *,
    input_key: str = "input",
    prediction_key: str = "output",
    tool_input_key: str = "input",
    tool_output_key: str = "output",
    prompt: BasePromptTemplate = TRAJECTORY_PROMPT,
    evaluation_name: str = "Agent Trajectory",
    **kwargs: Any,
) -> RunEvaluatorChain:
    """Get an eval chain for grading a model's response against a map of criteria."""
    tool_descriptions = [
        f"Tool {i}: {tool.name}\nDescription: {tool.description}"
        if isinstance(tool, BaseTool)
        else f"Tool {i}: {tool}"
        for i, tool in enumerate(agent_tools, 1)
    ]
    input_mapper = kwargs.pop(
        "input_mapper",
        TrajectoryInputMapper(
            agent_input_key=input_key,
            agent_output_key=prediction_key,
            tool_input_key=tool_input_key,
            tool_output_key=tool_output_key,
            tool_descriptions=tool_descriptions,
        ),
    )
    parser = kwargs.pop(
        "output_parser",


),
    )
    parser = kwargs.pop(
        "output_parser",
        TrajectoryEvalOutputParser(evaluation_name=evaluation_name),
    )
    eval_chain = LLMChain(llm=llm, prompt=prompt, **kwargs)
    tags = kwargs.pop("tags", [])
    return RunEvaluatorChain(
        eval_chain=eval_chain,
        input_mapper=input_mapper,
        output_parser=parser,
        tags=tags + [evaluation_name],
        **kwargs,
    )


Source code for langchain.evaluation.run_evaluators.base
from __future__ import annotations
from abc import abstractmethod
from typing import Any, Dict, List, Optional
from langchainplus_sdk import EvaluationResult, RunEvaluator
from langchainplus_sdk.schemas import Example, Run
from langchain.callbacks.manager import (
    AsyncCallbackManagerForChainRun,
    CallbackManagerForChainRun,
)
from langchain.chains.base import Chain
from langchain.schema import RUN_KEY, BaseOutputParser
class RunEvaluatorInputMapper:
    """Map the inputs of a run to the inputs of an evaluation."""
    @abstractmethod
    def map(self, run: Run, example: Optional[Example] = None) -> Dict[str, Any]:
        """Maps the Run and Optional[Example] to a dictionary"""
[docs]class RunEvaluatorOutputParser(BaseOutputParser[EvaluationResult]):
    """Parse the output of a run."""
    eval_chain_output_key: str = "text"
[docs]    def parse_chain_output(self, output: Dict[str, Any]) -> EvaluationResult:
        """Parse the output of a run."""
        text = output[self.eval_chain_output_key]
        return self.parse(text)
[docs]class RunEvaluatorChain(Chain, RunEvaluator):
    """Evaluate Run and optional examples."""
    input_mapper: RunEvaluatorInputMapper
    """Maps the Run and Optional example to a dictionary for the eval chain."""
    eval_chain: Chain
    """The evaluation chain."""
    output_parser: RunEvaluatorOutputParser
    """Parse the output of the eval chain into feedback."""
    @property
    def input_keys(self) -> List[str]:
        return ["run", "example"]
    @property
    def output_keys(self) -> List[str]:
        return ["feedback"]
    def _call(


return ["feedback"]
    def _call(
        self,
        inputs: Dict[str, Any],
        run_manager: Optional[CallbackManagerForChainRun] = None,
    ) -> Dict[str, Any]:
        """Call the evaluation chain."""
        run: Run = inputs["run"]
        example: Optional[Example] = inputs.get("example")
        chain_input = self.input_mapper.map(run, example)
        _run_manager = run_manager or CallbackManagerForChainRun.get_noop_manager()
        callbacks = _run_manager.get_child()
        chain_output = self.eval_chain(
            chain_input, callbacks=callbacks, include_run_info=True
        )
        run_info = chain_output[RUN_KEY]
        feedback = self.output_parser.parse_chain_output(chain_output)
        feedback.evaluator_info[RUN_KEY] = run_info
        return {"feedback": feedback}
    async def _acall(
        self,
        inputs: Dict[str, Any],
        run_manager: AsyncCallbackManagerForChainRun | None = None,
    ) -> Dict[str, Any]:
        run: Run = inputs["run"]
        example: Optional[Example] = inputs.get("example")
        chain_input = self.input_mapper.map(run, example)
        _run_manager = run_manager or AsyncCallbackManagerForChainRun.get_noop_manager()
        callbacks = _run_manager.get_child()
        chain_output = await self.eval_chain.acall(
            chain_input,
            callbacks=callbacks,
            include_run_info=True,
        )
        run_info = chain_output[RUN_KEY]
        feedback = self.output_parser.parse_chain_output(chain_output)
        feedback.evaluator_info[RUN_KEY] = run_info
        return {"feedback": feedback}
[docs]    def evaluate_run(


return {"feedback": feedback}
[docs]    def evaluate_run(
        self, run: Run, example: Optional[Example] = None
    ) -> EvaluationResult:
        """Evaluate an example."""
        return self({"run": run, "example": example})["feedback"]
[docs]    async def aevaluate_run(
        self, run: Run, example: Optional[Example] = None
    ) -> EvaluationResult:
        """Evaluate an example."""
        result = await self.acall({"run": run, "example": example})
        return result["feedback"]


Source code for langchain.callbacks.streaming_aiter
from __future__ import annotations
import asyncio
from typing import Any, AsyncIterator, Dict, List, Literal, Union, cast
from langchain.callbacks.base import AsyncCallbackHandler
from langchain.schema import LLMResult
# TODO If used by two LLM runs in parallel this won't work as expected
[docs]class AsyncIteratorCallbackHandler(AsyncCallbackHandler):
    """Callback handler that returns an async iterator."""
    queue: asyncio.Queue[str]
    done: asyncio.Event
    @property
    def always_verbose(self) -> bool:
        return True
    def __init__(self) -> None:
        self.queue = asyncio.Queue()
        self.done = asyncio.Event()
[docs]    async def on_llm_start(
        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
    ) -> None:
        # If two calls are made in a row, this resets the state
        self.done.clear()
[docs]    async def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
        self.queue.put_nowait(token)
[docs]    async def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
        self.done.set()
[docs]    async def on_llm_error(
        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
    ) -> None:
        self.done.set()
    # TODO implement the other methods
[docs]    async def aiter(self) -> AsyncIterator[str]:
        while not self.queue.empty() or not self.done.is_set():
            # Wait for the next token in the queue,
            # but stop waiting if the done event is set
            done, other = await asyncio.wait(


done, other = await asyncio.wait(
                [
                    # NOTE: If you add other tasks here, update the code below,
                    # which assumes each set has exactly one task each
                    asyncio.ensure_future(self.queue.get()),
                    asyncio.ensure_future(self.done.wait()),
                ],
                return_when=asyncio.FIRST_COMPLETED,
            )
            # Cancel the other task
            if other:
                other.pop().cancel()
            # Extract the value of the first completed task
            token_or_done = cast(Union[str, Literal[True]], done.pop().result())
            # If the extracted value is the boolean True, the done event was set
            if token_or_done is True:
                break
            # Otherwise, the extracted value is a token, which we yield
            yield token_or_done


Source code for langchain.callbacks.utils
import hashlib
from pathlib import Path
from typing import Any, Dict, Iterable, Tuple, Union
[docs]def import_spacy() -> Any:
    """Import the spacy python package and raise an error if it is not installed."""
    try:
        import spacy
    except ImportError:
        raise ImportError(
            "This callback manager requires the `spacy` python "
            "package installed. Please install it with `pip install spacy`"
        )
    return spacy
[docs]def import_pandas() -> Any:
    """Import the pandas python package and raise an error if it is not installed."""
    try:
        import pandas
    except ImportError:
        raise ImportError(
            "This callback manager requires the `pandas` python "
            "package installed. Please install it with `pip install pandas`"
        )
    return pandas
[docs]def import_textstat() -> Any:
    """Import the textstat python package and raise an error if it is not installed."""
    try:
        import textstat
    except ImportError:
        raise ImportError(
            "This callback manager requires the `textstat` python "
            "package installed. Please install it with `pip install textstat`"
        )
    return textstat
def _flatten_dict(
    nested_dict: Dict[str, Any], parent_key: str = "", sep: str = "_"
) -> Iterable[Tuple[str, Any]]:
    """
    Generator that yields flattened items from a nested dictionary for a flat dict.
    Parameters:
        nested_dict (dict): The nested dictionary to flatten.
        parent_key (str): The prefix to prepend to the keys of the flattened dict.


parent_key (str): The prefix to prepend to the keys of the flattened dict.
        sep (str): The separator to use between the parent key and the key of the
            flattened dictionary.
    Yields:
        (str, any): A key-value pair from the flattened dictionary.
    """
    for key, value in nested_dict.items():
        new_key = parent_key + sep + key if parent_key else key
        if isinstance(value, dict):
            yield from _flatten_dict(value, new_key, sep)
        else:
            yield new_key, value
[docs]def flatten_dict(
    nested_dict: Dict[str, Any], parent_key: str = "", sep: str = "_"
) -> Dict[str, Any]:
    """Flattens a nested dictionary into a flat dictionary.
    Parameters:
        nested_dict (dict): The nested dictionary to flatten.
        parent_key (str): The prefix to prepend to the keys of the flattened dict.
        sep (str): The separator to use between the parent key and the key of the
            flattened dictionary.
    Returns:
        (dict): A flat dictionary.
    """
    flat_dict = {k: v for k, v in _flatten_dict(nested_dict, parent_key, sep)}
    return flat_dict
[docs]def hash_string(s: str) -> str:
    """Hash a string using sha1.
    Parameters:
        s (str): The string to hash.
    Returns:
        (str): The hashed string.
    """
    return hashlib.sha1(s.encode("utf-8")).hexdigest()
[docs]def load_json(json_path: Union[str, Path]) -> str:
    """Load json file to a string.
    Parameters:


"""Load json file to a string.
    Parameters:
        json_path (str): The path to the json file.
    Returns:
        (str): The string representation of the json file.
    """
    with open(json_path, "r") as f:
        data = f.read()
    return data
class BaseMetadataCallbackHandler:
    """This class handles the metadata and associated function states for callbacks.
    Attributes:
        step (int): The current step.
        starts (int): The number of times the start method has been called.
        ends (int): The number of times the end method has been called.
        errors (int): The number of times the error method has been called.
        text_ctr (int): The number of times the text method has been called.
        ignore_llm_ (bool): Whether to ignore llm callbacks.
        ignore_chain_ (bool): Whether to ignore chain callbacks.
        ignore_agent_ (bool): Whether to ignore agent callbacks.
        ignore_retriever_ (bool): Whether to ignore retriever callbacks.
        always_verbose_ (bool): Whether to always be verbose.
        chain_starts (int): The number of times the chain start method has been called.
        chain_ends (int): The number of times the chain end method has been called.
        llm_starts (int): The number of times the llm start method has been called.
        llm_ends (int): The number of times the llm end method has been called.
        llm_streams (int): The number of times the text method has been called.
        tool_starts (int): The number of times the tool start method has been called.
        tool_ends (int): The number of times the tool end method has been called.


tool_ends (int): The number of times the tool end method has been called.
        agent_ends (int): The number of times the agent end method has been called.
        on_llm_start_records (list): A list of records of the on_llm_start method.
        on_llm_token_records (list): A list of records of the on_llm_token method.
        on_llm_end_records (list): A list of records of the on_llm_end method.
        on_chain_start_records (list): A list of records of the on_chain_start method.
        on_chain_end_records (list): A list of records of the on_chain_end method.
        on_tool_start_records (list): A list of records of the on_tool_start method.
        on_tool_end_records (list): A list of records of the on_tool_end method.
        on_agent_finish_records (list): A list of records of the on_agent_end method.
    """
    def __init__(self) -> None:
        self.step = 0
        self.starts = 0
        self.ends = 0
        self.errors = 0
        self.text_ctr = 0
        self.ignore_llm_ = False
        self.ignore_chain_ = False
        self.ignore_agent_ = False
        self.ignore_retriever_ = False
        self.always_verbose_ = False
        self.chain_starts = 0
        self.chain_ends = 0
        self.llm_starts = 0
        self.llm_ends = 0
        self.llm_streams = 0
        self.tool_starts = 0
        self.tool_ends = 0
        self.agent_ends = 0
        self.on_llm_start_records: list = []
        self.on_llm_token_records: list = []


self.on_llm_token_records: list = []
        self.on_llm_end_records: list = []
        self.on_chain_start_records: list = []
        self.on_chain_end_records: list = []
        self.on_tool_start_records: list = []
        self.on_tool_end_records: list = []
        self.on_text_records: list = []
        self.on_agent_finish_records: list = []
        self.on_agent_action_records: list = []
    @property
    def always_verbose(self) -> bool:
        """Whether to call verbose callbacks even if verbose is False."""
        return self.always_verbose_
    @property
    def ignore_llm(self) -> bool:
        """Whether to ignore LLM callbacks."""
        return self.ignore_llm_
    @property
    def ignore_chain(self) -> bool:
        """Whether to ignore chain callbacks."""
        return self.ignore_chain_
    @property
    def ignore_agent(self) -> bool:
        """Whether to ignore agent callbacks."""
        return self.ignore_agent_
    def get_custom_callback_meta(self) -> Dict[str, Any]:
        return {
            "step": self.step,
            "starts": self.starts,
            "ends": self.ends,
            "errors": self.errors,
            "text_ctr": self.text_ctr,
            "chain_starts": self.chain_starts,
            "chain_ends": self.chain_ends,
            "llm_starts": self.llm_starts,
            "llm_ends": self.llm_ends,
            "llm_streams": self.llm_streams,
            "tool_starts": self.tool_starts,
            "tool_ends": self.tool_ends,
            "agent_ends": self.agent_ends,
        }
    def reset_callback_meta(self) -> None:
        """Reset the callback metadata."""


def reset_callback_meta(self) -> None:
        """Reset the callback metadata."""
        self.step = 0
        self.starts = 0
        self.ends = 0
        self.errors = 0
        self.text_ctr = 0
        self.ignore_llm_ = False
        self.ignore_chain_ = False
        self.ignore_agent_ = False
        self.always_verbose_ = False
        self.chain_starts = 0
        self.chain_ends = 0
        self.llm_starts = 0
        self.llm_ends = 0
        self.llm_streams = 0
        self.tool_starts = 0
        self.tool_ends = 0
        self.agent_ends = 0
        self.on_llm_start_records = []
        self.on_llm_token_records = []
        self.on_llm_end_records = []
        self.on_chain_start_records = []
        self.on_chain_end_records = []
        self.on_tool_start_records = []
        self.on_tool_end_records = []
        self.on_text_records = []
        self.on_agent_finish_records = []
        self.on_agent_action_records = []
        return None


Source code for langchain.callbacks.aim_callback
from copy import deepcopy
from typing import Any, Dict, List, Optional, Union
from langchain.callbacks.base import BaseCallbackHandler
from langchain.schema import AgentAction, AgentFinish, LLMResult
[docs]def import_aim() -> Any:
    """Import the aim python package and raise an error if it is not installed."""
    try:
        import aim
    except ImportError:
        raise ImportError(
            "To use the Aim callback manager you need to have the"
            " `aim` python package installed."
            "Please install it with `pip install aim`"
        )
    return aim
class BaseMetadataCallbackHandler:
    """This class handles the metadata and associated function states for callbacks.
    Attributes:
        step (int): The current step.
        starts (int): The number of times the start method has been called.
        ends (int): The number of times the end method has been called.
        errors (int): The number of times the error method has been called.
        text_ctr (int): The number of times the text method has been called.
        ignore_llm_ (bool): Whether to ignore llm callbacks.
        ignore_chain_ (bool): Whether to ignore chain callbacks.
        ignore_agent_ (bool): Whether to ignore agent callbacks.
        ignore_retriever_ (bool): Whether to ignore retriever callbacks.
        always_verbose_ (bool): Whether to always be verbose.
        chain_starts (int): The number of times the chain start method has been called.
        chain_ends (int): The number of times the chain end method has been called.
        llm_starts (int): The number of times the llm start method has been called.


llm_ends (int): The number of times the llm end method has been called.
        llm_streams (int): The number of times the text method has been called.
        tool_starts (int): The number of times the tool start method has been called.
        tool_ends (int): The number of times the tool end method has been called.
        agent_ends (int): The number of times the agent end method has been called.
    """
    def __init__(self) -> None:
        self.step = 0
        self.starts = 0
        self.ends = 0
        self.errors = 0
        self.text_ctr = 0
        self.ignore_llm_ = False
        self.ignore_chain_ = False
        self.ignore_agent_ = False
        self.ignore_retriever_ = False
        self.always_verbose_ = False
        self.chain_starts = 0
        self.chain_ends = 0
        self.llm_starts = 0
        self.llm_ends = 0
        self.llm_streams = 0
        self.tool_starts = 0
        self.tool_ends = 0
        self.agent_ends = 0
    @property
    def always_verbose(self) -> bool:
        """Whether to call verbose callbacks even if verbose is False."""
        return self.always_verbose_
    @property
    def ignore_llm(self) -> bool:
        """Whether to ignore LLM callbacks."""
        return self.ignore_llm_
    @property
    def ignore_chain(self) -> bool:
        """Whether to ignore chain callbacks."""
        return self.ignore_chain_
    @property
    def ignore_agent(self) -> bool:
        """Whether to ignore agent callbacks."""
        return self.ignore_agent_
    @property


"""Whether to ignore agent callbacks."""
        return self.ignore_agent_
    @property
    def ignore_retriever(self) -> bool:
        """Whether to ignore retriever callbacks."""
        return self.ignore_retriever_
    def get_custom_callback_meta(self) -> Dict[str, Any]:
        return {
            "step": self.step,
            "starts": self.starts,
            "ends": self.ends,
            "errors": self.errors,
            "text_ctr": self.text_ctr,
            "chain_starts": self.chain_starts,
            "chain_ends": self.chain_ends,
            "llm_starts": self.llm_starts,
            "llm_ends": self.llm_ends,
            "llm_streams": self.llm_streams,
            "tool_starts": self.tool_starts,
            "tool_ends": self.tool_ends,
            "agent_ends": self.agent_ends,
        }
    def reset_callback_meta(self) -> None:
        """Reset the callback metadata."""
        self.step = 0
        self.starts = 0
        self.ends = 0
        self.errors = 0
        self.text_ctr = 0
        self.ignore_llm_ = False
        self.ignore_chain_ = False
        self.ignore_agent_ = False
        self.always_verbose_ = False
        self.chain_starts = 0
        self.chain_ends = 0
        self.llm_starts = 0
        self.llm_ends = 0
        self.llm_streams = 0
        self.tool_starts = 0
        self.tool_ends = 0
        self.agent_ends = 0
        return None
[docs]class AimCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
    """Callback Handler that logs to Aim.
    Parameters:


"""Callback Handler that logs to Aim.
    Parameters:
        repo (:obj:`str`, optional): Aim repository path or Repo object to which
            Run object is bound. If skipped, default Repo is used.
        experiment_name (:obj:`str`, optional): Sets Run's `experiment` property.
            'default' if not specified. Can be used later to query runs/sequences.
        system_tracking_interval (:obj:`int`, optional): Sets the tracking interval
            in seconds for system usage metrics (CPU, Memory, etc.). Set to `None`
             to disable system metrics tracking.
        log_system_params (:obj:`bool`, optional): Enable/Disable logging of system
            params such as installed packages, git info, environment variables, etc.
    This handler will utilize the associated callback method called and formats
    the input of each callback function with metadata regarding the state of LLM run
    and then logs the response to Aim.
    """
    def __init__(
        self,
        repo: Optional[str] = None,
        experiment_name: Optional[str] = None,
        system_tracking_interval: Optional[int] = 10,
        log_system_params: bool = True,
    ) -> None:
        """Initialize callback handler."""
        super().__init__()
        aim = import_aim()
        self.repo = repo
        self.experiment_name = experiment_name
        self.system_tracking_interval = system_tracking_interval
        self.log_system_params = log_system_params
        self._run = aim.Run(
            repo=self.repo,
            experiment=self.experiment_name,
            system_tracking_interval=self.system_tracking_interval,
            log_system_params=self.log_system_params,
        )
        self._run_hash = self._run.hash
        self.action_records: list = []


self._run_hash = self._run.hash
        self.action_records: list = []
[docs]    def setup(self, **kwargs: Any) -> None:
        aim = import_aim()
        if not self._run:
            if self._run_hash:
                self._run = aim.Run(
                    self._run_hash,
                    repo=self.repo,
                    system_tracking_interval=self.system_tracking_interval,
                )
            else:
                self._run = aim.Run(
                    repo=self.repo,
                    experiment=self.experiment_name,
                    system_tracking_interval=self.system_tracking_interval,
                    log_system_params=self.log_system_params,
                )
                self._run_hash = self._run.hash
        if kwargs:
            for key, value in kwargs.items():
                self._run.set(key, value, strict=False)
[docs]    def on_llm_start(
        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
    ) -> None:
        """Run when LLM starts."""
        aim = import_aim()
        self.step += 1
        self.llm_starts += 1
        self.starts += 1
        resp = {"action": "on_llm_start"}
        resp.update(self.get_custom_callback_meta())
        prompts_res = deepcopy(prompts)
        self._run.track(
            [aim.Text(prompt) for prompt in prompts_res],
            name="on_llm_start",
            context=resp,
        )
[docs]    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
        """Run when LLM ends running."""
        aim = import_aim()
        self.step += 1
        self.llm_ends += 1
        self.ends += 1


self.llm_ends += 1
        self.ends += 1
        resp = {"action": "on_llm_end"}
        resp.update(self.get_custom_callback_meta())
        response_res = deepcopy(response)
        generated = [
            aim.Text(generation.text)
            for generations in response_res.generations
            for generation in generations
        ]
        self._run.track(
            generated,
            name="on_llm_end",
            context=resp,
        )
[docs]    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
        """Run when LLM generates a new token."""
        self.step += 1
        self.llm_streams += 1
[docs]    def on_llm_error(
        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
    ) -> None:
        """Run when LLM errors."""
        self.step += 1
        self.errors += 1
[docs]    def on_chain_start(
        self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any
    ) -> None:
        """Run when chain starts running."""
        aim = import_aim()
        self.step += 1
        self.chain_starts += 1
        self.starts += 1
        resp = {"action": "on_chain_start"}
        resp.update(self.get_custom_callback_meta())
        inputs_res = deepcopy(inputs)
        self._run.track(
            aim.Text(inputs_res["input"]), name="on_chain_start", context=resp
        )
[docs]    def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
        """Run when chain ends running."""
        aim = import_aim()


"""Run when chain ends running."""
        aim = import_aim()
        self.step += 1
        self.chain_ends += 1
        self.ends += 1
        resp = {"action": "on_chain_end"}
        resp.update(self.get_custom_callback_meta())
        outputs_res = deepcopy(outputs)
        self._run.track(
            aim.Text(outputs_res["output"]), name="on_chain_end", context=resp
        )
[docs]    def on_chain_error(
        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
    ) -> None:
        """Run when chain errors."""
        self.step += 1
        self.errors += 1
[docs]    def on_tool_start(
        self, serialized: Dict[str, Any], input_str: str, **kwargs: Any
    ) -> None:
        """Run when tool starts running."""
        aim = import_aim()
        self.step += 1
        self.tool_starts += 1
        self.starts += 1
        resp = {"action": "on_tool_start"}
        resp.update(self.get_custom_callback_meta())
        self._run.track(aim.Text(input_str), name="on_tool_start", context=resp)
[docs]    def on_tool_end(self, output: str, **kwargs: Any) -> None:
        """Run when tool ends running."""
        aim = import_aim()
        self.step += 1
        self.tool_ends += 1
        self.ends += 1
        resp = {"action": "on_tool_end"}
        resp.update(self.get_custom_callback_meta())
        self._run.track(aim.Text(output), name="on_tool_end", context=resp)
[docs]    def on_tool_error(


[docs]    def on_tool_error(
        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
    ) -> None:
        """Run when tool errors."""
        self.step += 1
        self.errors += 1
[docs]    def on_text(self, text: str, **kwargs: Any) -> None:
        """
        Run when agent is ending.
        """
        self.step += 1
        self.text_ctr += 1
[docs]    def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> None:
        """Run when agent ends running."""
        aim = import_aim()
        self.step += 1
        self.agent_ends += 1
        self.ends += 1
        resp = {"action": "on_agent_finish"}
        resp.update(self.get_custom_callback_meta())
        finish_res = deepcopy(finish)
        text = "OUTPUT:\n{}\n\nLOG:\n{}".format(
            finish_res.return_values["output"], finish_res.log
        )
        self._run.track(aim.Text(text), name="on_agent_finish", context=resp)
[docs]    def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
        """Run on agent action."""
        aim = import_aim()
        self.step += 1
        self.tool_starts += 1
        self.starts += 1
        resp = {
            "action": "on_agent_action",
            "tool": action.tool,
        }
        resp.update(self.get_custom_callback_meta())
        action_res = deepcopy(action)
        text = "TOOL INPUT:\n{}\n\nLOG:\n{}".format(
            action_res.tool_input, action_res.log
        )


action_res.tool_input, action_res.log
        )
        self._run.track(aim.Text(text), name="on_agent_action", context=resp)
[docs]    def flush_tracker(
        self,
        repo: Optional[str] = None,
        experiment_name: Optional[str] = None,
        system_tracking_interval: Optional[int] = 10,
        log_system_params: bool = True,
        langchain_asset: Any = None,
        reset: bool = True,
        finish: bool = False,
    ) -> None:
        """Flush the tracker and reset the session.
        Args:
            repo (:obj:`str`, optional): Aim repository path or Repo object to which
                Run object is bound. If skipped, default Repo is used.
            experiment_name (:obj:`str`, optional): Sets Run's `experiment` property.
                'default' if not specified. Can be used later to query runs/sequences.
            system_tracking_interval (:obj:`int`, optional): Sets the tracking interval
                in seconds for system usage metrics (CPU, Memory, etc.). Set to `None`
                 to disable system metrics tracking.
            log_system_params (:obj:`bool`, optional): Enable/Disable logging of system
                params such as installed packages, git info, environment variables, etc.
            langchain_asset: The langchain asset to save.
            reset: Whether to reset the session.
            finish: Whether to finish the run.
            Returns:
                None
        """
        if langchain_asset:
            try:
                for key, value in langchain_asset.dict().items():
                    self._run.set(key, value, strict=False)
            except Exception:
                pass
        if finish or reset:
            self._run.close()
            self.reset_callback_meta()


self._run.close()
            self.reset_callback_meta()
        if reset:
            self.__init__(  # type: ignore
                repo=repo if repo else self.repo,
                experiment_name=experiment_name
                if experiment_name
                else self.experiment_name,
                system_tracking_interval=system_tracking_interval
                if system_tracking_interval
                else self.system_tracking_interval,
                log_system_params=log_system_params
                if log_system_params
                else self.log_system_params,
            )


Source code for langchain.callbacks.clearml_callback
import tempfile
from copy import deepcopy
from pathlib import Path
from typing import Any, Dict, List, Optional, Sequence, Union
from langchain.callbacks.base import BaseCallbackHandler
from langchain.callbacks.utils import (
    BaseMetadataCallbackHandler,
    flatten_dict,
    hash_string,
    import_pandas,
    import_spacy,
    import_textstat,
    load_json,
)
from langchain.schema import AgentAction, AgentFinish, LLMResult
[docs]def import_clearml() -> Any:
    """Import the clearml python package and raise an error if it is not installed."""
    try:
        import clearml  # noqa: F401
    except ImportError:
        raise ImportError(
            "To use the clearml callback manager you need to have the `clearml` python "
            "package installed. Please install it with `pip install clearml`"
        )
    return clearml
[docs]class ClearMLCallbackHandler(BaseMetadataCallbackHandler, BaseCallbackHandler):
    """Callback Handler that logs to ClearML.
    Parameters:
        job_type (str): The type of clearml task such as "inference", "testing" or "qc"
        project_name (str): The clearml project name
        tags (list): Tags to add to the task
        task_name (str): Name of the clearml task
        visualize (bool): Whether to visualize the run.
        complexity_metrics (bool): Whether to log complexity metrics
        stream_logs (bool): Whether to stream callback actions to ClearML
    This handler will utilize the associated callback method and formats
    the input of each callback function with metadata regarding the state of LLM run,
    and adds the response to the list of records for both the {method}_records and


and adds the response to the list of records for both the {method}_records and
    action. It then logs the response to the ClearML console.
    """
    def __init__(
        self,
        task_type: Optional[str] = "inference",
        project_name: Optional[str] = "langchain_callback_demo",
        tags: Optional[Sequence] = None,
        task_name: Optional[str] = None,
        visualize: bool = False,
        complexity_metrics: bool = False,
        stream_logs: bool = False,
    ) -> None:
        """Initialize callback handler."""
        clearml = import_clearml()
        spacy = import_spacy()
        super().__init__()
        self.task_type = task_type
        self.project_name = project_name
        self.tags = tags
        self.task_name = task_name
        self.visualize = visualize
        self.complexity_metrics = complexity_metrics
        self.stream_logs = stream_logs
        self.temp_dir = tempfile.TemporaryDirectory()
        # Check if ClearML task already exists (e.g. in pipeline)
        if clearml.Task.current_task():
            self.task = clearml.Task.current_task()
        else:
            self.task = clearml.Task.init(  # type: ignore
                task_type=self.task_type,
                project_name=self.project_name,
                tags=self.tags,
                task_name=self.task_name,
                output_uri=True,
            )
        self.logger = self.task.get_logger()
        warning = (
            "The clearml callback is currently in beta and is subject to change "
            "based on updates to `langchain`. Please report any issues to "
            "https://github.com/allegroai/clearml/issues with the tag `langchain`."
        )


)
        self.logger.report_text(warning, level=30, print_console=True)
        self.callback_columns: list = []
        self.action_records: list = []
        self.complexity_metrics = complexity_metrics
        self.visualize = visualize
        self.nlp = spacy.load("en_core_web_sm")
    def _init_resp(self) -> Dict:
        return {k: None for k in self.callback_columns}
[docs]    def on_llm_start(
        self, serialized: Dict[str, Any], prompts: List[str], **kwargs: Any
    ) -> None:
        """Run when LLM starts."""
        self.step += 1
        self.llm_starts += 1
        self.starts += 1
        resp = self._init_resp()
        resp.update({"action": "on_llm_start"})
        resp.update(flatten_dict(serialized))
        resp.update(self.get_custom_callback_meta())
        for prompt in prompts:
            prompt_resp = deepcopy(resp)
            prompt_resp["prompts"] = prompt
            self.on_llm_start_records.append(prompt_resp)
            self.action_records.append(prompt_resp)
            if self.stream_logs:
                self.logger.report_text(prompt_resp)
[docs]    def on_llm_new_token(self, token: str, **kwargs: Any) -> None:
        """Run when LLM generates a new token."""
        self.step += 1
        self.llm_streams += 1
        resp = self._init_resp()
        resp.update({"action": "on_llm_new_token", "token": token})
        resp.update(self.get_custom_callback_meta())
        self.on_llm_token_records.append(resp)
        self.action_records.append(resp)
        if self.stream_logs:
            self.logger.report_text(resp)


if self.stream_logs:
            self.logger.report_text(resp)
[docs]    def on_llm_end(self, response: LLMResult, **kwargs: Any) -> None:
        """Run when LLM ends running."""
        self.step += 1
        self.llm_ends += 1
        self.ends += 1
        resp = self._init_resp()
        resp.update({"action": "on_llm_end"})
        resp.update(flatten_dict(response.llm_output or {}))
        resp.update(self.get_custom_callback_meta())
        for generations in response.generations:
            for generation in generations:
                generation_resp = deepcopy(resp)
                generation_resp.update(flatten_dict(generation.dict()))
                generation_resp.update(self.analyze_text(generation.text))
                self.on_llm_end_records.append(generation_resp)
                self.action_records.append(generation_resp)
                if self.stream_logs:
                    self.logger.report_text(generation_resp)
[docs]    def on_llm_error(
        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
    ) -> None:
        """Run when LLM errors."""
        self.step += 1
        self.errors += 1
[docs]    def on_chain_start(
        self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any
    ) -> None:
        """Run when chain starts running."""
        self.step += 1
        self.chain_starts += 1
        self.starts += 1
        resp = self._init_resp()
        resp.update({"action": "on_chain_start"})
        resp.update(flatten_dict(serialized))
        resp.update(self.get_custom_callback_meta())
        chain_input = inputs["input"]
        if isinstance(chain_input, str):


chain_input = inputs["input"]
        if isinstance(chain_input, str):
            input_resp = deepcopy(resp)
            input_resp["input"] = chain_input
            self.on_chain_start_records.append(input_resp)
            self.action_records.append(input_resp)
            if self.stream_logs:
                self.logger.report_text(input_resp)
        elif isinstance(chain_input, list):
            for inp in chain_input:
                input_resp = deepcopy(resp)
                input_resp.update(inp)
                self.on_chain_start_records.append(input_resp)
                self.action_records.append(input_resp)
                if self.stream_logs:
                    self.logger.report_text(input_resp)
        else:
            raise ValueError("Unexpected data format provided!")
[docs]    def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
        """Run when chain ends running."""
        self.step += 1
        self.chain_ends += 1
        self.ends += 1
        resp = self._init_resp()
        resp.update({"action": "on_chain_end", "outputs": outputs["output"]})
        resp.update(self.get_custom_callback_meta())
        self.on_chain_end_records.append(resp)
        self.action_records.append(resp)
        if self.stream_logs:
            self.logger.report_text(resp)
[docs]    def on_chain_error(
        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
    ) -> None:
        """Run when chain errors."""
        self.step += 1
        self.errors += 1
[docs]    def on_tool_start(
        self, serialized: Dict[str, Any], input_str: str, **kwargs: Any
    ) -> None:
        """Run when tool starts running."""
        self.step += 1
        self.tool_starts += 1


self.step += 1
        self.tool_starts += 1
        self.starts += 1
        resp = self._init_resp()
        resp.update({"action": "on_tool_start", "input_str": input_str})
        resp.update(flatten_dict(serialized))
        resp.update(self.get_custom_callback_meta())
        self.on_tool_start_records.append(resp)
        self.action_records.append(resp)
        if self.stream_logs:
            self.logger.report_text(resp)
[docs]    def on_tool_end(self, output: str, **kwargs: Any) -> None:
        """Run when tool ends running."""
        self.step += 1
        self.tool_ends += 1
        self.ends += 1
        resp = self._init_resp()
        resp.update({"action": "on_tool_end", "output": output})
        resp.update(self.get_custom_callback_meta())
        self.on_tool_end_records.append(resp)
        self.action_records.append(resp)
        if self.stream_logs:
            self.logger.report_text(resp)
[docs]    def on_tool_error(
        self, error: Union[Exception, KeyboardInterrupt], **kwargs: Any
    ) -> None:
        """Run when tool errors."""
        self.step += 1
        self.errors += 1
[docs]    def on_text(self, text: str, **kwargs: Any) -> None:
        """
        Run when agent is ending.
        """
        self.step += 1
        self.text_ctr += 1
        resp = self._init_resp()
        resp.update({"action": "on_text", "text": text})
        resp.update(self.get_custom_callback_meta())
        self.on_text_records.append(resp)
        self.action_records.append(resp)
        if self.stream_logs:
            self.logger.report_text(resp)


if self.stream_logs:
            self.logger.report_text(resp)
[docs]    def on_agent_finish(self, finish: AgentFinish, **kwargs: Any) -> None:
        """Run when agent ends running."""
        self.step += 1
        self.agent_ends += 1
        self.ends += 1
        resp = self._init_resp()
        resp.update(
            {
                "action": "on_agent_finish",
                "output": finish.return_values["output"],
                "log": finish.log,
            }
        )
        resp.update(self.get_custom_callback_meta())
        self.on_agent_finish_records.append(resp)
        self.action_records.append(resp)
        if self.stream_logs:
            self.logger.report_text(resp)
[docs]    def on_agent_action(self, action: AgentAction, **kwargs: Any) -> Any:
        """Run on agent action."""
        self.step += 1
        self.tool_starts += 1
        self.starts += 1
        resp = self._init_resp()
        resp.update(
            {
                "action": "on_agent_action",
                "tool": action.tool,
                "tool_input": action.tool_input,
                "log": action.log,
            }
        )
        resp.update(self.get_custom_callback_meta())
        self.on_agent_action_records.append(resp)
        self.action_records.append(resp)
        if self.stream_logs:
            self.logger.report_text(resp)
[docs]    def analyze_text(self, text: str) -> dict:
        """Analyze text using textstat and spacy.
        Parameters:
            text (str): The text to analyze.
        Returns:
            (dict): A dictionary containing the complexity metrics.
        """
        resp = {}
        textstat = import_textstat()


"""
        resp = {}
        textstat = import_textstat()
        spacy = import_spacy()
        if self.complexity_metrics:
            text_complexity_metrics = {
                "flesch_reading_ease": textstat.flesch_reading_ease(text),
                "flesch_kincaid_grade": textstat.flesch_kincaid_grade(text),
                "smog_index": textstat.smog_index(text),
                "coleman_liau_index": textstat.coleman_liau_index(text),
                "automated_readability_index": textstat.automated_readability_index(
                    text
                ),
                "dale_chall_readability_score": textstat.dale_chall_readability_score(
                    text
                ),
                "difficult_words": textstat.difficult_words(text),
                "linsear_write_formula": textstat.linsear_write_formula(text),
                "gunning_fog": textstat.gunning_fog(text),
                "text_standard": textstat.text_standard(text),
                "fernandez_huerta": textstat.fernandez_huerta(text),
                "szigriszt_pazos": textstat.szigriszt_pazos(text),
                "gutierrez_polini": textstat.gutierrez_polini(text),
                "crawford": textstat.crawford(text),
                "gulpease_index": textstat.gulpease_index(text),
                "osman": textstat.osman(text),
            }
            resp.update(text_complexity_metrics)
        if self.visualize and self.nlp and self.temp_dir.name is not None:
            doc = self.nlp(text)
            dep_out = spacy.displacy.render(  # type: ignore


dep_out = spacy.displacy.render(  # type: ignore
                doc, style="dep", jupyter=False, page=True
            )
            dep_output_path = Path(
                self.temp_dir.name, hash_string(f"dep-{text}") + ".html"
            )
            dep_output_path.open("w", encoding="utf-8").write(dep_out)
            ent_out = spacy.displacy.render(  # type: ignore
                doc, style="ent", jupyter=False, page=True
            )
            ent_output_path = Path(
                self.temp_dir.name, hash_string(f"ent-{text}") + ".html"
            )
            ent_output_path.open("w", encoding="utf-8").write(ent_out)
            self.logger.report_media(
                "Dependencies Plot", text, local_path=dep_output_path
            )
            self.logger.report_media("Entities Plot", text, local_path=ent_output_path)
        return resp
    def _create_session_analysis_df(self) -> Any:
        """Create a dataframe with all the information from the session."""
        pd = import_pandas()
        on_llm_start_records_df = pd.DataFrame(self.on_llm_start_records)
        on_llm_end_records_df = pd.DataFrame(self.on_llm_end_records)
        llm_input_prompts_df = (
            on_llm_start_records_df[["step", "prompts", "name"]]
            .dropna(axis=1)
            .rename({"step": "prompt_step"}, axis=1)
        )
        complexity_metrics_columns = []
        visualizations_columns: List = []
        if self.complexity_metrics:
            complexity_metrics_columns = [
                "flesch_reading_ease",
                "flesch_kincaid_grade",


"flesch_kincaid_grade",
                "smog_index",
                "coleman_liau_index",
                "automated_readability_index",
                "dale_chall_readability_score",
                "difficult_words",
                "linsear_write_formula",
                "gunning_fog",
                "text_standard",
                "fernandez_huerta",
                "szigriszt_pazos",
                "gutierrez_polini",
                "crawford",
                "gulpease_index",
                "osman",
            ]
        llm_outputs_df = (
            on_llm_end_records_df[
                [
                    "step",
                    "text",
                    "token_usage_total_tokens",
                    "token_usage_prompt_tokens",
                    "token_usage_completion_tokens",
                ]
                + complexity_metrics_columns
                + visualizations_columns
            ]
            .dropna(axis=1)
            .rename({"step": "output_step", "text": "output"}, axis=1)
        )
        session_analysis_df = pd.concat([llm_input_prompts_df, llm_outputs_df], axis=1)
        # session_analysis_df["chat_html"] = session_analysis_df[
        #     ["prompts", "output"]
        # ].apply(
        #     lambda row: construct_html_from_prompt_and_generation(
        #         row["prompts"], row["output"]
        #     ),
        #     axis=1,
        # )
        return session_analysis_df
[docs]    def flush_tracker(
        self,
        name: Optional[str] = None,
        langchain_asset: Any = None,
        finish: bool = False,
    ) -> None:


finish: bool = False,
    ) -> None:
        """Flush the tracker and setup the session.
        Everything after this will be a new table.
        Args:
            name: Name of the preformed session so far so it is identifyable
            langchain_asset: The langchain asset to save.
            finish: Whether to finish the run.
            Returns:
                None
        """
        pd = import_pandas()
        clearml = import_clearml()
        # Log the action records
        self.logger.report_table(
            "Action Records", name, table_plot=pd.DataFrame(self.action_records)
        )
        # Session analysis
        session_analysis_df = self._create_session_analysis_df()
        self.logger.report_table(
            "Session Analysis", name, table_plot=session_analysis_df
        )
        if self.stream_logs:
            self.logger.report_text(
                {
                    "action_records": pd.DataFrame(self.action_records),
                    "session_analysis": session_analysis_df,
                }
            )
        if langchain_asset:
            langchain_asset_path = Path(self.temp_dir.name, "model.json")
            try:
                langchain_asset.save(langchain_asset_path)
                # Create output model and connect it to the task
                output_model = clearml.OutputModel(
                    task=self.task, config_text=load_json(langchain_asset_path)
                )
                output_model.update_weights(
                    weights_filename=str(langchain_asset_path),
                    auto_delete_file=False,
                    target_filename=name,
                )
            except ValueError:
                langchain_asset.save_agent(langchain_asset_path)
                output_model = clearml.OutputModel(
                    task=self.task, config_text=load_json(langchain_asset_path)
                )
                output_model.update_weights(


)
                output_model.update_weights(
                    weights_filename=str(langchain_asset_path),
                    auto_delete_file=False,
                    target_filename=name,
                )
            except NotImplementedError as e:
                print("Could not save model.")
                print(repr(e))
                pass
        # Cleanup after adding everything to ClearML
        self.task.flush(wait_for_uploads=True)
        self.temp_dir.cleanup()
        self.temp_dir = tempfile.TemporaryDirectory()
        self.reset_callback_meta()
        if finish:
            self.task.close()


Source code for langchain.callbacks.file
"""Callback Handler that writes to a file."""
from typing import Any, Dict, Optional, TextIO, cast
from langchain.callbacks.base import BaseCallbackHandler
from langchain.input import print_text
from langchain.schema import AgentAction, AgentFinish
[docs]class FileCallbackHandler(BaseCallbackHandler):
    """Callback Handler that writes to a file."""
    def __init__(
        self, filename: str, mode: str = "a", color: Optional[str] = None
    ) -> None:
        """Initialize callback handler."""
        self.file = cast(TextIO, open(filename, mode))
        self.color = color
    def __del__(self) -> None:
        """Destructor to cleanup when done."""
        self.file.close()
[docs]    def on_chain_start(
        self, serialized: Dict[str, Any], inputs: Dict[str, Any], **kwargs: Any
    ) -> None:
        """Print out that we are entering a chain."""
        class_name = serialized["name"]
        print_text(
            f"\n\n\033[1m> Entering new {class_name} chain...\033[0m",
            end="\n",
            file=self.file,
        )
[docs]    def on_chain_end(self, outputs: Dict[str, Any], **kwargs: Any) -> None:
        """Print out that we finished a chain."""
        print_text("\n\033[1m> Finished chain.\033[0m", end="\n", file=self.file)
[docs]    def on_agent_action(
        self, action: AgentAction, color: Optional[str] = None, **kwargs: Any
    ) -> Any:
        """Run on agent action."""