Add files using upload-large-folder tool

bioagent-bench/.venv/lib/python3.14/site-packages/attr/_version_info.py

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: MIT
+
+
+from functools import total_ordering
+
+from ._funcs import astuple
+from ._make import attrib, attrs
+
+
+@total_ordering
+@attrs(eq=False, order=False, slots=True, frozen=True)
+class VersionInfo:
+    """
+    A version object that can be compared to tuple of length 1--4:
+
+    >>> attr.VersionInfo(19, 1, 0, "final")  <= (19, 2)
+    True
+    >>> attr.VersionInfo(19, 1, 0, "final") < (19, 1, 1)
+    True
+    >>> vi = attr.VersionInfo(19, 2, 0, "final")
+    >>> vi < (19, 1, 1)
+    False
+    >>> vi < (19,)
+    False
+    >>> vi == (19, 2,)
+    True
+    >>> vi == (19, 2, 1)
+    False
+
+    .. versionadded:: 19.2
+    """
+
+    year = attrib(type=int)
+    minor = attrib(type=int)
+    micro = attrib(type=int)
+    releaselevel = attrib(type=str)
+
+    @classmethod
+    def _from_version_string(cls, s):
+        """
+        Parse *s* and return a _VersionInfo.
+        """
+        v = s.split(".")
+        if len(v) == 3:
+            v.append("final")
+
+        return cls(
+            year=int(v[0]), minor=int(v[1]), micro=int(v[2]), releaselevel=v[3]
+        )
+
+    def _ensure_tuple(self, other):
+        """
+        Ensure *other* is a tuple of a valid length.
+
+        Returns a possibly transformed *other* and ourselves as a tuple of
+        the same length as *other*.
+        """
+
+        if self.__class__ is other.__class__:
+            other = astuple(other)
+
+        if not isinstance(other, tuple):
+            raise NotImplementedError
+
+        if not (1 <= len(other) <= 4):
+            raise NotImplementedError
+
+        return astuple(self)[: len(other)], other
+
+    def __eq__(self, other):
+        try:
+            us, them = self._ensure_tuple(other)
+        except NotImplementedError:
+            return NotImplemented
+
+        return us == them
+
+    def __lt__(self, other):
+        try:
+            us, them = self._ensure_tuple(other)
+        except NotImplementedError:
+            return NotImplemented
+
+        # Since alphabetically "dev0" < "final" < "post1" < "post2", we don't
+        # have to do anything special with releaselevel for now.
+        return us < them
+
+    def __hash__(self):
+        return hash((self.year, self.minor, self.micro, self.releaselevel))

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/filesystems/__init__.py

@@ -0,0 +1,47 @@
+import importlib
+import shutil
+import warnings
+from typing import List
+
+import fsspec
+import fsspec.asyn
+from fsspec.implementations.local import LocalFileSystem
+
+from . import compression
+
+
+COMPRESSION_FILESYSTEMS: list[compression.BaseCompressedFileFileSystem] = [
+    compression.Bz2FileSystem,
+    compression.GzipFileSystem,
+    compression.Lz4FileSystem,
+    compression.XzFileSystem,
+    compression.ZstdFileSystem,
+]
+
+# Register custom filesystems
+for fs_class in COMPRESSION_FILESYSTEMS:
+    if fs_class.protocol in fsspec.registry and fsspec.registry[fs_class.protocol] is not fs_class:
+        warnings.warn(f"A filesystem protocol was already set for {fs_class.protocol} and will be overwritten.")
+    fsspec.register_implementation(fs_class.protocol, fs_class, clobber=True)
+
+
+def is_remote_filesystem(fs: fsspec.AbstractFileSystem) -> bool:
+    """
+    Checks if `fs` is a remote filesystem.
+
+    Args:
+        fs (`fsspec.spec.AbstractFileSystem`):
+            An abstract super-class for pythonic file-systems, e.g. `fsspec.filesystem(\'file\')` or `s3fs.S3FileSystem`.
+    """
+    return not isinstance(fs, LocalFileSystem)
+
+
+def rename(fs: fsspec.AbstractFileSystem, src: str, dst: str):
+    """
+    Renames the file `src` in `fs` to `dst`.
+    """
+    if not is_remote_filesystem(fs):
+        # LocalFileSystem.mv does copy + rm, it is more efficient to simply move a local directory
+        shutil.move(fs._strip_protocol(src), fs._strip_protocol(dst))
+    else:
+        fs.mv(src, dst, recursive=True)

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/formatting/__init__.py

@@ -0,0 +1,136 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the TensorFlow Datasets Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from typing import Dict, List, Optional, Type
+
+from .. import config
+from ..utils import logging
+from .formatting import (
+    ArrowFormatter,
+    CustomFormatter,
+    Formatter,
+    PandasFormatter,
+    PythonFormatter,
+    TableFormatter,
+    TensorFormatter,
+    format_table,
+    query_table,
+)
+from .np_formatter import NumpyFormatter
+
+
+logger = logging.get_logger(__name__)
+
+_FORMAT_TYPES: dict[Optional[str], type[Formatter]] = {}
+_FORMAT_TYPES_ALIASES: dict[Optional[str], str] = {}
+_FORMAT_TYPES_ALIASES_UNAVAILABLE: dict[Optional[str], Exception] = {}
+
+
+def _register_formatter(
+    formatter_cls: type,
+    format_type: Optional[str],
+    aliases: Optional[list[str]] = None,
+):
+    """
+    Register a Formatter object using a name and optional aliases.
+    This function must be used on a Formatter class.
+    """
+    aliases = aliases if aliases is not None else []
+    if format_type in _FORMAT_TYPES:
+        logger.warning(
+            f"Overwriting format type '{format_type}' ({_FORMAT_TYPES[format_type].__name__} -> {formatter_cls.__name__})"
+        )
+    _FORMAT_TYPES[format_type] = formatter_cls
+    for alias in set(aliases + [format_type]):
+        if alias in _FORMAT_TYPES_ALIASES:
+            logger.warning(
+                f"Overwriting format type alias '{alias}' ({_FORMAT_TYPES_ALIASES[alias]} -> {format_type})"
+            )
+        _FORMAT_TYPES_ALIASES[alias] = format_type
+
+
+def _register_unavailable_formatter(
+    unavailable_error: Exception, format_type: Optional[str], aliases: Optional[list[str]] = None
+):
+    """
+    Register an unavailable Formatter object using a name and optional aliases.
+    This function must be used on an Exception object that is raised when trying to get the unavailable formatter.
+    """
+    aliases = aliases if aliases is not None else []
+    for alias in set(aliases + [format_type]):
+        _FORMAT_TYPES_ALIASES_UNAVAILABLE[alias] = unavailable_error
+
+
+# Here we define all the available formatting functions that can be used by `Dataset.set_format`
+_register_formatter(PythonFormatter, None, aliases=["python"])
+_register_formatter(ArrowFormatter, "arrow", aliases=["pa", "pyarrow"])
+_register_formatter(NumpyFormatter, "numpy", aliases=["np"])
+_register_formatter(PandasFormatter, "pandas", aliases=["pd"])
+_register_formatter(CustomFormatter, "custom")
+
+if config.POLARS_AVAILABLE:
+    from .polars_formatter import PolarsFormatter
+
+    _register_formatter(PolarsFormatter, "polars", aliases=["pl"])
+else:
+    _polars_error = ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+    _register_unavailable_formatter(_polars_error, "polars", aliases=["pl"])
+
+if config.TORCH_AVAILABLE:
+    from .torch_formatter import TorchFormatter
+
+    _register_formatter(TorchFormatter, "torch", aliases=["pt", "pytorch"])
+else:
+    _torch_error = ValueError("PyTorch needs to be installed to be able to return PyTorch tensors.")
+    _register_unavailable_formatter(_torch_error, "torch", aliases=["pt", "pytorch"])
+
+if config.TF_AVAILABLE:
+    from .tf_formatter import TFFormatter
+
+    _register_formatter(TFFormatter, "tensorflow", aliases=["tf"])
+else:
+    _tf_error = ValueError("Tensorflow needs to be installed to be able to return Tensorflow tensors.")
+    _register_unavailable_formatter(_tf_error, "tensorflow", aliases=["tf"])
+
+if config.JAX_AVAILABLE:
+    from .jax_formatter import JaxFormatter
+
+    _register_formatter(JaxFormatter, "jax", aliases=[])
+else:
+    _jax_error = ValueError("JAX needs to be installed to be able to return JAX arrays.")
+    _register_unavailable_formatter(_jax_error, "jax", aliases=[])
+
+
+def get_format_type_from_alias(format_type: Optional[str]) -> Optional[str]:
+    """If the given format type is a known alias, then return its main type name. Otherwise return the type with no change."""
+    if format_type in _FORMAT_TYPES_ALIASES:
+        return _FORMAT_TYPES_ALIASES[format_type]
+    else:
+        return format_type
+
+
+def get_formatter(format_type: Optional[str], **format_kwargs) -> Formatter:
+    """
+    Factory function to get a Formatter given its type name and keyword arguments.
+    A formatter is an object that extracts and formats data from pyarrow table.
+    It defines the formatting for rows, columns and batches.
+    If the formatter for a given type name doesn't exist or is not available, an error is raised.
+    """
+    format_type = get_format_type_from_alias(format_type)
+    if format_type in _FORMAT_TYPES:
+        return _FORMAT_TYPES[format_type](**format_kwargs)
+    if format_type in _FORMAT_TYPES_ALIASES_UNAVAILABLE:
+        raise _FORMAT_TYPES_ALIASES_UNAVAILABLE[format_type]
+    else:
+        raise ValueError(f"Format type should be one of {list(_FORMAT_TYPES.keys())}, but got '{format_type}'")

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/formatting/formatting.py

@@ -0,0 +1,678 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import numbers
+import operator
+from collections.abc import Iterable, Mapping, MutableMapping
+from functools import partial
+
+# Lint as: python3
+from typing import Any, Callable, Generic, Optional, TypeVar, Union
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+
+from ..features import Features
+from ..features.features import _ArrayXDExtensionType, _is_zero_copy_only, decode_nested_example, pandas_types_mapper
+from ..table import Table
+from ..utils.py_utils import no_op_if_value_is_null
+
+
+T = TypeVar("T")
+
+RowFormat = TypeVar("RowFormat")
+ColumnFormat = TypeVar("ColumnFormat")
+BatchFormat = TypeVar("BatchFormat")
+
+
+def _is_range_contiguous(key: range) -> bool:
+    return key.step == 1 and key.stop >= key.start
+
+
+def _raise_bad_key_type(key: Any):
+    raise TypeError(
+        f"Wrong key type: '{key}' of type '{type(key)}'. Expected one of int, slice, range, str or Iterable."
+    )
+
+
+def _query_table_with_indices_mapping(
+    table: Table, key: Union[int, slice, range, str, Iterable], indices: Table
+) -> pa.Table:
+    """
+    Query a pyarrow Table to extract the subtable that correspond to the given key.
+    The :obj:`indices` parameter corresponds to the indices mapping in case we cant to take into
+    account a shuffling or an indices selection for example.
+    The indices table must contain one column named "indices" of type uint64.
+    """
+    if isinstance(key, int):
+        key = indices.fast_slice(key % indices.num_rows, 1).column(0)[0].as_py()
+        return _query_table(table, key)
+    if isinstance(key, slice):
+        key = range(*key.indices(indices.num_rows))
+    if isinstance(key, range):
+        if _is_range_contiguous(key) and key.start >= 0:
+            return _query_table(
+                table, [i.as_py() for i in indices.fast_slice(key.start, key.stop - key.start).column(0)]
+            )
+        else:
+            pass  # treat as an iterable
+    if isinstance(key, str):
+        table = table.select([key])
+        return _query_table(table, indices.column(0).to_pylist())
+    if isinstance(key, Iterable):
+        return _query_table(table, [indices.fast_slice(i, 1).column(0)[0].as_py() for i in key])
+
+    _raise_bad_key_type(key)
+
+
+def _query_table(table: Table, key: Union[int, slice, range, str, Iterable]) -> pa.Table:
+    """
+    Query a pyarrow Table to extract the subtable that correspond to the given key.
+    """
+    if isinstance(key, int):
+        return table.fast_slice(key % table.num_rows, 1)
+    if isinstance(key, slice):
+        key = range(*key.indices(table.num_rows))
+    if isinstance(key, range):
+        if _is_range_contiguous(key) and key.start >= 0:
+            return table.fast_slice(key.start, key.stop - key.start)
+        else:
+            pass  # treat as an iterable
+    if isinstance(key, str):
+        return table.table.drop([column for column in table.column_names if column != key])
+    if isinstance(key, Iterable):
+        key = np.fromiter(key, np.int64)
+        if len(key) == 0:
+            return table.table.slice(0, 0)
+        # don't use pyarrow.Table.take even for pyarrow >=1.0 (see https://issues.apache.org/jira/browse/ARROW-9773)
+        return table.fast_gather(key % table.num_rows)
+
+    _raise_bad_key_type(key)
+
+
+def _is_array_with_nulls(pa_array: pa.Array) -> bool:
+    return pa_array.null_count > 0
+
+
+class BaseArrowExtractor(Generic[RowFormat, ColumnFormat, BatchFormat]):
+    """
+    Arrow extractor are used to extract data from pyarrow tables.
+    It makes it possible to extract rows, columns and batches.
+    These three extractions types have to be implemented.
+    """
+
+    def extract_row(self, pa_table: pa.Table) -> RowFormat:
+        raise NotImplementedError
+
+    def extract_column(self, pa_table: pa.Table) -> ColumnFormat:
+        raise NotImplementedError
+
+    def extract_batch(self, pa_table: pa.Table) -> BatchFormat:
+        raise NotImplementedError
+
+
+def _unnest(py_dict: dict[str, list[T]]) -> dict[str, T]:
+    """Return the first element of a batch (dict) as a row (dict)"""
+    return {key: array[0] for key, array in py_dict.items()}
+
+
+class SimpleArrowExtractor(BaseArrowExtractor[pa.Table, pa.Array, pa.Table]):
+    def extract_row(self, pa_table: pa.Table) -> pa.Table:
+        return pa_table
+
+    def extract_column(self, pa_table: pa.Table) -> pa.Array:
+        return pa_table.column(0)
+
+    def extract_batch(self, pa_table: pa.Table) -> pa.Table:
+        return pa_table
+
+
+class PythonArrowExtractor(BaseArrowExtractor[dict, list, dict]):
+    def extract_row(self, pa_table: pa.Table) -> dict:
+        return _unnest(pa_table.to_pydict())
+
+    def extract_column(self, pa_table: pa.Table) -> list:
+        return pa_table.column(0).to_pylist()
+
+    def extract_batch(self, pa_table: pa.Table) -> dict:
+        return pa_table.to_pydict()
+
+
+class NumpyArrowExtractor(BaseArrowExtractor[dict, np.ndarray, dict]):
+    def __init__(self, **np_array_kwargs):
+        self.np_array_kwargs = np_array_kwargs
+
+    def extract_row(self, pa_table: pa.Table) -> dict:
+        return _unnest(self.extract_batch(pa_table))
+
+    def extract_column(self, pa_table: pa.Table) -> np.ndarray:
+        return self._arrow_array_to_numpy(pa_table[pa_table.column_names[0]])
+
+    def extract_batch(self, pa_table: pa.Table) -> dict:
+        return {col: self._arrow_array_to_numpy(pa_table[col]) for col in pa_table.column_names}
+
+    def _arrow_array_to_numpy(self, pa_array: pa.Array) -> np.ndarray:
+        if isinstance(pa_array, pa.ChunkedArray):
+            if isinstance(pa_array.type, _ArrayXDExtensionType):
+                # don't call to_pylist() to preserve dtype of the fixed-size array
+                zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True)
+                array: list = [
+                    row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only)
+                ]
+            else:
+                zero_copy_only = _is_zero_copy_only(pa_array.type) and all(
+                    not _is_array_with_nulls(chunk) for chunk in pa_array.chunks
+                )
+                array: list = [
+                    row for chunk in pa_array.chunks for row in chunk.to_numpy(zero_copy_only=zero_copy_only)
+                ]
+        else:
+            if isinstance(pa_array.type, _ArrayXDExtensionType):
+                # don't call to_pylist() to preserve dtype of the fixed-size array
+                zero_copy_only = _is_zero_copy_only(pa_array.type.storage_dtype, unnest=True)
+                array: list = pa_array.to_numpy(zero_copy_only=zero_copy_only)
+            else:
+                zero_copy_only = _is_zero_copy_only(pa_array.type) and not _is_array_with_nulls(pa_array)
+                array: list = pa_array.to_numpy(zero_copy_only=zero_copy_only).tolist()
+
+        if len(array) > 0:
+            if any(
+                (isinstance(x, np.ndarray) and (x.dtype == object or x.shape != array[0].shape))
+                or (isinstance(x, float) and np.isnan(x))
+                for x in array
+            ):
+                if np.lib.NumpyVersion(np.__version__) >= "2.0.0b1":
+                    return np.asarray(array, dtype=object)
+                return np.array(array, copy=False, dtype=object)
+        if np.lib.NumpyVersion(np.__version__) >= "2.0.0b1":
+            return np.asarray(array)
+        else:
+            return np.array(array, copy=False)
+
+
+class PandasArrowExtractor(BaseArrowExtractor[pd.DataFrame, pd.Series, pd.DataFrame]):
+    def extract_row(self, pa_table: pa.Table) -> pd.DataFrame:
+        return pa_table.slice(length=1).to_pandas(types_mapper=pandas_types_mapper)
+
+    def extract_column(self, pa_table: pa.Table) -> pd.Series:
+        return pa_table.select([0]).to_pandas(types_mapper=pandas_types_mapper)[pa_table.column_names[0]]
+
+    def extract_batch(self, pa_table: pa.Table) -> pd.DataFrame:
+        return pa_table.to_pandas(types_mapper=pandas_types_mapper)
+
+
+class PythonFeaturesDecoder:
+    def __init__(
+        self, features: Optional[Features], token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None
+    ):
+        self.features = features
+        self.token_per_repo_id = token_per_repo_id
+
+    def decode_row(self, row: dict) -> dict:
+        return self.features.decode_example(row, token_per_repo_id=self.token_per_repo_id) if self.features else row
+
+    def decode_column(self, column: list, column_name: str) -> list:
+        return (
+            self.features.decode_column(column, column_name, token_per_repo_id=self.token_per_repo_id)
+            if self.features
+            else column
+        )
+
+    def decode_batch(self, batch: dict) -> dict:
+        return self.features.decode_batch(batch, token_per_repo_id=self.token_per_repo_id) if self.features else batch
+
+
+class PandasFeaturesDecoder:
+    def __init__(self, features: Optional[Features]):
+        self.features = features
+
+    def decode_row(self, row: pd.DataFrame) -> pd.DataFrame:
+        decode = (
+            {
+                column_name: no_op_if_value_is_null(partial(decode_nested_example, feature))
+                for column_name, feature in self.features.items()
+                if self.features._column_requires_decoding[column_name]
+            }
+            if self.features
+            else {}
+        )
+        if decode:
+            row[list(decode.keys())] = row.transform(decode)
+        return row
+
+    def decode_column(self, column: pd.Series, column_name: str) -> pd.Series:
+        decode = (
+            no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name]))
+            if self.features and column_name in self.features and self.features._column_requires_decoding[column_name]
+            else None
+        )
+        if decode:
+            column = column.transform(decode)
+        return column
+
+    def decode_batch(self, batch: pd.DataFrame) -> pd.DataFrame:
+        return self.decode_row(batch)
+
+
+class LazyDict(MutableMapping):
+    """A dictionary backed by Arrow data. The values are formatted on-the-fly when accessing the dictionary."""
+
+    def __init__(self, pa_table: pa.Table, formatter: "Formatter"):
+        self.pa_table = pa_table
+        self.formatter = formatter
+
+        self.data = dict.fromkeys(pa_table.column_names)
+        self.keys_to_format = set(self.data.keys())
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, key):
+        value = self.data[key]
+        if key in self.keys_to_format:
+            value = self.format(key)
+            self.data[key] = value
+            self.keys_to_format.remove(key)
+        return value
+
+    def __setitem__(self, key, value):
+        if key in self.keys_to_format:
+            self.keys_to_format.remove(key)
+        self.data[key] = value
+
+    def __delitem__(self, key) -> None:
+        if key in self.keys_to_format:
+            self.keys_to_format.remove(key)
+        del self.data[key]
+
+    def __iter__(self):
+        return iter(self.data)
+
+    def __contains__(self, key):
+        return key in self.data
+
+    def __repr__(self):
+        self._format_all()
+        return repr(self.data)
+
+    def __or__(self, other):
+        if isinstance(other, LazyDict):
+            inst = self.copy()
+            other = other.copy()
+            other._format_all()
+            inst.keys_to_format -= other.data.keys()
+            inst.data = inst.data | other.data
+            return inst
+        if isinstance(other, dict):
+            inst = self.copy()
+            inst.keys_to_format -= other.keys()
+            inst.data = inst.data | other
+            return inst
+        return NotImplemented
+
+    def __ror__(self, other):
+        if isinstance(other, LazyDict):
+            inst = self.copy()
+            other = other.copy()
+            other._format_all()
+            inst.keys_to_format -= other.data.keys()
+            inst.data = other.data | inst.data
+            return inst
+        if isinstance(other, dict):
+            inst = self.copy()
+            inst.keys_to_format -= other.keys()
+            inst.data = other | inst.data
+            return inst
+        return NotImplemented
+
+    def __ior__(self, other):
+        if isinstance(other, LazyDict):
+            other = other.copy()
+            other._format_all()
+            self.keys_to_format -= other.data.keys()
+            self.data |= other.data
+        else:
+            self.keys_to_format -= other.keys()
+            self.data |= other
+        return self
+
+    def __copy__(self):
+        # Identical to `UserDict.__copy__`
+        inst = self.__class__.__new__(self.__class__)
+        inst.__dict__.update(self.__dict__)
+        # Create a copy and avoid triggering descriptors
+        inst.__dict__["data"] = self.__dict__["data"].copy()
+        inst.__dict__["keys_to_format"] = self.__dict__["keys_to_format"].copy()
+        return inst
+
+    def copy(self):
+        import copy
+
+        return copy.copy(self)
+
+    @classmethod
+    def fromkeys(cls, iterable, value=None):
+        raise NotImplementedError
+
+    def format(self, key):
+        raise NotImplementedError
+
+    def _format_all(self):
+        for key in self.keys_to_format:
+            self.data[key] = self.format(key)
+        self.keys_to_format.clear()
+
+
+class LazyRow(LazyDict):
+    def format(self, key):
+        return self.formatter.format_column(self.pa_table.select([key]))[0]
+
+
+class LazyBatch(LazyDict):
+    def format(self, key):
+        return self.formatter.format_column(self.pa_table.select([key]))
+
+
+class Formatter(Generic[RowFormat, ColumnFormat, BatchFormat]):
+    """
+    A formatter is an object that extracts and formats data from pyarrow tables.
+    It defines the formatting for rows, columns and batches.
+    """
+
+    simple_arrow_extractor = SimpleArrowExtractor
+    python_arrow_extractor = PythonArrowExtractor
+    numpy_arrow_extractor = NumpyArrowExtractor
+    pandas_arrow_extractor = PandasArrowExtractor
+
+    def __init__(
+        self,
+        features: Optional[Features] = None,
+        token_per_repo_id: Optional[dict[str, Union[str, bool, None]]] = None,
+    ):
+        self.features = features
+        self.token_per_repo_id = token_per_repo_id
+        self.python_features_decoder = PythonFeaturesDecoder(self.features, self.token_per_repo_id)
+        self.pandas_features_decoder = PandasFeaturesDecoder(self.features)
+
+    def __call__(self, pa_table: pa.Table, query_type: str) -> Union[RowFormat, ColumnFormat, BatchFormat]:
+        if query_type == "row":
+            return self.format_row(pa_table)
+        elif query_type == "column":
+            return self.format_column(pa_table)
+        elif query_type == "batch":
+            return self.format_batch(pa_table)
+
+    def format_row(self, pa_table: pa.Table) -> RowFormat:
+        raise NotImplementedError
+
+    def format_column(self, pa_table: pa.Table) -> ColumnFormat:
+        raise NotImplementedError
+
+    def format_batch(self, pa_table: pa.Table) -> BatchFormat:
+        raise NotImplementedError
+
+
+class TensorFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]):
+    def recursive_tensorize(self, data_struct: dict):
+        raise NotImplementedError
+
+
+class TableFormatter(Formatter[RowFormat, ColumnFormat, BatchFormat]):
+    table_type: str
+    column_type: str
+
+
+class ArrowFormatter(TableFormatter[pa.Table, pa.Array, pa.Table]):
+    table_type = "arrow table"
+    column_type = "arrow array"
+
+    def format_row(self, pa_table: pa.Table) -> pa.Table:
+        return self.simple_arrow_extractor().extract_row(pa_table)
+
+    def format_column(self, pa_table: pa.Table) -> pa.Array:
+        return self.simple_arrow_extractor().extract_column(pa_table)
+
+    def format_batch(self, pa_table: pa.Table) -> pa.Table:
+        return self.simple_arrow_extractor().extract_batch(pa_table)
+
+
+class PythonFormatter(Formatter[Mapping, list, Mapping]):
+    def __init__(self, features=None, lazy=False, token_per_repo_id=None):
+        super().__init__(features, token_per_repo_id)
+        self.lazy = lazy
+
+    def format_row(self, pa_table: pa.Table) -> Mapping:
+        if self.lazy:
+            return LazyRow(pa_table, self)
+        row = self.python_arrow_extractor().extract_row(pa_table)
+        row = self.python_features_decoder.decode_row(row)
+        return row
+
+    def format_column(self, pa_table: pa.Table) -> list:
+        column = self.python_arrow_extractor().extract_column(pa_table)
+        column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+        return column
+
+    def format_batch(self, pa_table: pa.Table) -> Mapping:
+        if self.lazy:
+            return LazyBatch(pa_table, self)
+        batch = self.python_arrow_extractor().extract_batch(pa_table)
+        batch = self.python_features_decoder.decode_batch(batch)
+        return batch
+
+
+class PandasFormatter(TableFormatter[pd.DataFrame, pd.Series, pd.DataFrame]):
+    table_type = "pandas dataframe"
+    column_type = "pandas series"
+
+    def format_row(self, pa_table: pa.Table) -> pd.DataFrame:
+        row = self.pandas_arrow_extractor().extract_row(pa_table)
+        row = self.pandas_features_decoder.decode_row(row)
+        return row
+
+    def format_column(self, pa_table: pa.Table) -> pd.Series:
+        column = self.pandas_arrow_extractor().extract_column(pa_table)
+        column = self.pandas_features_decoder.decode_column(column, pa_table.column_names[0])
+        return column
+
+    def format_batch(self, pa_table: pa.Table) -> pd.DataFrame:
+        row = self.pandas_arrow_extractor().extract_batch(pa_table)
+        row = self.pandas_features_decoder.decode_batch(row)
+        return row
+
+
+class CustomFormatter(Formatter[dict, ColumnFormat, dict]):
+    """
+    A user-defined custom formatter function defined by a ``transform``.
+    The transform must take as input a batch of data extracted for an arrow table using the python extractor,
+    and return a batch.
+    If the output batch is not a dict, then output_all_columns won't work.
+    If the output batch has several fields, then querying a single column won't work since we don't know which field
+    to return.
+    """
+
+    def __init__(self, transform: Callable[[dict], dict], features=None, token_per_repo_id=None, **kwargs):
+        super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+        self.transform = transform
+
+    def format_row(self, pa_table: pa.Table) -> dict:
+        formatted_batch = self.format_batch(pa_table)
+        try:
+            return _unnest(formatted_batch)
+        except Exception as exc:
+            raise TypeError(
+                f"Custom formatting function must return a dict of sequences to be able to pick a row, but got {formatted_batch}"
+            ) from exc
+
+    def format_column(self, pa_table: pa.Table) -> ColumnFormat:
+        formatted_batch = self.format_batch(pa_table)
+        if hasattr(formatted_batch, "keys"):
+            if len(formatted_batch.keys()) > 1:
+                raise TypeError(
+                    "Tried to query a column but the custom formatting function returns too many columns. "
+                    f"Only one column was expected but got columns {list(formatted_batch.keys())}."
+                )
+        else:
+            raise TypeError(
+                f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}"
+            )
+        try:
+            return formatted_batch[pa_table.column_names[0]]
+        except Exception as exc:
+            raise TypeError(
+                f"Custom formatting function must return a dict to be able to pick a row, but got {formatted_batch}"
+            ) from exc
+
+    def format_batch(self, pa_table: pa.Table) -> dict:
+        batch = self.python_arrow_extractor().extract_batch(pa_table)
+        batch = self.python_features_decoder.decode_batch(batch)
+        return self.transform(batch)
+
+
+def _check_valid_column_key(key: str, columns: list[str]) -> None:
+    if key not in columns:
+        raise KeyError(f"Column {key} not in the dataset. Current columns in the dataset: {columns}")
+
+
+def _check_valid_index_key(key: Union[int, slice, range, Iterable], size: int) -> None:
+    if isinstance(key, int):
+        if (key < 0 and key + size < 0) or (key >= size):
+            raise IndexError(f"Invalid key: {key} is out of bounds for size {size}")
+        return
+    elif isinstance(key, slice):
+        pass
+    elif isinstance(key, range):
+        if len(key) > 0:
+            _check_valid_index_key(max(key), size=size)
+            _check_valid_index_key(min(key), size=size)
+    elif isinstance(key, Iterable):
+        if len(key) > 0:
+            _check_valid_index_key(int(max(key)), size=size)
+            _check_valid_index_key(int(min(key)), size=size)
+    else:
+        _raise_bad_key_type(key)
+
+
+def key_to_query_type(key: Union[int, slice, range, str, Iterable]) -> str:
+    if isinstance(key, numbers.Integral):
+        return "row"
+    elif isinstance(key, str):
+        return "column"
+    elif isinstance(key, (slice, range, Iterable)):
+        return "batch"
+    _raise_bad_key_type(key)
+
+
+def query_table(
+    table: Table,
+    key: Union[int, slice, range, str, Iterable],
+    indices: Optional[Table] = None,
+) -> pa.Table:
+    """
+    Query a Table to extract the subtable that correspond to the given key.
+
+    Args:
+        table (``datasets.table.Table``): The input Table to query from
+        key (``Union[int, slice, range, str, Iterable]``): The key can be of different types:
+            - an integer i: the subtable containing only the i-th row
+            - a slice [i:j:k]: the subtable containing the rows that correspond to this slice
+            - a range(i, j, k): the subtable containing the rows that correspond to this range
+            - a string c: the subtable containing all the rows but only the column c
+            - an iterable l: the subtable that is the concatenation of all the i-th rows for all i in the iterable
+        indices (Optional ``datasets.table.Table``): If not None, it is used to re-map the given key to the table rows.
+            The indices table must contain one column named "indices" of type uint64.
+            This is used in case of shuffling or rows selection.
+
+
+    Returns:
+        ``pyarrow.Table``: the result of the query on the input table
+    """
+    # Check if key is valid
+    if not isinstance(key, (int, slice, range, str, Iterable)):
+        try:
+            key = operator.index(key)
+        except TypeError:
+            _raise_bad_key_type(key)
+    if isinstance(key, str):
+        _check_valid_column_key(key, table.column_names)
+    else:
+        size = indices.num_rows if indices is not None else table.num_rows
+        _check_valid_index_key(key, size)
+    # Query the main table
+    if indices is None:
+        pa_subtable = _query_table(table, key)
+    else:
+        pa_subtable = _query_table_with_indices_mapping(table, key, indices=indices)
+    return pa_subtable
+
+
+def format_table(
+    table: Table,
+    key: Union[int, slice, range, str, Iterable],
+    formatter: Formatter,
+    format_columns: Optional[list] = None,
+    output_all_columns=False,
+):
+    """
+    Format a Table depending on the key that was used and a Formatter object.
+
+    Args:
+        table (``datasets.table.Table``): The input Table to format
+        key (``Union[int, slice, range, str, Iterable]``): Depending on the key that was used, the formatter formats
+            the table as either a row, a column or a batch.
+        formatter (``datasets.formatting.formatting.Formatter``): Any subclass of a Formatter such as
+            PythonFormatter, NumpyFormatter, etc.
+        format_columns (:obj:`List[str]`, optional): if not None, it defines the columns that will be formatted using the
+            given formatter. Other columns are discarded (unless ``output_all_columns`` is True)
+        output_all_columns (:obj:`bool`, defaults to False). If True, the formatted output is completed using the columns
+            that are not in the ``format_columns`` list. For these columns, the PythonFormatter is used.
+
+
+    Returns:
+        A row, column or batch formatted object defined by the Formatter:
+        - the PythonFormatter returns a dictionary for a row or a batch, and a list for a column.
+        - the NumpyFormatter returns a dictionary for a row or a batch, and a np.array for a column.
+        - the PandasFormatter returns a pd.DataFrame for a row or a batch, and a pd.Series for a column.
+        - the TorchFormatter returns a dictionary for a row or a batch, and a torch.Tensor for a column.
+        - the TFFormatter returns a dictionary for a row or a batch, and a tf.Tensor for a column.
+    """
+    if isinstance(table, Table):
+        pa_table = table.table
+    else:
+        pa_table = table
+    query_type = key_to_query_type(key)
+    python_formatter = PythonFormatter(features=formatter.features)
+    if format_columns is None:
+        return formatter(pa_table, query_type=query_type)
+    elif query_type == "column":
+        if key in format_columns:
+            return formatter(pa_table, query_type)
+        else:
+            return python_formatter(pa_table, query_type=query_type)
+    else:
+        pa_table_to_format = pa_table.drop(col for col in pa_table.column_names if col not in format_columns)
+        formatted_output = formatter(pa_table_to_format, query_type=query_type)
+        if output_all_columns:
+            if isinstance(formatted_output, MutableMapping):
+                pa_table_with_remaining_columns = pa_table.drop(
+                    col for col in pa_table.column_names if col in format_columns
+                )
+                remaining_columns_dict = python_formatter(pa_table_with_remaining_columns, query_type=query_type)
+                formatted_output.update(remaining_columns_dict)
+            else:
+                raise TypeError(
+                    f"Custom formatting function must return a dict to work with output_all_columns=True, but got {formatted_output}"
+                )
+        return formatted_output

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/formatting/jax_formatter.py

@@ -0,0 +1,174 @@
+# Copyright 2021 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import sys
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Optional
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.logging import get_logger
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+if TYPE_CHECKING:
+    import jax
+    import jaxlib
+
+logger = get_logger()
+
+DEVICE_MAPPING: Optional[dict] = None
+
+
+class JaxFormatter(TensorFormatter[Mapping, "jax.Array", Mapping]):
+    def __init__(self, features=None, device=None, token_per_repo_id=None, **jnp_array_kwargs):
+        super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+        import jax
+        from jaxlib.xla_client import Device
+
+        if isinstance(device, Device):
+            raise ValueError(
+                f"Expected {device} to be a `str` not {type(device)}, as `jaxlib.xla_extension.Device` "
+                "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
+                "the device with `str()` to get its string identifier that will be internally mapped "
+                "to the actual `jaxlib.xla_extension.Device`."
+            )
+        self.device = device if isinstance(device, str) else str(jax.devices()[0])
+        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
+        # with `pickle` nor with `dill`, so we need to use a global variable instead
+        global DEVICE_MAPPING
+        if DEVICE_MAPPING is None:
+            DEVICE_MAPPING = self._map_devices_to_str()
+        if self.device not in list(DEVICE_MAPPING.keys()):
+            logger.warning(
+                f"Device with string identifier {self.device} not listed among the available "
+                f"devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default "
+                f"device: {str(jax.devices()[0])}."
+            )
+            self.device = str(jax.devices()[0])
+        self.jnp_array_kwargs = jnp_array_kwargs
+
+    @staticmethod
+    def _map_devices_to_str() -> dict[str, "jaxlib.xla_extension.Device"]:
+        import jax
+
+        return {str(device): device for device in jax.devices()}
+
+    def _consolidate(self, column):
+        import jax
+        import jax.numpy as jnp
+
+        if isinstance(column, list) and column:
+            if all(
+                isinstance(x, jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column
+            ):
+                return jnp.stack(column, axis=0)
+        return column
+
+    def _tensorize(self, value):
+        import jax
+        import jax.numpy as jnp
+
+        if isinstance(value, (str, bytes, type(None))):
+            return value
+        elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character):
+            return value.tolist()
+
+        default_dtype = {}
+
+        if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer):
+            # the default int precision depends on the jax config
+            # see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
+            if jax.config.jax_enable_x64:
+                default_dtype = {"dtype": jnp.int64}
+            else:
+                default_dtype = {"dtype": jnp.int32}
+        elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating):
+            default_dtype = {"dtype": jnp.float32}
+
+        if config.PIL_AVAILABLE and "PIL" in sys.modules:
+            import PIL.Image
+
+            if isinstance(value, PIL.Image.Image):
+                value = np.asarray(value)
+        if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+            try:
+                from torchvision.io import VideoReader
+
+                if isinstance(value, VideoReader):
+                    return value  # TODO(QL): set output to jax arrays ?
+            except ImportError:
+                pass
+        if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+            from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+            if isinstance(value, (VideoDecoder, AudioDecoder)):
+                return value  # TODO(QL): set output to jax arrays ?
+
+        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
+        # with `pickle` nor with `dill`, so we need to use a global variable instead
+        global DEVICE_MAPPING
+        if DEVICE_MAPPING is None:
+            DEVICE_MAPPING = self._map_devices_to_str()
+
+        with jax.default_device(DEVICE_MAPPING[self.device]):
+            # calling jnp.array on a np.ndarray does copy the data
+            # see https://github.com/google/jax/issues/4486
+            return jnp.array(value, **{**default_dtype, **self.jnp_array_kwargs})
+
+    def _recursive_tensorize(self, data_struct):
+        import jax
+
+        # support for torch, tf, jax etc.
+        if config.TORCH_AVAILABLE and "torch" in sys.modules:
+            import torch
+
+            if isinstance(data_struct, torch.Tensor):
+                return self._tensorize(data_struct.detach().cpu().numpy()[()])
+        if hasattr(data_struct, "__array__") and not isinstance(data_struct, jax.Array):
+            data_struct = data_struct.__array__()
+        # support for nested types like struct of list of struct
+        if isinstance(data_struct, np.ndarray):
+            if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
+                return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+        elif isinstance(data_struct, (list, tuple)):
+            return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+        return self._tensorize(data_struct)
+
+    def recursive_tensorize(self, data_struct: dict):
+        return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+    def format_row(self, pa_table: pa.Table) -> Mapping:
+        row = self.numpy_arrow_extractor().extract_row(pa_table)
+        row = self.python_features_decoder.decode_row(row)
+        return self.recursive_tensorize(row)
+
+    def format_column(self, pa_table: pa.Table) -> "jax.Array":
+        column = self.numpy_arrow_extractor().extract_column(pa_table)
+        column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+        column = self.recursive_tensorize(column)
+        column = self._consolidate(column)
+        return column
+
+    def format_batch(self, pa_table: pa.Table) -> Mapping:
+        batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+        batch = self.python_features_decoder.decode_batch(batch)
+        batch = self.recursive_tensorize(batch)
+        for column_name in batch:
+            batch[column_name] = self._consolidate(batch[column_name])
+        return batch

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/formatting/np_formatter.py

@@ -0,0 +1,120 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from collections.abc import Mapping
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+class NumpyFormatter(TensorFormatter[Mapping, np.ndarray, Mapping]):
+    def __init__(self, features=None, token_per_repo_id=None, **np_array_kwargs):
+        super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+        self.np_array_kwargs = np_array_kwargs
+
+    def _consolidate(self, column):
+        if isinstance(column, list):
+            if column and all(
+                isinstance(x, np.ndarray) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column
+            ):
+                return np.stack(column)
+            else:
+                # don't use np.array(column, dtype=object)
+                # since it fails in certain cases
+                # see https://stackoverflow.com/q/51005699
+                out = np.empty(len(column), dtype=object)
+                out[:] = column
+                return out
+        return column
+
+    def _tensorize(self, value):
+        if isinstance(value, (str, bytes, type(None))):
+            return value
+        elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character):
+            return value
+        elif isinstance(value, np.number):
+            return value
+
+        default_dtype = {}
+
+        if isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.integer):
+            default_dtype = {"dtype": np.int64}
+        elif isinstance(value, np.ndarray) and np.issubdtype(value.dtype, np.floating):
+            default_dtype = {"dtype": np.float32}
+
+        if config.PIL_AVAILABLE and "PIL" in sys.modules:
+            import PIL.Image
+
+            if isinstance(value, PIL.Image.Image):
+                return np.asarray(value, **self.np_array_kwargs)
+        if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+            try:
+                from torchvision.io import VideoReader
+
+                if isinstance(value, VideoReader):
+                    return value  # TODO(QL): set output to np arrays ?
+            except ImportError:
+                pass
+        if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+            from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+            if isinstance(value, (VideoDecoder, AudioDecoder)):
+                return value  # TODO(QL): set output to np arrays ?
+
+        return np.asarray(value, **{**default_dtype, **self.np_array_kwargs})
+
+    def _recursive_tensorize(self, data_struct):
+        # support for torch, tf, jax etc.
+        if config.TORCH_AVAILABLE and "torch" in sys.modules:
+            import torch
+
+            if isinstance(data_struct, torch.Tensor):
+                return self._tensorize(data_struct.detach().cpu().numpy()[()])
+        if hasattr(data_struct, "__array__") and not isinstance(data_struct, (np.ndarray, np.character, np.number)):
+            data_struct = data_struct.__array__()
+        # support for nested types like struct of list of struct
+        if isinstance(data_struct, np.ndarray):
+            if data_struct.dtype == object:
+                return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+        if isinstance(data_struct, (list, tuple)):
+            return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+        return self._tensorize(data_struct)
+
+    def recursive_tensorize(self, data_struct: dict):
+        return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+    def format_row(self, pa_table: pa.Table) -> Mapping:
+        row = self.numpy_arrow_extractor().extract_row(pa_table)
+        row = self.python_features_decoder.decode_row(row)
+        return self.recursive_tensorize(row)
+
+    def format_column(self, pa_table: pa.Table) -> np.ndarray:
+        column = self.numpy_arrow_extractor().extract_column(pa_table)
+        column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+        column = self.recursive_tensorize(column)
+        column = self._consolidate(column)
+        return column
+
+    def format_batch(self, pa_table: pa.Table) -> Mapping:
+        batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+        batch = self.python_features_decoder.decode_batch(batch)
+        batch = self.recursive_tensorize(batch)
+        for column_name in batch:
+            batch[column_name] = self._consolidate(batch[column_name])
+        return batch

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/formatting/polars_formatter.py

@@ -0,0 +1,124 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+from functools import partial
+from typing import TYPE_CHECKING, Optional
+
+import pyarrow as pa
+
+from .. import config
+from ..features import Features
+from ..features.features import decode_nested_example
+from ..utils.py_utils import no_op_if_value_is_null
+from .formatting import BaseArrowExtractor, TableFormatter
+
+
+if TYPE_CHECKING:
+    import polars as pl
+
+
+class PolarsArrowExtractor(BaseArrowExtractor["pl.DataFrame", "pl.Series", "pl.DataFrame"]):
+    def extract_row(self, pa_table: pa.Table) -> "pl.DataFrame":
+        if config.POLARS_AVAILABLE:
+            if "polars" not in sys.modules:
+                import polars
+            else:
+                polars = sys.modules["polars"]
+
+            return polars.from_arrow(pa_table.slice(length=1))
+        else:
+            raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+    def extract_column(self, pa_table: pa.Table) -> "pl.Series":
+        if config.POLARS_AVAILABLE:
+            if "polars" not in sys.modules:
+                import polars
+            else:
+                polars = sys.modules["polars"]
+
+            return polars.from_arrow(pa_table.select([0]))[pa_table.column_names[0]]
+        else:
+            raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+    def extract_batch(self, pa_table: pa.Table) -> "pl.DataFrame":
+        if config.POLARS_AVAILABLE:
+            if "polars" not in sys.modules:
+                import polars
+            else:
+                polars = sys.modules["polars"]
+
+            return polars.from_arrow(pa_table)
+        else:
+            raise ValueError("Polars needs to be installed to be able to return Polars dataframes.")
+
+
+class PolarsFeaturesDecoder:
+    def __init__(self, features: Optional[Features]):
+        self.features = features
+        import polars as pl  # noqa: F401 - import pl at initialization
+
+    def decode_row(self, row: "pl.DataFrame") -> "pl.DataFrame":
+        decode = (
+            {
+                column_name: no_op_if_value_is_null(partial(decode_nested_example, feature))
+                for column_name, feature in self.features.items()
+                if self.features._column_requires_decoding[column_name]
+            }
+            if self.features
+            else {}
+        )
+        if decode:
+            row[list(decode.keys())] = row.map_rows(decode)
+        return row
+
+    def decode_column(self, column: "pl.Series", column_name: str) -> "pl.Series":
+        decode = (
+            no_op_if_value_is_null(partial(decode_nested_example, self.features[column_name]))
+            if self.features and column_name in self.features and self.features._column_requires_decoding[column_name]
+            else None
+        )
+        if decode:
+            column = column.map_elements(decode)
+        return column
+
+    def decode_batch(self, batch: "pl.DataFrame") -> "pl.DataFrame":
+        return self.decode_row(batch)
+
+
+class PolarsFormatter(TableFormatter["pl.DataFrame", "pl.Series", "pl.DataFrame"]):
+    table_type = "polars dataframe"
+    column_type = "polars series"
+
+    def __init__(self, features=None, **np_array_kwargs):
+        super().__init__(features=features)
+        self.np_array_kwargs = np_array_kwargs
+        self.polars_arrow_extractor = PolarsArrowExtractor
+        self.polars_features_decoder = PolarsFeaturesDecoder(features)
+        import polars as pl  # noqa: F401 - import pl at initialization
+
+    def format_row(self, pa_table: pa.Table) -> "pl.DataFrame":
+        row = self.polars_arrow_extractor().extract_row(pa_table)
+        row = self.polars_features_decoder.decode_row(row)
+        return row
+
+    def format_column(self, pa_table: pa.Table) -> "pl.Series":
+        column = self.polars_arrow_extractor().extract_column(pa_table)
+        column = self.polars_features_decoder.decode_column(column, pa_table.column_names[0])
+        return column
+
+    def format_batch(self, pa_table: pa.Table) -> "pl.DataFrame":
+        row = self.polars_arrow_extractor().extract_batch(pa_table)
+        row = self.polars_features_decoder.decode_batch(row)
+        return row

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/formatting/tf_formatter.py

@@ -0,0 +1,129 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import sys
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+if TYPE_CHECKING:
+    import tensorflow as tf
+
+
+class TFFormatter(TensorFormatter[Mapping, "tf.Tensor", Mapping]):
+    def __init__(self, features=None, token_per_repo_id=None, **tf_tensor_kwargs):
+        super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+        self.tf_tensor_kwargs = tf_tensor_kwargs
+        import tensorflow as tf  # noqa: F401 - import tf at initialization
+
+    def _consolidate(self, column):
+        import tensorflow as tf
+
+        if isinstance(column, list) and column:
+            if all(
+                isinstance(x, tf.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column
+            ):
+                return tf.stack(column)
+            elif all(
+                isinstance(x, (tf.Tensor, tf.RaggedTensor)) and x.ndim == 1 and x.dtype == column[0].dtype
+                for x in column
+            ):
+                # only rag 1-D tensors, otherwise some dimensions become ragged even though they were consolidated
+                return tf.ragged.stack(column)
+
+        return column
+
+    def _tensorize(self, value):
+        import tensorflow as tf
+
+        if value is None:
+            return value
+
+        default_dtype = {}
+
+        if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer):
+            default_dtype = {"dtype": tf.int64}
+        elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating):
+            default_dtype = {"dtype": tf.float32}
+
+        if config.PIL_AVAILABLE and "PIL" in sys.modules:
+            import PIL.Image
+
+            if isinstance(value, PIL.Image.Image):
+                value = np.asarray(value)
+        if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+            try:
+                from torchvision.io import VideoReader
+
+                if isinstance(value, VideoReader):
+                    return value  # TODO(QL): set output to tf tensors ?
+            except ImportError:
+                pass
+        if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+            from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+            if isinstance(value, (VideoDecoder, AudioDecoder)):
+                return value  # TODO(QL): set output to jax arrays ?
+
+        return tf.convert_to_tensor(value, **{**default_dtype, **self.tf_tensor_kwargs})
+
+    def _recursive_tensorize(self, data_struct):
+        import tensorflow as tf
+
+        # support for torch, tf, jax etc.
+        if config.TORCH_AVAILABLE and "torch" in sys.modules:
+            import torch
+
+            if isinstance(data_struct, torch.Tensor):
+                return self._tensorize(data_struct.detach().cpu().numpy()[()])
+        if hasattr(data_struct, "__array__") and not isinstance(data_struct, tf.Tensor):
+            data_struct = data_struct.__array__()
+        # support for nested types like struct of list of struct
+        if isinstance(data_struct, np.ndarray):
+            if data_struct.dtype == object:  # tf tensors cannot be instantied from an array of objects
+                return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+        elif isinstance(data_struct, (list, tuple)):
+            return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+        return self._tensorize(data_struct)
+
+    def recursive_tensorize(self, data_struct: dict):
+        return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+    def format_row(self, pa_table: pa.Table) -> Mapping:
+        row = self.numpy_arrow_extractor().extract_row(pa_table)
+        row = self.python_features_decoder.decode_row(row)
+        return self.recursive_tensorize(row)
+
+    def format_column(self, pa_table: pa.Table) -> "tf.Tensor":
+        column = self.numpy_arrow_extractor().extract_column(pa_table)
+        column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+        column = self.recursive_tensorize(column)
+        column = self._consolidate(column)
+        return column
+
+    def format_batch(self, pa_table: pa.Table) -> Mapping:
+        batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+        batch = self.python_features_decoder.decode_batch(batch)
+        batch = self.recursive_tensorize(batch)
+        for column_name in batch:
+            batch[column_name] = self._consolidate(batch[column_name])
+        return batch

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/formatting/torch_formatter.py

@@ -0,0 +1,130 @@
+# Copyright 2020 The HuggingFace Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Lint as: python3
+import sys
+from collections.abc import Mapping
+from typing import TYPE_CHECKING
+
+import numpy as np
+import pyarrow as pa
+
+from .. import config
+from ..utils.py_utils import map_nested
+from .formatting import TensorFormatter
+
+
+if TYPE_CHECKING:
+    import torch
+
+
+class TorchFormatter(TensorFormatter[Mapping, "torch.Tensor", Mapping]):
+    def __init__(self, features=None, token_per_repo_id=None, **torch_tensor_kwargs):
+        super().__init__(features=features, token_per_repo_id=token_per_repo_id)
+        self.torch_tensor_kwargs = torch_tensor_kwargs
+        import torch  # noqa import torch at initialization
+
+    def _consolidate(self, column):
+        import torch
+
+        if isinstance(column, list) and column:
+            if all(
+                isinstance(x, torch.Tensor) and x.shape == column[0].shape and x.dtype == column[0].dtype
+                for x in column
+            ):
+                return torch.stack(column)
+        return column
+
+    def _tensorize(self, value):
+        import torch
+
+        if isinstance(value, (str, bytes, type(None))):
+            return value
+        elif isinstance(value, (np.character, np.ndarray)) and np.issubdtype(value.dtype, np.character):
+            return value.tolist()
+
+        default_dtype = {}
+
+        if isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.integer):
+            default_dtype = {"dtype": torch.int64}
+
+            # Convert dtype to np.int64 if it's either np.uint16 or np.uint32 to ensure compatibility.
+            # np.uint64 is excluded from this conversion as there is no compatible PyTorch dtype that can handle it without loss.
+            if value.dtype in [np.uint16, np.uint32]:
+                value = value.astype(np.int64)
+
+        elif isinstance(value, (np.number, np.ndarray)) and np.issubdtype(value.dtype, np.floating):
+            default_dtype = {"dtype": torch.float32}
+
+        if config.PIL_AVAILABLE and "PIL" in sys.modules:
+            import PIL.Image
+
+            if isinstance(value, PIL.Image.Image):
+                value = np.asarray(value)
+                if value.ndim == 2:
+                    value = value[:, :, np.newaxis]
+
+                value = value.transpose((2, 0, 1))
+        if config.TORCHVISION_AVAILABLE and "torchvision" in sys.modules:
+            try:
+                from torchvision.io import VideoReader
+
+                if isinstance(value, VideoReader):
+                    return value  # TODO(QL): set output to torch tensors ?
+            except ImportError:
+                pass
+        if config.TORCHCODEC_AVAILABLE and "torchcodec" in sys.modules:
+            from torchcodec.decoders import AudioDecoder, VideoDecoder
+
+            if isinstance(value, (VideoDecoder, AudioDecoder)):
+                return value  # TODO(QL): set output to jax arrays ?
+
+        return torch.tensor(value, **{**default_dtype, **self.torch_tensor_kwargs})
+
+    def _recursive_tensorize(self, data_struct):
+        import torch
+
+        # support for torch, tf, jax etc.
+        if hasattr(data_struct, "__array__") and not isinstance(data_struct, torch.Tensor):
+            data_struct = data_struct.__array__()
+        # support for nested types like struct of list of struct
+        if isinstance(data_struct, np.ndarray):
+            if data_struct.dtype == object:  # torch tensors cannot be instantied from an array of objects
+                return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+        elif isinstance(data_struct, (list, tuple)):
+            return self._consolidate([self.recursive_tensorize(substruct) for substruct in data_struct])
+        return self._tensorize(data_struct)
+
+    def recursive_tensorize(self, data_struct: dict):
+        return map_nested(self._recursive_tensorize, data_struct, map_list=False)
+
+    def format_row(self, pa_table: pa.Table) -> Mapping:
+        row = self.numpy_arrow_extractor().extract_row(pa_table)
+        row = self.python_features_decoder.decode_row(row)
+        return self.recursive_tensorize(row)
+
+    def format_column(self, pa_table: pa.Table) -> "torch.Tensor":
+        column = self.numpy_arrow_extractor().extract_column(pa_table)
+        column = self.python_features_decoder.decode_column(column, pa_table.column_names[0])
+        column = self.recursive_tensorize(column)
+        column = self._consolidate(column)
+        return column
+
+    def format_batch(self, pa_table: pa.Table) -> Mapping:
+        batch = self.numpy_arrow_extractor().extract_batch(pa_table)
+        batch = self.python_features_decoder.decode_batch(batch)
+        batch = self.recursive_tensorize(batch)
+        for column_name in batch:
+            batch[column_name] = self._consolidate(batch[column_name])
+        return batch

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/abc.py

@@ -0,0 +1,53 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Union
+
+from .. import Dataset, DatasetDict, Features, IterableDataset, IterableDatasetDict, NamedSplit
+from ..utils.typing import NestedDataStructureLike, PathLike
+
+
+class AbstractDatasetReader(ABC):
+    def __init__(
+        self,
+        path_or_paths: Optional[NestedDataStructureLike[PathLike]] = None,
+        split: Optional[NamedSplit] = None,
+        features: Optional[Features] = None,
+        cache_dir: str = None,
+        keep_in_memory: bool = False,
+        streaming: bool = False,
+        num_proc: Optional[int] = None,
+        **kwargs,
+    ):
+        self.path_or_paths = path_or_paths
+        self.split = split if split or isinstance(path_or_paths, dict) else "train"
+        self.features = features
+        self.cache_dir = cache_dir
+        self.keep_in_memory = keep_in_memory
+        self.streaming = streaming
+        self.num_proc = num_proc
+        self.kwargs = kwargs
+
+    @abstractmethod
+    def read(self) -> Union[Dataset, DatasetDict, IterableDataset, IterableDatasetDict]:
+        pass
+
+
+class AbstractDatasetInputStream(ABC):
+    def __init__(
+        self,
+        features: Optional[Features] = None,
+        cache_dir: str = None,
+        keep_in_memory: bool = False,
+        streaming: bool = False,
+        num_proc: Optional[int] = None,
+        **kwargs,
+    ):
+        self.features = features
+        self.cache_dir = cache_dir
+        self.keep_in_memory = keep_in_memory
+        self.streaming = streaming
+        self.num_proc = num_proc
+        self.kwargs = kwargs
+
+    @abstractmethod
+    def read(self) -> Union[Dataset, IterableDataset]:
+        pass

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/generator.py

@@ -0,0 +1,62 @@
+from typing import Callable, Optional
+
+from .. import Features, NamedSplit, Split
+from ..packaged_modules.generator.generator import Generator
+from .abc import AbstractDatasetInputStream
+
+
+class GeneratorDatasetInputStream(AbstractDatasetInputStream):
+    def __init__(
+        self,
+        generator: Callable,
+        features: Optional[Features] = None,
+        cache_dir: str = None,
+        keep_in_memory: bool = False,
+        streaming: bool = False,
+        gen_kwargs: Optional[dict] = None,
+        num_proc: Optional[int] = None,
+        split: NamedSplit = Split.TRAIN,
+        fingerprint: Optional[str] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            features=features,
+            cache_dir=cache_dir,
+            keep_in_memory=keep_in_memory,
+            streaming=streaming,
+            num_proc=num_proc,
+            **kwargs,
+        )
+        self.builder = Generator(
+            cache_dir=cache_dir,
+            features=features,
+            generator=generator,
+            gen_kwargs=gen_kwargs,
+            split=split,
+            config_id="default-fingerprint=" + fingerprint if fingerprint else None,
+            **kwargs,
+        )
+        self.fingerprint = fingerprint
+
+    def read(self):
+        # Build iterable dataset
+        if self.streaming:
+            dataset = self.builder.as_streaming_dataset(split=self.builder.config.split)
+        # Build regular (map-style) dataset
+        else:
+            download_config = None
+            download_mode = None
+            verification_mode = None
+            base_path = None
+
+            self.builder.download_and_prepare(
+                download_config=download_config,
+                download_mode=download_mode,
+                verification_mode=verification_mode,
+                base_path=base_path,
+                num_proc=self.num_proc,
+            )
+            dataset = self.builder.as_dataset(split=self.builder.config.split, in_memory=self.keep_in_memory)
+            if self.fingerprint:
+                dataset._fingerprint = self.fingerprint
+        return dataset

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/parquet.py

@@ -0,0 +1,159 @@
+import json
+import os
+from typing import BinaryIO, Optional, Union
+
+import fsspec
+import pyarrow.parquet as pq
+
+from .. import Dataset, Features, NamedSplit, config
+from ..arrow_writer import get_writer_batch_size_from_data_size, get_writer_batch_size_from_features
+from ..features.features import require_storage_embed
+from ..formatting import query_table
+from ..packaged_modules import _PACKAGED_DATASETS_MODULES
+from ..packaged_modules.parquet.parquet import Parquet
+from ..utils import tqdm as hf_tqdm
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class ParquetDatasetReader(AbstractDatasetReader):
+    def __init__(
+        self,
+        path_or_paths: NestedDataStructureLike[PathLike],
+        split: Optional[NamedSplit] = None,
+        features: Optional[Features] = None,
+        cache_dir: str = None,
+        keep_in_memory: bool = False,
+        streaming: bool = False,
+        num_proc: Optional[int] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            path_or_paths,
+            split=split,
+            features=features,
+            cache_dir=cache_dir,
+            keep_in_memory=keep_in_memory,
+            streaming=streaming,
+            num_proc=num_proc,
+            **kwargs,
+        )
+        path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+        hash = _PACKAGED_DATASETS_MODULES["parquet"][1]
+        self.builder = Parquet(
+            cache_dir=cache_dir,
+            data_files=path_or_paths,
+            features=features,
+            hash=hash,
+            **kwargs,
+        )
+
+    def read(self):
+        # Build iterable dataset
+        if self.streaming:
+            dataset = self.builder.as_streaming_dataset(split=self.split)
+        # Build regular (map-style) dataset
+        else:
+            download_config = None
+            download_mode = None
+            verification_mode = None
+            base_path = None
+
+            self.builder.download_and_prepare(
+                download_config=download_config,
+                download_mode=download_mode,
+                verification_mode=verification_mode,
+                base_path=base_path,
+                num_proc=self.num_proc,
+            )
+            dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
+        return dataset
+
+
+class ParquetDatasetWriter:
+    def __init__(
+        self,
+        dataset: Dataset,
+        path_or_buf: Union[PathLike, BinaryIO],
+        batch_size: Optional[int] = None,
+        storage_options: Optional[dict] = None,
+        use_content_defined_chunking: Union[bool, dict] = True,
+        write_page_index: bool = True,
+        **parquet_writer_kwargs,
+    ):
+        self.dataset = dataset
+        self.path_or_buf = path_or_buf
+        self.batch_size = (
+            batch_size
+            or get_writer_batch_size_from_features(dataset.features)
+            or get_writer_batch_size_from_data_size(len(dataset), dataset._estimate_nbytes())
+        )
+        self.storage_options = storage_options or {}
+        self.parquet_writer_kwargs = parquet_writer_kwargs
+        if use_content_defined_chunking is True:
+            use_content_defined_chunking = config.DEFAULT_CDC_OPTIONS
+        self.use_content_defined_chunking = use_content_defined_chunking
+        self.write_page_index = write_page_index
+
+    def write(self) -> int:
+        if isinstance(self.path_or_buf, (str, bytes, os.PathLike)):
+            with fsspec.open(self.path_or_buf, "wb", **(self.storage_options or {})) as buffer:
+                written = self._write(
+                    file_obj=buffer,
+                    batch_size=self.batch_size,
+                    **self.parquet_writer_kwargs,
+                )
+        else:
+            written = self._write(
+                file_obj=self.path_or_buf,
+                batch_size=self.batch_size,
+                **self.parquet_writer_kwargs,
+            )
+        return written
+
+    def _write(self, file_obj: BinaryIO, batch_size: int, **parquet_writer_kwargs) -> int:
+        """Writes the pyarrow table as Parquet to a binary file handle.
+
+        Caller is responsible for opening and closing the handle.
+        """
+        written = 0
+        _ = parquet_writer_kwargs.pop("path_or_buf", None)
+        schema = self.dataset.features.arrow_schema
+
+        writer = pq.ParquetWriter(
+            file_obj,
+            schema=schema,
+            use_content_defined_chunking=self.use_content_defined_chunking,
+            write_page_index=self.write_page_index,
+            compression={
+                col: "none" if require_storage_embed(feature) else "snappy"
+                for col, feature in self.dataset.features.items()
+            },
+            use_dictionary=[
+                col for col, feature in self.dataset.features.items() if not require_storage_embed(feature)
+            ],
+            column_encoding={
+                col: "PLAIN" for col, feature in self.dataset.features.items() if require_storage_embed(feature)
+            },
+            **parquet_writer_kwargs,
+        )
+
+        for offset in hf_tqdm(
+            range(0, len(self.dataset), batch_size),
+            unit="ba",
+            desc="Creating parquet from Arrow format",
+        ):
+            batch = query_table(
+                table=self.dataset._data,
+                key=slice(offset, offset + batch_size),
+                indices=self.dataset._indices,
+            )
+            writer.write_table(batch)
+            written += batch.nbytes
+
+        # TODO(kszucs): we may want to persist multiple parameters
+        if self.use_content_defined_chunking is not False:
+            writer.add_key_value_metadata({"content_defined_chunking": json.dumps(self.use_content_defined_chunking)})
+
+        writer.close()
+        return written

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/spark.py

@@ -0,0 +1,57 @@
+from typing import Optional
+
+import pyspark
+
+from .. import Features, NamedSplit
+from ..download import DownloadMode
+from ..packaged_modules.spark.spark import Spark
+from .abc import AbstractDatasetReader
+
+
+class SparkDatasetReader(AbstractDatasetReader):
+    """A dataset reader that reads from a Spark DataFrame.
+
+    When caching, cache materialization is parallelized over Spark; an NFS that is accessible to the driver must be
+    provided. Streaming is not currently supported.
+    """
+
+    def __init__(
+        self,
+        df: pyspark.sql.DataFrame,
+        split: Optional[NamedSplit] = None,
+        features: Optional[Features] = None,
+        streaming: bool = True,
+        cache_dir: str = None,
+        keep_in_memory: bool = False,
+        working_dir: str = None,
+        load_from_cache_file: bool = True,
+        file_format: str = "arrow",
+        **kwargs,
+    ):
+        super().__init__(
+            split=split,
+            features=features,
+            cache_dir=cache_dir,
+            keep_in_memory=keep_in_memory,
+            streaming=streaming,
+            **kwargs,
+        )
+        self._load_from_cache_file = load_from_cache_file
+        self._file_format = file_format
+        self.builder = Spark(
+            df=df,
+            features=features,
+            cache_dir=cache_dir,
+            working_dir=working_dir,
+            **kwargs,
+        )
+
+    def read(self):
+        if self.streaming:
+            return self.builder.as_streaming_dataset(split=self.split)
+        download_mode = None if self._load_from_cache_file else DownloadMode.FORCE_REDOWNLOAD
+        self.builder.download_and_prepare(
+            download_mode=download_mode,
+            file_format=self._file_format,
+        )
+        return self.builder.as_dataset(split=self.split)

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/io/text.py

@@ -0,0 +1,58 @@
+from typing import Optional
+
+from .. import Features, NamedSplit
+from ..packaged_modules.text.text import Text
+from ..utils.typing import NestedDataStructureLike, PathLike
+from .abc import AbstractDatasetReader
+
+
+class TextDatasetReader(AbstractDatasetReader):
+    def __init__(
+        self,
+        path_or_paths: NestedDataStructureLike[PathLike],
+        split: Optional[NamedSplit] = None,
+        features: Optional[Features] = None,
+        cache_dir: str = None,
+        keep_in_memory: bool = False,
+        streaming: bool = False,
+        num_proc: Optional[int] = None,
+        **kwargs,
+    ):
+        super().__init__(
+            path_or_paths,
+            split=split,
+            features=features,
+            cache_dir=cache_dir,
+            keep_in_memory=keep_in_memory,
+            streaming=streaming,
+            num_proc=num_proc,
+            **kwargs,
+        )
+        path_or_paths = path_or_paths if isinstance(path_or_paths, dict) else {self.split: path_or_paths}
+        self.builder = Text(
+            cache_dir=cache_dir,
+            data_files=path_or_paths,
+            features=features,
+            **kwargs,
+        )
+
+    def read(self):
+        # Build iterable dataset
+        if self.streaming:
+            dataset = self.builder.as_streaming_dataset(split=self.split)
+        # Build regular (map-style) dataset
+        else:
+            download_config = None
+            download_mode = None
+            verification_mode = None
+            base_path = None
+
+            self.builder.download_and_prepare(
+                download_config=download_config,
+                download_mode=download_mode,
+                verification_mode=verification_mode,
+                base_path=base_path,
+                num_proc=self.num_proc,
+            )
+            dataset = self.builder.as_dataset(split=self.split, in_memory=self.keep_in_memory)
+        return dataset

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/arrow/arrow.py

@@ -0,0 +1,77 @@
+from dataclasses import dataclass
+from typing import Optional
+
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class ArrowConfig(datasets.BuilderConfig):
+    """BuilderConfig for Arrow."""
+
+    features: Optional[datasets.Features] = None
+
+    def __post_init__(self):
+        super().__post_init__()
+
+
+class Arrow(datasets.ArrowBasedBuilder):
+    BUILDER_CONFIG_CLASS = ArrowConfig
+
+    def _info(self):
+        return datasets.DatasetInfo(features=self.config.features)
+
+    def _split_generators(self, dl_manager):
+        """We handle string, list and dicts in datafiles"""
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+        data_files = dl_manager.download(self.config.data_files)
+        splits = []
+        for split_name, files in data_files.items():
+            # Infer features if they are stored in the arrow schema
+            if self.info.features is None:
+                for file in files:
+                    with open(file, "rb") as f:
+                        try:
+                            reader = pa.ipc.open_stream(f)
+                        except (OSError, pa.lib.ArrowInvalid):
+                            reader = pa.ipc.open_file(f)
+                    self.info.features = datasets.Features.from_arrow_schema(reader.schema)
+                    break
+            splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+        return splits
+
+    def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+        if self.info.features is not None:
+            # more expensive cast to support nested features with keys in a different order
+            # allows str <-> int/float or str to Audio for example
+            pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+        return pa_table
+
+    def _generate_shards(self, files):
+        yield from files
+
+    def _generate_tables(self, files):
+        for file_idx, file in enumerate(files):
+            with open(file, "rb") as f:
+                try:
+                    try:
+                        batches = pa.ipc.open_stream(f)
+                    except (OSError, pa.lib.ArrowInvalid):
+                        reader = pa.ipc.open_file(f)
+                        batches = (reader.get_batch(i) for i in range(reader.num_record_batches))
+                    for batch_idx, record_batch in enumerate(batches):
+                        pa_table = pa.Table.from_batches([record_batch])
+                        # Uncomment for debugging (will print the Arrow table size and elements)
+                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+                        yield Key(file_idx, batch_idx), self._cast_table(pa_table)
+                except ValueError as e:
+                    logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+                    raise

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/audiofolder/audiofolder.py

@@ -0,0 +1,86 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class AudioFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+    """Builder Config for AudioFolder."""
+
+    drop_labels: bool = None
+    drop_metadata: bool = None
+
+    def __post_init__(self):
+        super().__post_init__()
+
+
+class AudioFolder(folder_based_builder.FolderBasedBuilder):
+    BASE_FEATURE = datasets.Audio
+    BASE_COLUMN_NAME = "audio"
+    BUILDER_CONFIG_CLASS = AudioFolderConfig
+    EXTENSIONS: list[str]  # definition at the bottom of the script
+
+
+# Obtained with:
+# ```
+# import soundfile as sf
+#
+# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()]
+#
+# # .opus decoding is supported if libsndfile >= 1.0.31:
+# AUDIO_EXTENSIONS.extend([".opus"])
+# ```
+# We intentionally did not run this code on launch because:
+# (1) Soundfile was an optional dependency, so importing it in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+# (3) We use TorchCodec now anyways instead of Soundfile
+AUDIO_EXTENSIONS = [
+    ".aiff",
+    ".au",
+    ".avr",
+    ".caf",
+    ".flac",
+    ".htk",
+    ".svx",
+    ".mat4",
+    ".mat5",
+    ".mpc2k",
+    ".ogg",
+    ".paf",
+    ".pvf",
+    ".raw",
+    ".rf64",
+    ".sd2",
+    ".sds",
+    ".ircam",
+    ".voc",
+    ".w64",
+    ".wav",
+    ".nist",
+    ".wavex",
+    ".wve",
+    ".xi",
+    ".mp3",
+    ".opus",
+    ".3gp",
+    ".3g2",
+    ".avi",
+    ".asf",
+    ".flv",
+    ".mp4",
+    ".mov",
+    ".m4v",
+    ".mkv",
+    ".mpg",
+    ".webm",
+    ".f4v",
+    ".wmv",
+    ".wma",
+    ".ogg",
+    ".ogm",
+    ".mxf",
+    ".nut",
+]
+AudioFolder.EXTENSIONS = AUDIO_EXTENSIONS

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/cache/cache.py

@@ -0,0 +1,195 @@
+import glob
+import json
+import os
+import shutil
+import time
+from pathlib import Path
+from typing import Optional, Union
+
+import pyarrow as pa
+
+import datasets
+import datasets.config
+import datasets.data_files
+from datasets.builder import Key
+from datasets.naming import camelcase_to_snakecase, filenames_for_dataset_split
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+def _get_modification_time(cached_directory_path):
+    return (Path(cached_directory_path)).stat().st_mtime
+
+
+def _find_hash_in_cache(
+    dataset_name: str,
+    config_name: Optional[str],
+    cache_dir: Optional[str],
+    config_kwargs: dict,
+    custom_features: Optional[datasets.Features],
+) -> tuple[str, str, str]:
+    if config_name or config_kwargs or custom_features:
+        config_id = datasets.BuilderConfig(config_name or "default").create_config_id(
+            config_kwargs=config_kwargs, custom_features=custom_features
+        )
+    else:
+        config_id = None
+    cache_dir = os.path.expanduser(str(cache_dir or datasets.config.HF_DATASETS_CACHE))
+    namespace_and_dataset_name = dataset_name.split("/")
+    namespace_and_dataset_name[-1] = camelcase_to_snakecase(namespace_and_dataset_name[-1])
+    cached_relative_path = "___".join(namespace_and_dataset_name)
+    cached_datasets_directory_path_root = os.path.join(cache_dir, cached_relative_path)
+    cached_directory_paths = [
+        cached_directory_path
+        for cached_directory_path in glob.glob(
+            os.path.join(cached_datasets_directory_path_root, config_id or "*", "*", "*")
+        )
+        if os.path.isdir(cached_directory_path)
+        and (
+            config_kwargs
+            or custom_features
+            or json.loads(Path(cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"]
+            == Path(cached_directory_path).parts[-3]  # no extra params => config_id == config_name
+        )
+    ]
+    if not cached_directory_paths:
+        cached_directory_paths = [
+            cached_directory_path
+            for cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", "*", "*"))
+            if os.path.isdir(cached_directory_path)
+        ]
+        available_configs = sorted(
+            {Path(cached_directory_path).parts[-3] for cached_directory_path in cached_directory_paths}
+        )
+        raise ValueError(
+            f"Couldn't find cache for {dataset_name}"
+            + (f" for config '{config_id}'" if config_id else "")
+            + (f"\nAvailable configs in the cache: {available_configs}" if available_configs else "")
+        )
+    # get most recent
+    cached_directory_path = Path(sorted(cached_directory_paths, key=_get_modification_time)[-1])
+    version, hash = cached_directory_path.parts[-2:]
+    other_configs = [
+        Path(_cached_directory_path).parts[-3]
+        for _cached_directory_path in glob.glob(os.path.join(cached_datasets_directory_path_root, "*", version, hash))
+        if os.path.isdir(_cached_directory_path)
+        and (
+            config_kwargs
+            or custom_features
+            or json.loads(Path(_cached_directory_path, "dataset_info.json").read_text(encoding="utf-8"))["config_name"]
+            == Path(_cached_directory_path).parts[-3]  # no extra params => config_id == config_name
+        )
+    ]
+    if not config_id and len(other_configs) > 1:
+        raise ValueError(
+            f"There are multiple '{dataset_name}' configurations in the cache: {', '.join(other_configs)}"
+            f"\nPlease specify which configuration to reload from the cache, e.g."
+            f"\n\tload_dataset('{dataset_name}', '{other_configs[0]}')"
+        )
+    config_name = cached_directory_path.parts[-3]
+    warning_msg = (
+        f"Found the latest cached dataset configuration '{config_name}' at {cached_directory_path} "
+        f"(last modified on {time.ctime(_get_modification_time(cached_directory_path))})."
+    )
+    logger.warning(warning_msg)
+    return config_name, version, hash
+
+
+class Cache(datasets.ArrowBasedBuilder):
+    def __init__(
+        self,
+        cache_dir: Optional[str] = None,
+        dataset_name: Optional[str] = None,
+        config_name: Optional[str] = None,
+        version: Optional[str] = "0.0.0",
+        hash: Optional[str] = None,
+        base_path: Optional[str] = None,
+        info: Optional[datasets.DatasetInfo] = None,
+        features: Optional[datasets.Features] = None,
+        token: Optional[Union[bool, str]] = None,
+        repo_id: Optional[str] = None,
+        data_files: Optional[Union[str, list, dict, datasets.data_files.DataFilesDict]] = None,
+        data_dir: Optional[str] = None,
+        storage_options: Optional[dict] = None,
+        writer_batch_size: Optional[int] = None,
+        **config_kwargs,
+    ):
+        if repo_id is None and dataset_name is None:
+            raise ValueError("repo_id or dataset_name is required for the Cache dataset builder")
+        if data_files is not None:
+            config_kwargs["data_files"] = data_files
+        if data_dir is not None:
+            config_kwargs["data_dir"] = data_dir
+        if hash == "auto" and version == "auto":
+            config_name, version, hash = _find_hash_in_cache(
+                dataset_name=repo_id or dataset_name,
+                config_name=config_name,
+                cache_dir=cache_dir,
+                config_kwargs=config_kwargs,
+                custom_features=features,
+            )
+        elif hash == "auto" or version == "auto":
+            raise NotImplementedError("Pass both hash='auto' and version='auto' instead")
+        super().__init__(
+            cache_dir=cache_dir,
+            dataset_name=dataset_name,
+            config_name=config_name,
+            version=version,
+            hash=hash,
+            base_path=base_path,
+            info=info,
+            token=token,
+            repo_id=repo_id,
+            storage_options=storage_options,
+            writer_batch_size=writer_batch_size,
+        )
+
+    def _info(self) -> datasets.DatasetInfo:
+        return datasets.DatasetInfo()
+
+    def download_and_prepare(self, output_dir: Optional[str] = None, *args, **kwargs):
+        if not os.path.exists(self.cache_dir):
+            raise ValueError(f"Cache directory for {self.dataset_name} doesn't exist at {self.cache_dir}")
+        if output_dir is not None and output_dir != self.cache_dir:
+            shutil.copytree(self.cache_dir, output_dir)
+
+    def _split_generators(self, dl_manager):
+        # used to stream from cache
+        if isinstance(self.info.splits, datasets.SplitDict):
+            split_infos: list[datasets.SplitInfo] = list(self.info.splits.values())
+        else:
+            raise ValueError(f"Missing splits info for {self.dataset_name} in cache directory {self.cache_dir}")
+        return [
+            datasets.SplitGenerator(
+                name=split_info.name,
+                gen_kwargs={
+                    "files": filenames_for_dataset_split(
+                        self.cache_dir,
+                        dataset_name=self.dataset_name,
+                        split=split_info.name,
+                        filetype_suffix="arrow",
+                        shard_lengths=split_info.shard_lengths,
+                    )
+                },
+            )
+            for split_info in split_infos
+        ]
+
+    def _generate_shards(self, files):
+        yield from files
+
+    def _generate_tables(self, files):
+        # used to stream from cache
+        for file_idx, file in enumerate(files):
+            with open(file, "rb") as f:
+                try:
+                    for batch_idx, record_batch in enumerate(pa.ipc.open_stream(f)):
+                        pa_table = pa.Table.from_batches([record_batch])
+                        # Uncomment for debugging (will print the Arrow table size and elements)
+                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+                        yield Key(file_idx, batch_idx), pa_table
+                except ValueError as e:
+                    logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+                    raise

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/eval/eval.py

@@ -0,0 +1,77 @@
+import json
+import os
+from itertools import islice
+from typing import Iterable
+
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class Eval(datasets.GeneratorBasedBuilder):
+    NUM_EXAMPLES_FOR_FEATURES_INFERENCE = 5
+
+    def _info(self):
+        return datasets.DatasetInfo()
+
+    def _split_generators(self, dl_manager):
+        """We handle string, list and dicts in datafiles"""
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+        dl_manager.download_config.extract_on_the_fly = True
+        base_data_files = dl_manager.download(self.config.data_files)
+        extracted_data_files = dl_manager.extract(base_data_files)
+        splits = []
+        for split_name, logs in extracted_data_files.items():
+            logs_files_iterables = [dl_manager.iter_files(log) for log in logs]
+            splits.append(
+                datasets.SplitGenerator(
+                    name=split_name,
+                    gen_kwargs={
+                        "logs_files_iterables": logs_files_iterables,
+                        "base_files": base_data_files[split_name],
+                    },
+                )
+            )
+        if not self.info.features:
+            first_examples = list(
+                islice(
+                    self._iter_samples_from_log_files(logs_files_iterables[0]),
+                    self.NUM_EXAMPLES_FOR_FEATURES_INFERENCE,
+                )
+            )
+            pa_tables = [pa.Table.from_pylist([example]) for example in first_examples]
+            inferred_arrow_schema = pa.concat_tables(pa_tables, promote_options="default").schema
+            self.info.features = datasets.Features.from_arrow_schema(inferred_arrow_schema)
+
+        return splits
+
+    def _sort_samples_key(self, sample_path: str):
+        # looks like "{sample_idx}_epoch_{epoch_idx}""
+        (sample_idx_str, epoch_idx_str) = os.path.splitext(os.path.basename(sample_path))[0].split("_epoch_")
+        return (int(epoch_idx_str), int(sample_idx_str))
+
+    def _iter_samples_from_log_files(self, log_files: Iterable[str]):
+        sample_files = [log_file for log_file in log_files if os.path.basename(os.path.dirname(log_file)) == "samples"]
+        sample_files.sort(key=self._sort_samples_key)
+        for sample_file in sample_files:
+            with open(sample_file) as f:
+                sample = json.load(f)
+                for field in sample:
+                    if isinstance(sample[field], dict):
+                        sample[field] = json.dumps(sample[field])
+                    if isinstance(sample[field], list):
+                        sample[field] = [json.dumps(x) for x in sample[field]]
+                yield sample
+
+    def _generate_shards(self, base_files, logs_files_iterables):
+        yield from base_files
+
+    def _generate_examples(self, base_files, logs_files_iterables):
+        for file_idx, log_files in enumerate(logs_files_iterables):
+            for sample_idx, sample in enumerate(self._iter_samples_from_log_files(log_files)):
+                yield Key(file_idx, sample_idx), sample

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/hdf5/hdf5.py

@@ -0,0 +1,389 @@
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Optional
+
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.features.features import (
+    Array2D,
+    Array3D,
+    Array4D,
+    Array5D,
+    Features,
+    LargeList,
+    List,
+    Value,
+    _ArrayXD,
+    _arrow_to_datasets_dtype,
+)
+from datasets.table import cast_table_to_features
+
+
+if TYPE_CHECKING:
+    import h5py
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+EXTENSIONS = [".h5", ".hdf5"]
+
+
+@dataclass
+class HDF5Config(datasets.BuilderConfig):
+    """BuilderConfig for HDF5."""
+
+    batch_size: Optional[int] = None
+    features: Optional[datasets.Features] = None
+
+
+class HDF5(datasets.ArrowBasedBuilder):
+    """ArrowBasedBuilder that converts HDF5 files to Arrow tables using the HF extension types."""
+
+    BUILDER_CONFIG_CLASS = HDF5Config
+
+    def _info(self):
+        return datasets.DatasetInfo(features=self.config.features)
+
+    def _split_generators(self, dl_manager):
+        import h5py
+
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+        data_files = dl_manager.download(self.config.data_files)
+        splits = []
+        for split_name, files in data_files.items():
+            # Infer features from first file
+            if self.info.features is None:
+                for first_file in files:
+                    with open(first_file, "rb") as f:
+                        with h5py.File(f, "r") as h5:
+                            self.info.features = _recursive_infer_features(h5)
+                    break
+            splits.append(datasets.SplitGenerator(name=split_name, gen_kwargs={"files": files}))
+        return splits
+
+    def _generate_shards(self, files):
+        yield from files
+
+    def _generate_tables(self, files):
+        import h5py
+
+        batch_size_cfg = self.config.batch_size
+        for file_idx, file in enumerate(files):
+            try:
+                with open(file, "rb") as f:
+                    with h5py.File(f, "r") as h5:
+                        # Infer features and lengths from first file
+                        if self.info.features is None:
+                            self.info.features = _recursive_infer_features(h5)
+                        num_rows = _check_dataset_lengths(h5, self.info.features)
+                        if num_rows is None:
+                            logger.warning(f"File {file} contains no data, skipping...")
+                            continue
+                        effective_batch = batch_size_cfg or self._writer_batch_size or num_rows
+                        for batch_idx, start in enumerate(range(0, num_rows, effective_batch)):
+                            end = min(start + effective_batch, num_rows)
+                            pa_table = _recursive_load_arrays(h5, self.info.features, start, end)
+                            if pa_table is None:
+                                logger.warning(f"File {file} contains no data, skipping...")
+                                continue
+                            yield Key(file_idx, batch_idx), cast_table_to_features(pa_table, self.info.features)
+            except ValueError as e:
+                logger.error(f"Failed to read file '{file}' with error {type(e)}: {e}")
+                raise
+
+
+# ┌───────────┐
+# │  Complex  │
+# └───────────┘
+
+
+def _is_complex_dtype(dtype: np.dtype) -> bool:
+    if dtype.kind == "c":
+        return True
+    if dtype.subdtype is not None:
+        return _is_complex_dtype(dtype.subdtype[0])
+    return False
+
+
+def _create_complex_features(dset) -> Features:
+    if dset.dtype.subdtype is not None:
+        dtype, data_shape = dset.dtype.subdtype
+    else:
+        data_shape = dset.shape[1:]
+        dtype = dset.dtype
+
+    if dtype == np.complex64:
+        # two float32s
+        value_type = Value("float32")
+    elif dtype == np.complex128:
+        # two float64s
+        value_type = Value("float64")
+    else:
+        logger.warning(f"Found complex dtype {dtype} that is not supported. Converting to float64...")
+        value_type = Value("float64")
+
+    return Features(
+        {
+            "real": _create_sized_feature_impl(data_shape, value_type),
+            "imag": _create_sized_feature_impl(data_shape, value_type),
+        }
+    )
+
+
+def _convert_complex_to_nested(arr: np.ndarray) -> pa.StructArray:
+    data = {
+        "real": datasets.features.features.numpy_to_pyarrow_listarray(arr.real),
+        "imag": datasets.features.features.numpy_to_pyarrow_listarray(arr.imag),
+    }
+    return pa.StructArray.from_arrays([data["real"], data["imag"]], names=["real", "imag"])
+
+
+# ┌────────────┐
+# │  Compound  │
+# └────────────┘
+
+
+def _is_compound_dtype(dtype: np.dtype) -> bool:
+    return dtype.kind == "V"
+
+
+@dataclass
+class _CompoundGroup:
+    dset: "h5py.Dataset"
+    data: np.ndarray = None
+
+    def items(self):
+        for field_name in self.dset.dtype.names:
+            field_dtype = self.dset.dtype[field_name]
+            yield field_name, _CompoundField(self.data, field_name, field_dtype)
+
+
+@dataclass
+class _CompoundField:
+    data: Optional[np.ndarray]
+    name: str
+    dtype: np.dtype
+    shape: tuple[int, ...] = field(init=False)
+
+    def __post_init__(self):
+        self.shape = (len(self.data) if self.data is not None else 0,) + self.dtype.shape
+
+    def __getitem__(self, key):
+        return self.data[key][self.name]
+
+
+def _create_compound_features(dset) -> Features:
+    mock_group = _CompoundGroup(dset)
+    return _recursive_infer_features(mock_group)
+
+
+def _convert_compound_to_nested(arr, dset) -> pa.StructArray:
+    mock_group = _CompoundGroup(dset, data=arr)
+    features = _create_compound_features(dset)
+    return _recursive_load_arrays(mock_group, features, 0, len(arr))
+
+
+# ┌───────────────────┐
+# │  Variable-Length  │
+# └───────────────────┘
+
+
+def _is_vlen_dtype(dtype: np.dtype) -> bool:
+    if dtype.metadata and "vlen" in dtype.metadata:
+        return True
+    return False
+
+
+def _create_vlen_features(dset) -> Features:
+    vlen_dtype = dset.dtype.metadata["vlen"]
+    if vlen_dtype in (str, bytes):
+        return Value("string")
+    inner_feature = _np_to_pa_to_hf_value(vlen_dtype)
+    return List(inner_feature)
+
+
+def _convert_vlen_to_array(arr: np.ndarray) -> pa.Array:
+    return datasets.features.features.numpy_to_pyarrow_listarray(arr)
+
+
+# ┌───────────┐
+# │  Generic  │
+# └───────────┘
+
+
+def _recursive_infer_features(h5_obj) -> Features:
+    features_dict = {}
+    for path, dset in h5_obj.items():
+        if _is_group(dset):
+            features = _recursive_infer_features(dset)
+            if features:
+                features_dict[path] = features
+        elif _is_dataset(dset):
+            features = _infer_feature(dset)
+            if features:
+                features_dict[path] = features
+
+    return Features(features_dict)
+
+
+def _infer_feature(dset):
+    if _is_complex_dtype(dset.dtype):
+        return _create_complex_features(dset)
+    elif _is_compound_dtype(dset.dtype) or dset.dtype.kind == "V":
+        return _create_compound_features(dset)
+    elif _is_vlen_dtype(dset.dtype):
+        return _create_vlen_features(dset)
+    return _create_sized_feature(dset)
+
+
+def _load_array(dset, path: str, start: int, end: int) -> pa.Array:
+    arr = dset[start:end]
+
+    if _is_vlen_dtype(dset.dtype):
+        return _convert_vlen_to_array(arr)
+    elif _is_complex_dtype(dset.dtype):
+        return _convert_complex_to_nested(arr)
+    elif _is_compound_dtype(dset.dtype):
+        return _convert_compound_to_nested(arr, dset)
+    elif dset.dtype.kind == "O":
+        raise ValueError(
+            f"Object dtype dataset '{path}' is not supported. "
+            f"For variable-length data, please use h5py.vlen_dtype() "
+            f"when creating the HDF5 file. "
+            f"See: https://docs.h5py.org/en/stable/special.html#variable-length-strings"
+        )
+    else:
+        # If any non-batch dimension is zero, emit an unsized pa.list_
+        # to avoid creating FixedSizeListArray with list_size=0.
+        if any(dim == 0 for dim in dset.shape[1:]):
+            inner_type = pa.from_numpy_dtype(dset.dtype)
+            return pa.array([[] for _ in arr], type=pa.list_(inner_type))
+        else:
+            return datasets.features.features.numpy_to_pyarrow_listarray(arr)
+
+
+def _recursive_load_arrays(h5_obj, features: Features, start: int, end: int):
+    batch_dict = {}
+    for path, dset in h5_obj.items():
+        if path not in features:
+            continue
+        if _is_group(dset):
+            arr = _recursive_load_arrays(dset, features[path], start, end)
+        elif _is_dataset(dset):
+            arr = _load_array(dset, path, start, end)
+        else:
+            raise ValueError(f"Unexpected type {type(dset)}")
+
+        if arr is not None:
+            batch_dict[path] = arr
+
+    if _is_file(h5_obj):
+        return pa.Table.from_pydict(batch_dict)
+
+    if batch_dict:
+        should_chunk, keys, values = False, [], []
+        for k, v in batch_dict.items():
+            if isinstance(v, pa.ChunkedArray):
+                should_chunk = True
+                v = v.combine_chunks()
+            keys.append(k)
+            values.append(v)
+
+        sarr = pa.StructArray.from_arrays(values, names=keys)
+        return pa.chunked_array(sarr) if should_chunk else sarr
+
+
+# ┌─────────────┐
+# │  Utilities  │
+# └─────────────┘
+
+
+def _create_sized_feature(dset):
+    dset_shape = dset.shape[1:]
+    value_feature = _np_to_pa_to_hf_value(dset.dtype)
+    return _create_sized_feature_impl(dset_shape, value_feature)
+
+
+def _create_sized_feature_impl(dset_shape, value_feature):
+    dtype_str = value_feature.dtype
+    if any(dim == 0 for dim in dset_shape):
+        logger.warning(
+            f"HDF5 to Arrow: Found a dataset with shape {dset_shape} and dtype {dtype_str} that has a dimension with size 0. Shape information will be lost in the conversion to List({value_feature})."
+        )
+        return List(value_feature)
+
+    rank = len(dset_shape)
+    if rank == 0:
+        return value_feature
+    elif rank == 1:
+        return List(value_feature, length=dset_shape[0])
+    elif rank <= 5:
+        return _sized_arrayxd(rank)(shape=dset_shape, dtype=dtype_str)
+    else:
+        raise TypeError(f"Array{rank}D not supported. Maximum 5 dimensions allowed.")
+
+
+def _sized_arrayxd(rank: int):
+    return {2: Array2D, 3: Array3D, 4: Array4D, 5: Array5D}[rank]
+
+
+def _np_to_pa_to_hf_value(numpy_dtype: np.dtype) -> Value:
+    return Value(dtype=_arrow_to_datasets_dtype(pa.from_numpy_dtype(numpy_dtype)))
+
+
+def _first_dataset(h5_obj, features: Features, prefix=""):
+    for path, dset in h5_obj.items():
+        if path not in features:
+            continue
+        if _is_group(dset):
+            found = _first_dataset(dset, features[path], prefix=f"{prefix}{path}/")
+            if found is not None:
+                return found
+        elif _is_dataset(dset):
+            return f"{prefix}{path}"
+
+
+def _check_dataset_lengths(h5_obj, features: Features) -> int:
+    first_path = _first_dataset(h5_obj, features)
+    if first_path is None:
+        return None
+
+    num_rows = h5_obj[first_path].shape[0]
+    for path, dset in h5_obj.items():
+        if path not in features:
+            continue
+        if _is_dataset(dset):
+            if dset.shape[0] != num_rows:
+                raise ValueError(f"Dataset '{path}' has length {dset.shape[0]} but expected {num_rows}")
+    return num_rows
+
+
+def _is_group(h5_obj) -> bool:
+    import h5py
+
+    return isinstance(h5_obj, h5py.Group) or isinstance(h5_obj, _CompoundGroup)
+
+
+def _is_dataset(h5_obj) -> bool:
+    import h5py
+
+    return isinstance(h5_obj, h5py.Dataset) or isinstance(h5_obj, _CompoundField)
+
+
+def _is_file(h5_obj) -> bool:
+    import h5py
+
+    return isinstance(h5_obj, h5py.File)
+
+
+def _has_zero_dimensions(feature):
+    if isinstance(feature, _ArrayXD):
+        return any(dim == 0 for dim in feature.shape)
+    elif isinstance(feature, List):
+        return feature.length == 0 or _has_zero_dimensions(feature.feature)
+    elif isinstance(feature, LargeList):
+        return _has_zero_dimensions(feature.feature)
+    else:
+        return False

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/json/json.py

@@ -0,0 +1,383 @@
+import io
+import os
+from dataclasses import dataclass
+from typing import Literal, Optional
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.json as paj
+
+import datasets
+import datasets.config
+from datasets.builder import Key
+from datasets.table import table_cast
+from datasets.utils.file_utils import readline
+from datasets.utils.json import (
+    find_mixed_struct_types_field_paths,
+    get_json_field_path_from_pyarrow_json_error,
+    get_json_field_paths_from_feature,
+    insert_json_field_path,
+    json_encode_field,
+    json_encode_fields_in_json_lines,
+    set_json_types_in_feature,
+    ujson_dumps,
+    ujson_loads,
+)
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+def pandas_read_json(path_or_buf, **kwargs):
+    if datasets.config.PANDAS_VERSION.major >= 2:
+        kwargs["dtype_backend"] = "pyarrow"
+    return pd.read_json(path_or_buf, **kwargs)
+
+
+class FullReadDisallowed(Exception):
+    pass
+
+
+@dataclass
+class JsonConfig(datasets.BuilderConfig):
+    """BuilderConfig for JSON."""
+
+    features: Optional[datasets.Features] = None
+    encoding: str = "utf-8"
+    encoding_errors: Optional[str] = None
+    field: Optional[str] = None
+    use_threads: bool = True  # deprecated
+    block_size: Optional[int] = None  # deprecated
+    chunksize: int = 10 << 20  # 10MB
+    newlines_in_values: Optional[bool] = None
+    on_mixed_types: Optional[Literal["use_json"]] = "use_json"
+    parse_agent_traces: bool = True
+
+    def __post_init__(self):
+        super().__post_init__()
+
+
+class Json(datasets.ArrowBasedBuilder):
+    BUILDER_CONFIG_CLASS = JsonConfig
+
+    def _info(self):
+        if self.config.block_size is not None:
+            logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead")
+            self.config.chunksize = self.config.block_size
+        if self.config.use_threads is not True:
+            logger.warning(
+                "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."
+            )
+        if self.config.newlines_in_values is not None:
+            raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported")
+        return datasets.DatasetInfo(features=self.config.features)
+
+    def _split_generators(self, dl_manager):
+        """We handle string, list and dicts in datafiles"""
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+        dl_manager.download_config.extract_on_the_fly = True
+        base_data_files = dl_manager.download(self.config.data_files)
+        extracted_data_files = dl_manager.extract(base_data_files)
+        splits = []
+        for split_name, extracted_files in extracted_data_files.items():
+            files_iterables = [dl_manager.iter_files(extracted_file) for extracted_file in extracted_files]
+            splits.append(
+                datasets.SplitGenerator(
+                    name=split_name,
+                    gen_kwargs={
+                        "files_iterables": files_iterables,
+                        "base_files": base_data_files[split_name],
+                        "original_files": self.config.data_files[split_name],
+                    },
+                )
+            )
+        if self.info.features is None:
+            try:
+                pa_table = next(iter(self._generate_tables(**splits[0].gen_kwargs, allow_full_read=False)))[1]
+                self.info.features = datasets.Features.from_arrow_schema(pa_table.schema)
+                if self.config.parse_agent_traces and has_agent_traces_markers(self.info.features):
+                    self.info.features = AGENT_TRACES_FEATURES
+            except FullReadDisallowed:
+                pass
+        return splits
+
+    def _cast_table(self, pa_table: pa.Table, json_field_paths=()) -> pa.Table:
+        if self.info.features is not None:
+            # adding missing columns
+            for column_name in set(self.info.features) - set(pa_table.column_names):
+                type = self.info.features.arrow_schema.field(column_name).type
+                pa_table = pa_table.append_column(column_name, pa.array([None] * len(pa_table), type=type))
+            # convert to string when needed
+            for i, column_name in enumerate(pa_table.column_names):
+                if pa.types.is_struct(pa_table[column_name].type) and self.info.features.get(
+                    column_name, None
+                ) == datasets.Value("string"):
+                    jsonl = (
+                        pa_table[column_name]
+                        .to_pandas(types_mapper=pd.ArrowDtype)
+                        .to_json(orient="records", lines=True)
+                    )
+                    string_array = pa.array(
+                        (None if x.strip() == "null" else x.strip() for x in jsonl.split("\n") if x.strip()),
+                        type=pa.string(),
+                    )
+                    pa_table = pa_table.set_column(i, column_name, string_array)
+            # more expensive cast to support nested structures with keys in a different order
+            # allows str <-> int/float or str to Audio for example
+            pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+        elif json_field_paths:
+            features = datasets.Features.from_arrow_schema(pa_table.schema)
+            features = set_json_types_in_feature(features, json_field_paths)
+            pa_table = table_cast(pa_table, features.arrow_schema)
+        return pa_table
+
+    def _generate_shards(self, base_files, files_iterables, original_files):
+        yield from base_files
+
+    def _generate_tables(self, base_files, files_iterables, original_files, allow_full_read=True):
+        json_field_paths = []
+        is_agent_traces = False
+
+        if self.info.features is not None:
+            if self.info.features == AGENT_TRACES_FEATURES:
+                is_agent_traces = True
+            else:
+                json_field_paths = get_json_field_paths_from_feature(self.info.features)
+
+        for shard_idx, files_iterable in enumerate(files_iterables):
+            for file in files_iterable:
+                # If the file is one json object and if we need to look at the items in one specific field
+                if self.config.field is not None:
+                    if not allow_full_read:
+                        raise FullReadDisallowed()
+                    with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f:
+                        dataset = ujson_loads(f.read())
+                    # We keep only the field we are interested in
+                    dataset = dataset[self.config.field]
+                    df = pandas_read_json(io.StringIO(ujson_dumps(dataset)))
+                    if df.columns.tolist() == [0]:
+                        df.columns = list(self.config.features) if self.config.features else ["text"]
+                    pa_table = pa.Table.from_pandas(df, preserve_index=False)
+                    yield Key(shard_idx, 0), self._cast_table(pa_table)
+
+                # If the files are agent traces (one row = one file)
+                elif is_agent_traces:
+                    with open(file, "r", encoding="utf-8") as f:
+                        traces = f.readlines()
+                    harness, session_id = parse_traces_info(traces)
+                    file_path = original_files[shard_idx]
+                    if file_path.startswith(self.base_path):
+                        file_path = os.path.relpath(file_path, self.base_path)
+                    pa_table = pa.Table.from_pydict(
+                        {
+                            "harness": [harness],
+                            "session_id": [session_id],
+                            "traces": [traces],
+                            "file_path": [file_path],
+                        }
+                    )
+                    yield Key(shard_idx, 0), self._cast_table(pa_table)
+
+                # If the file has one json object per line
+                else:
+                    with open(file, "rb") as f:
+                        batch_idx = 0
+                        # Use block_size equal to the chunk size divided by 32 to leverage multithreading
+                        # Set a default minimum value of 16kB if the chunk size is really small
+                        block_size = max(self.config.chunksize // 32, 16 << 10)
+                        encoding_errors = (
+                            self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
+                        )
+                        while True:
+                            batch = f.read(self.config.chunksize)
+                            if not batch:
+                                break
+                            if batch.startswith(b"["):
+                                if not allow_full_read:
+                                    raise FullReadDisallowed()
+                                else:
+                                    # convert to JSON Lines
+                                    full_data = batch + f.read()
+                                    if b"{" in batch[:100].split(b'"', 1)[0]:  # list of objects
+                                        batch = "\n".join(ujson_dumps(x) for x in ujson_loads(full_data)).encode()
+                                    else:  # list of strings
+                                        batch = "\n".join(
+                                            ujson_dumps({"text": x}) for x in ujson_loads(full_data)
+                                        ).encode()
+                            # Finish current line
+                            try:
+                                batch += f.readline()
+                            except (AttributeError, io.UnsupportedOperation):
+                                batch += readline(f)
+                            # PyArrow only accepts utf-8 encoded bytes
+                            if self.config.encoding != "utf-8":
+                                batch = batch.decode(self.config.encoding, errors=encoding_errors).encode("utf-8")
+                            # On first batch we check for lists of objects with arbitrary fields
+                            if (
+                                shard_idx == 0
+                                and batch_idx == 0
+                                and self.info.features is None
+                                and self.config.on_mixed_types == "use_json"
+                            ):
+                                try:
+                                    examples = [ujson_loads(line) for line in batch.splitlines()]
+                                except ValueError:
+                                    # the file is likely not JSON Lines and may contain one single multi-line JSON object
+                                    pass
+                                else:
+                                    json_field_paths += find_mixed_struct_types_field_paths(examples)
+                            # Re-encode JSON fields
+                            original_batch = batch
+                            if json_field_paths:
+                                examples = [ujson_loads(line) for line in batch.splitlines()]
+                                for json_field_path in json_field_paths:
+                                    examples = [json_encode_field(examples, json_field_path) for examples in examples]
+                                batch = "\n".join(ujson_dumps(example) for example in examples).encode()
+                            # Disable parallelism if block size is ~ len(batch) to avoid segfault
+                            block_size = len(batch) if len(batch) // 8 > block_size else block_size
+                            try:
+                                while True:
+                                    try:
+                                        pa_table = paj.read_json(
+                                            io.BytesIO(batch), read_options=paj.ReadOptions(block_size=block_size)
+                                        )
+                                        break
+                                    except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
+                                        if batch.startswith(b"["):  # paj.read_json only supports json lines
+                                            raise
+                                        elif self.config.on_mixed_types == "use_json" and (
+                                            isinstance(e, pa.ArrowInvalid)
+                                            and "JSON parse error: Column(" in str(e)
+                                            and ") changed from" in str(e)
+                                        ):
+                                            json_field_path = get_json_field_path_from_pyarrow_json_error(str(e))
+                                            insert_json_field_path(json_field_paths, json_field_path)
+                                            batch = json_encode_fields_in_json_lines(original_batch, json_field_paths)
+                                        elif (
+                                            "straddling" in str(e) or "JSON conversion to" in str(e)
+                                        ) and block_size < len(batch):
+                                            # Increase the block size in case it was too small.
+                                            # The block size will be reset for the next file.
+                                            # this is needed in case of "stradding" or for some JSON conversions (see https://github.com/huggingface/datasets/issues/2799)
+                                            logger.debug(
+                                                f"Batch of {len(batch)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."
+                                            )
+                                            block_size *= 2
+                                        else:
+                                            raise
+                            except pa.ArrowInvalid as e:
+                                if not allow_full_read:
+                                    raise FullReadDisallowed()
+                                try:
+                                    with open(
+                                        file, encoding=self.config.encoding, errors=self.config.encoding_errors
+                                    ) as f:
+                                        df = pandas_read_json(f)
+                                except ValueError:
+                                    logger.error(f"Failed to load JSON from file '{file}' with error {type(e)}: {e}")
+                                    raise e
+                                if df.columns.tolist() == [0]:
+                                    df.columns = list(self.config.features) if self.config.features else ["text"]
+                                try:
+                                    pa_table = pa.Table.from_pandas(df, preserve_index=False)
+                                except pa.ArrowInvalid as e:
+                                    logger.error(
+                                        f"Failed to convert pandas DataFrame to Arrow Table from file '{file}' with error {type(e)}: {e}"
+                                    )
+                                    raise ValueError(
+                                        f"Failed to convert pandas DataFrame to Arrow Table from file {file}."
+                                    ) from None
+                                yield Key(shard_idx, 0), self._cast_table(pa_table)
+                                break
+                            yield (
+                                Key(shard_idx, batch_idx),
+                                self._cast_table(pa_table, json_field_paths=json_field_paths),
+                            )
+                            batch_idx += 1
+
+
+AGENT_TRACES_TYPES_VALUES = {
+    "claude_code": ["user", "assistant", "system"],
+    "pi": ["session", "message"],
+    "codex": ["session_meta", "turn_context", "response_item", "event_msg"],
+}
+AGENT_TRACES_TYPE_TO_HARNESS = {}
+for _harness, _trace_types in AGENT_TRACES_TYPES_VALUES.items():
+    for _trace_type in _trace_types:
+        AGENT_TRACES_TYPE_TO_HARNESS[_trace_type] = _harness
+
+
+AGENT_TRACES_FEATURES_MARKERS = {
+    "claude_code": datasets.Features(
+        {
+            "type": datasets.Value("string"),
+            "message": datasets.Json(),
+        }
+    ),
+    "pi": datasets.Features(
+        {
+            "type": datasets.Value("string"),
+            "message": datasets.Json(),
+        }
+    ),
+    "codex": datasets.Features(
+        {
+            "type": datasets.Value("string"),
+            "payload": datasets.Json(),
+        }
+    ),
+}
+
+AGENT_TRACES_FEATURES = datasets.Features(
+    {
+        "harness": datasets.Value("string"),
+        "session_id": datasets.Value("string"),
+        "traces": datasets.List(datasets.Json()),
+        "file_path": datasets.Value("string"),
+    }
+)
+
+
+def has_agent_traces_markers(features: datasets.Features) -> bool:
+    for agent_traces_features_marker in AGENT_TRACES_FEATURES_MARKERS.values():
+        if all(features.get(key) == feature for key, feature in agent_traces_features_marker.items()):
+            return True
+    return False
+
+
+def parse_traces_info(traces: list[str]) -> tuple[Optional[str], Optional[str]]:
+    harness, session_id = None, None
+    for trace in traces:
+        decoded_trace = ujson_loads(trace)
+        if harness is None:
+            if "type" in decoded_trace and isinstance(decoded_trace["type"], str):
+                harness = AGENT_TRACES_TYPE_TO_HARNESS.get(decoded_trace["type"])
+        if session_id is None:
+            # claude
+            if "sessionId" in decoded_trace and isinstance(decoded_trace["sessionId"], str):
+                session_id = decoded_trace["sessionId"]
+            # claude (not sure but this format does exist online)
+            elif "session_id" in decoded_trace and isinstance(decoded_trace["session_id"], str):
+                session_id = decoded_trace["session_id"]
+            # codex
+            elif (
+                "payload" in decoded_trace
+                and isinstance(decoded_trace["payload"], dict)
+                and "id" in decoded_trace["payload"]
+                and isinstance(decoded_trace["payload"]["id"], str)
+            ):
+                session_id = decoded_trace["payload"]["id"]
+            # pi / openclaw (openclaw embeds pi-agent; distinguish via cwd)
+            elif (
+                "type" in decoded_trace
+                and decoded_trace["type"] == "session"
+                and "id" in decoded_trace
+                and isinstance(decoded_trace["id"], str)
+            ):
+                session_id = decoded_trace["id"]
+                if isinstance(decoded_trace.get("cwd"), str) and "/.openclaw/" in decoded_trace["cwd"]:
+                    harness = "openclaw"
+        if harness and session_id:
+            break
+    return harness, session_id

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/lance/lance.py

@@ -0,0 +1,235 @@
+import re
+from dataclasses import dataclass
+from pathlib import Path
+from typing import TYPE_CHECKING, Dict, List, Optional
+
+import pyarrow as pa
+from huggingface_hub import HfApi
+
+import datasets
+from datasets import Audio, Image, Video
+from datasets.builder import Key
+from datasets.table import table_cast
+from datasets.utils.file_utils import is_local_path
+
+
+if TYPE_CHECKING:
+    import lance
+    import lance.file
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+MAGIC_BYTES_EXTENSION_AND_FEATURE_TYPES = [
+    ("1A 45 DF A3", ".mkv", Video()),
+    ("66 74 79 70 69 73 6F 6D", ".mp4", Video()),
+    ("66 74 79 70 4D 53 4E 56", ".mp4", Video()),
+    ("52 49 46 46", ".avi", Video()),
+    ("00 00 01 BA", ".mpeg", Video()),
+    ("00 00 01 BA", ".mpeg", Video()),
+    ("00 00 01 B3", ".mov", Video()),
+    ("89 50 4E 47", ".png", Image()),
+    ("FF D8", ".jpg", Image()),
+    ("49 49", ".tif", Image()),
+    ("47 49 46 38", ".gif", Image()),
+    ("52 49 46 46", ".wav", Audio()),
+    ("49 44 33", ".mp3", Audio()),
+    ("66 4C 61 43", ".flac", Audio()),
+]
+
+
+@dataclass
+class LanceConfig(datasets.BuilderConfig):
+    """
+    BuilderConfig for Lance format.
+
+    Args:
+        features: (`Features`, *optional*):
+            Cast the data to `features`.
+        columns: (`List[str]`, *optional*):
+            List of columns to load, the other ones are ignored.
+        batch_size: (`int`, *optional*):
+            Size of the RecordBatches to iterate on. Default to 256.
+        token: (`str`, *optional*):
+            Optional HF token to use to download datasets.
+    """
+
+    features: Optional[datasets.Features] = None
+    columns: Optional[List[str]] = None
+    batch_size: Optional[int] = 256
+    token: Optional[str] = None
+
+
+def resolve_dataset_uris(files: List[str]) -> Dict[str, List[str]]:
+    dataset_uris = set()
+    for file_path in files:
+        path = Path(file_path)
+        if path.parent.name in {"_transactions", "_indices", "_versions"}:
+            dataset_root = path.parent.parent
+            dataset_uris.add(str(dataset_root))
+    return list(dataset_uris)
+
+
+def _fix_hf_uri(uri: str) -> str:
+    # replace the revision tag from hf uri
+    if "@" in uri:
+        matched = re.match(r"(hf://.+?)(@[0-9a-f]+)(/.*)", uri)
+        if matched:
+            uri = matched.group(1) + matched.group(3)
+    return uri
+
+
+def _fix_local_version_file(uri: str) -> str:
+    # replace symlinks with real files for _version
+    if "/_versions/" in uri and is_local_path(uri):
+        path = Path(uri)
+        if path.is_symlink():
+            data = path.read_bytes()
+            path.unlink()
+            path.write_bytes(data)
+    return uri
+
+
+class Lance(datasets.ArrowBasedBuilder, datasets.builder._CountableBuilderMixin):
+    BUILDER_CONFIG_CLASS = LanceConfig
+    METADATA_EXTENSIONS = [".idx", ".txn", ".manifest"]
+
+    def _info(self):
+        return datasets.DatasetInfo(features=self.config.features)
+
+    def _split_generators(self, dl_manager):
+        import lance
+        import lance.file
+
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+        if self.repo_id:
+            api = HfApi(**dl_manager.download_config.storage_options.get("hf", {}))
+            dataset_sha = api.dataset_info(self.repo_id).sha
+            if dataset_sha != self.hash:
+                raise NotImplementedError(
+                    f"lance doesn't support loading other revisions than 'main' yet, but got {self.hash}"
+                )
+        data_files = dl_manager.download(self.config.data_files)
+
+        # TODO: remove once Lance supports HF links with revisions
+        data_files = {split: [_fix_hf_uri(file) for file in files] for split, files in data_files.items()}
+        # TODO: remove once Lance supports symlinks for _version files
+        data_files = {split: [_fix_local_version_file(file) for file in files] for split, files in data_files.items()}
+
+        splits: list[datasets.SplitGenerator] = []
+        for split_name, files in data_files.items():
+            storage_options = dl_manager.download_config.storage_options.get(files[0].split("://", 0)[0] + "://")
+
+            lance_dataset_uris = resolve_dataset_uris(files)
+            if lance_dataset_uris:
+                lance_datasets = [lance.dataset(uri, storage_options=storage_options) for uri in lance_dataset_uris]
+                fragments = [frag for lance_dataset in lance_datasets for frag in lance_dataset.get_fragments()]
+                if self.info.features is None:
+                    pa_schema = fragments[0]._ds.schema
+                    first_row_first_bytes = {}
+                    for field in pa_schema:
+                        if self.config.columns is not None and field.name not in self.config.columns:
+                            continue
+                        if pa.types.is_binary(field.type) or pa.types.is_large_binary(field.type):
+                            try:
+                                first_row_first_bytes[field.name] = (
+                                    lance_datasets[0].take_blobs(field.name, [0])[0].read(16)
+                                )
+                            except ValueError:
+                                first_row_first_bytes[field.name] = (
+                                    lance_datasets[0].take([0], [field.name]).to_pylist()[0][field.name][:16]
+                                )
+                splits.append(
+                    datasets.SplitGenerator(
+                        name=split_name,
+                        gen_kwargs={"fragments": fragments, "lance_files_paths": None, "lance_files": None},
+                    )
+                )
+            else:
+                lance_files = [
+                    lance.file.LanceFileReader(file, storage_options=storage_options, columns=self.config.columns)
+                    for file in files
+                ]
+                if self.info.features is None:
+                    pa_schema = lance_files[0].metadata().schema
+                    first_row_first_bytes = {
+                        field_name: value[:16]
+                        for field_name, value in lance_files[0].take_rows([0]).to_table().to_pylist()[0].items()
+                        if isinstance(value, bytes)
+                    }
+                splits.append(
+                    datasets.SplitGenerator(
+                        name=split_name,
+                        gen_kwargs={"fragments": None, "lance_files_paths": files, "lance_files": lance_files},
+                    )
+                )
+            if self.info.features is None:
+                if self.config.columns:
+                    fields = [
+                        pa_schema.field(name) for name in self.config.columns if pa_schema.get_field_index(name) != -1
+                    ]
+                    pa_schema = pa.schema(fields)
+                features = datasets.Features.from_arrow_schema(pa_schema)
+                for field_name, first_bytes in first_row_first_bytes.items():
+                    for magic_bytes_hex, _, feature_type in MAGIC_BYTES_EXTENSION_AND_FEATURE_TYPES:
+                        magic_bytes = bytes.fromhex(magic_bytes_hex)
+                        if magic_bytes in first_bytes[: len(magic_bytes) * 2]:  # allow some padding
+                            features[field_name] = feature_type
+                            break
+                self.info.features = features
+
+        return splits
+
+    def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+        if self.info.features is not None:
+            # more expensive cast to support nested features with keys in a different order
+            # allows str <-> int/float or str to Audio for example
+            pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+        return pa_table
+
+    def _generate_shards(
+        self,
+        fragments: Optional[List["lance.LanceFragment"]],
+        lance_files_paths: Optional[list[str]],
+        lance_files: Optional[List["lance.file.LanceFileReader"]],
+    ):
+        if fragments:
+            for fragment in fragments:
+                paths = [data_file.path for data_file in fragment.metadata.data_files()]
+                yield paths[0] if len(paths) == 1 else {"fragment_data_files": paths}
+        else:
+            yield from lance_files_paths
+
+    def _generate_num_examples(
+        self,
+        fragments: Optional[List["lance.LanceFragment"]],
+        lance_files_paths: Optional[list[str]],
+        lance_files: Optional[List["lance.file.LanceFileReader"]],
+    ):
+        if fragments:
+            for fragment in fragments:
+                yield fragment.count_rows()
+        else:
+            for lance_file in lance_files:
+                yield lance_file.num_rows()
+
+    def _generate_tables(
+        self,
+        fragments: Optional[List["lance.LanceFragment"]],
+        lance_files_paths: Optional[list[str]],
+        lance_files: Optional[List["lance.file.LanceFileReader"]],
+    ):
+        if fragments:
+            for frag_idx, fragment in enumerate(fragments):
+                for batch_idx, batch in enumerate(
+                    fragment.to_batches(
+                        columns=self.config.columns, batch_size=self.config.batch_size, blob_handling="all_binary"
+                    )
+                ):
+                    table = pa.Table.from_batches([batch])
+                    yield Key(frag_idx, batch_idx), self._cast_table(table)
+        else:
+            for file_idx, lance_file in enumerate(lance_files):
+                for batch_idx, batch in enumerate(lance_file.read_all(batch_size=self.config.batch_size).to_batches()):
+                    table = pa.Table.from_batches([batch])
+                    yield Key(file_idx, batch_idx), self._cast_table(table)

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/niftifolder/niftifolder.py

@@ -0,0 +1,23 @@
+import datasets
+
+from ..folder_based_builder import folder_based_builder
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class NiftiFolderConfig(folder_based_builder.FolderBasedBuilderConfig):
+    """BuilderConfig for NiftiFolder."""
+
+    drop_labels: bool = None
+    drop_metadata: bool = None
+
+    def __post_init__(self):
+        super().__post_init__()
+
+
+class NiftiFolder(folder_based_builder.FolderBasedBuilder):
+    BASE_FEATURE = datasets.Nifti
+    BASE_COLUMN_NAME = "nifti"
+    BUILDER_CONFIG_CLASS = NiftiFolderConfig
+    EXTENSIONS: list[str] = [".nii"]

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/parquet/parquet.py

@@ -0,0 +1,217 @@
+from dataclasses import dataclass
+from typing import Literal, Optional, Union
+
+import pyarrow as pa
+import pyarrow.dataset as ds
+import pyarrow.parquet as pq
+
+import datasets
+from datasets.builder import Key
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class ParquetConfig(datasets.BuilderConfig):
+    """
+    BuilderConfig for Parquet.
+
+    Args:
+        batch_size (`int`, *optional*):
+            Size of the RecordBatches to iterate on.
+            The default is the row group size (defined by the first row group).
+        columns (`list[str]`, *optional*)
+            List of columns to load, the other ones are ignored.
+            All columns are loaded by default.
+        features: (`Features`, *optional*):
+            Cast the data to `features`.
+        filters (`Union[pyarrow.dataset.Expression, list[tuple], list[list[tuple]]]`, *optional*):
+            Return only the rows matching the filter.
+            If possible the predicate will be pushed down to exploit the partition information
+            or internal metadata found in the data source, e.g. Parquet statistics.
+            Otherwise filters the loaded RecordBatches before yielding them.
+        fragment_scan_options (`pyarrow.dataset.ParquetFragmentScanOptions`, *optional*)
+            Scan-specific options for Parquet fragments.
+            This is especially useful to configure buffering and caching.
+
+            <Added version="4.2.0"/>
+        on_bad_files (`Literal["error", "warn", "skip"]`, *optional*, defaults to "error")
+            Specify what to do upon encountering a bad file (a file that can't be read). Allowed values are :
+            * 'error', raise an Exception when a bad file is encountered.
+            * 'warn', raise a warning when a bad file is encountered and skip that file.
+            * 'skip', skip bad files without raising or warning when they are encountered.
+
+            <Added version="4.2.0"/>
+
+    Example:
+
+    Load a subset of columns:
+
+    ```python
+    >>> ds = load_dataset(parquet_dataset_id, columns=["col_0", "col_1"])
+    ```
+
+    Stream data and efficiently filter data, possibly skipping entire files or row groups:
+
+    ```python
+    >>> filters = [("col_0", "==", 0)]
+    >>> ds = load_dataset(parquet_dataset_id, streaming=True, filters=filters)
+    ```
+
+    Increase the minimum request size when streaming from 32MiB (default) to 128MiB and enable prefetching:
+
+    ```python
+    >>> import pyarrow
+    >>> import pyarrow.dataset
+    >>> fragment_scan_options = pyarrow.dataset.ParquetFragmentScanOptions(
+    ...     cache_options=pyarrow.CacheOptions(
+    ...         prefetch_limit=1,
+    ...         range_size_limit=128 << 20
+    ...     ),
+    ... )
+    >>> ds = load_dataset(parquet_dataset_id, streaming=True, fragment_scan_options=fragment_scan_options)
+    ```
+
+    """
+
+    batch_size: Optional[int] = None
+    columns: Optional[list[str]] = None
+    features: Optional[datasets.Features] = None
+    filters: Optional[Union[ds.Expression, list[tuple], list[list[tuple]]]] = None
+    fragment_scan_options: Optional[ds.ParquetFragmentScanOptions] = None
+    on_bad_files: Literal["error", "warn", "skip"] = "error"
+
+    def __post_init__(self):
+        super().__post_init__()
+
+
+class Parquet(datasets.ArrowBasedBuilder):
+    BUILDER_CONFIG_CLASS = ParquetConfig
+
+    def _info(self):
+        if (
+            self.config.columns is not None
+            and self.config.features is not None
+            and set(self.config.columns) != set(self.config.features)
+        ):
+            raise ValueError(
+                "The columns and features argument must contain the same columns, but got ",
+                f"{self.config.columns} and {self.config.features}",
+            )
+        return datasets.DatasetInfo(features=self.config.features)
+
+    def _split_generators(self, dl_manager):
+        """We handle string, list and dicts in datafiles"""
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+        dl_manager.download_config.extract_on_the_fly = True
+        data_files = dl_manager.download(self.config.data_files)
+        splits = []
+        for split_name, files in data_files.items():
+            # Infer features if they are stored in the arrow schema
+            if self.info.features is None:
+                for file in files:
+                    try:
+                        with open(file, "rb") as f:
+                            self.info.features = datasets.Features.from_arrow_schema(pq.read_schema(f))
+                            break
+                    except pa.ArrowInvalid as e:
+                        if self.config.on_bad_files == "error":
+                            logger.error(f"Failed to read schema from '{file}' with error {type(e).__name__}: {e}")
+                            raise
+                        elif self.config.on_bad_files == "warn":
+                            logger.warning(f"Skipping bad schema from '{file}'. {type(e).__name__}: {e}`")
+                        else:
+                            logger.debug(f"Skipping bad schema from '{file}'. {type(e).__name__}: {e}`")
+            if self.info.features is None:
+                raise ValueError(
+                    f"At least one valid data file must be specified, all the data_files are invalid: {self.config.data_files}"
+                )
+            splits.append(
+                datasets.SplitGenerator(
+                    name=split_name, gen_kwargs={"files": files, "row_groups_list": [None] * len(files)}
+                )
+            )
+        if self.config.columns is not None and set(self.config.columns) != set(self.info.features):
+            self.info.features = datasets.Features(
+                {col: feat for col, feat in self.info.features.items() if col in self.config.columns}
+            )
+        return splits
+
+    def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+        if self.info.features is not None:
+            # more expensive cast to support nested features with keys in a different order
+            # allows str <-> int/float or str to Audio for example
+            pa_table = table_cast(pa_table, self.info.features.arrow_schema)
+        return pa_table
+
+    def _generate_shards(self, files, row_groups_list):
+        if not row_groups_list:
+            yield from files
+        else:
+            for file, row_groups in zip(files, row_groups_list):
+                yield {
+                    "fragment_data_file": file,
+                    "fragment_row_groups": row_groups,
+                }
+
+    def _generate_more_gen_kwargs(self, files, row_groups_list):
+        if not row_groups_list:
+            parquet_file_format = ds.ParquetFileFormat(default_fragment_scan_options=self.config.fragment_scan_options)
+            for file in files:
+                with open(file, "rb") as f:
+                    parquet_fragment = parquet_file_format.make_fragment(f)
+                    yield {
+                        "files": [file] * parquet_fragment.num_row_groups,
+                        "row_groups_list": [
+                            (row_group_id,) for row_group_id in range(parquet_fragment.num_row_groups)
+                        ],
+                    }
+        else:
+            for file, row_groups in zip(files, row_groups_list):
+                yield {"files": [file], "row_groups_list": [row_groups]}
+
+    def _generate_tables(self, files, row_groups_list):
+        if self.config.features is not None and self.config.columns is not None:
+            if sorted(field.name for field in self.info.features.arrow_schema) != sorted(self.config.columns):
+                raise ValueError(
+                    f"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"
+                )
+        filter_expr = (
+            pq.filters_to_expression(self.config.filters)
+            if isinstance(self.config.filters, list)
+            else self.config.filters
+        )
+        parquet_file_format = ds.ParquetFileFormat(default_fragment_scan_options=self.config.fragment_scan_options)
+        for file_idx, (file, row_groups) in enumerate(zip(files, row_groups_list)):
+            try:
+                with open(file, "rb") as f:
+                    parquet_fragment = parquet_file_format.make_fragment(f)
+                    if row_groups is not None:
+                        parquet_fragment.subset(row_group_ids=row_groups)
+                    if parquet_fragment.row_groups:
+                        batch_size = self.config.batch_size or parquet_fragment.row_groups[0].num_rows
+                        for batch_idx, record_batch in enumerate(
+                            parquet_fragment.to_batches(
+                                batch_size=batch_size,
+                                columns=self.config.columns,
+                                filter=filter_expr,
+                                batch_readahead=0,
+                                fragment_readahead=0,
+                            )
+                        ):
+                            pa_table = pa.Table.from_batches([record_batch])
+                            # Uncomment for debugging (will print the Arrow table size and elements)
+                            # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+                            # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+                            yield Key(file_idx, batch_idx), self._cast_table(pa_table)
+            except (pa.ArrowInvalid, ValueError) as e:
+                if self.config.on_bad_files == "error":
+                    logger.error(f"Failed to read file '{file}' with error {type(e).__name__}: {e}")
+                    raise
+                elif self.config.on_bad_files == "warn":
+                    logger.warning(f"Skipping bad file '{file}'. {type(e).__name__}: {e}`")
+                else:
+                    logger.debug(f"Skipping bad file '{file}'. {type(e).__name__}: {e}`")

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/spark/spark.py

@@ -0,0 +1,367 @@
+import os
+import posixpath
+import uuid
+from collections.abc import Iterable
+from dataclasses import dataclass
+from itertools import islice
+from typing import TYPE_CHECKING, Optional, Union
+
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.arrow_writer import ArrowWriter, ParquetWriter
+from datasets.config import MAX_SHARD_SIZE
+from datasets.filesystems import (
+    is_remote_filesystem,
+    rename,
+)
+from datasets.iterable_dataset import _BaseExamplesIterable
+from datasets.utils import experimental
+from datasets.utils.py_utils import convert_file_size_to_int
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+if TYPE_CHECKING:
+    import pyspark
+    import pyspark.sql
+
+
+@dataclass
+class SparkConfig(datasets.BuilderConfig):
+    """BuilderConfig for Spark."""
+
+    features: Optional[datasets.Features] = None
+
+    def __post_init__(self):
+        super().__post_init__()
+
+
+def _reorder_dataframe_by_partition(df: "pyspark.sql.DataFrame", new_partition_order: list[int]):
+    df_combined = df.select("*").where(f"part_id = {new_partition_order[0]}")
+    for partition_id in new_partition_order[1:]:
+        partition_df = df.select("*").where(f"part_id = {partition_id}")
+        df_combined = df_combined.union(partition_df)
+    return df_combined
+
+
+def _generate_iterable_examples(
+    df: "pyspark.sql.DataFrame",
+    partition_order: list[int],
+    state_dict: Optional[dict] = None,
+):
+    import pyspark
+
+    df_with_partition_id = df.select("*", pyspark.sql.functions.spark_partition_id().alias("part_id"))
+    partition_idx_start = state_dict["partition_idx"] if state_dict else 0
+    partition_df = _reorder_dataframe_by_partition(df_with_partition_id, partition_order[partition_idx_start:])
+    # pipeline next partition in parallel to hide latency
+    rows = partition_df.toLocalIterator(prefetchPartitions=True)
+    curr_partition = None
+    row_id = state_dict["partition_example_idx"] if state_dict else 0
+    for row in islice(rows, row_id, None):
+        row_as_dict = row.asDict()
+        part_id = row_as_dict["part_id"]
+        row_as_dict.pop("part_id")
+        if curr_partition != part_id:
+            if state_dict and curr_partition is not None:
+                state_dict["partition_idx"] += 1
+            curr_partition = part_id
+            row_id = 0
+        if state_dict:
+            state_dict["partition_example_idx"] = row_id + 1
+        yield (part_id, row_id), row_as_dict
+        row_id += 1
+
+
+class SparkExamplesIterable(_BaseExamplesIterable):
+    def __init__(
+        self,
+        df: "pyspark.sql.DataFrame",
+        partition_order=None,
+    ):
+        super().__init__()
+        self.df = df
+        self.partition_order = partition_order or range(self.df.rdd.getNumPartitions())
+
+    def _init_state_dict(self) -> dict:
+        self._state_dict = {"partition_idx": 0, "partition_example_idx": 0}
+        return self._state_dict
+
+    @experimental
+    def load_state_dict(self, state_dict: dict) -> dict:
+        return super().load_state_dict(state_dict)
+
+    def __iter__(self):
+        yield from _generate_iterable_examples(self.df, self.partition_order, self._state_dict)
+
+    def shuffle_data_sources(self, generator: np.random.Generator) -> "SparkExamplesIterable":
+        partition_order = list(range(self.df.rdd.getNumPartitions()))
+        generator.shuffle(partition_order)
+        return SparkExamplesIterable(self.df, partition_order=partition_order)
+
+    def shard_data_sources(self, num_shards: int, index: int, contiguous=True) -> "SparkExamplesIterable":
+        partition_order = self.split_shard_indices_by_worker(num_shards=num_shards, index=index, contiguous=contiguous)
+        return SparkExamplesIterable(self.df, partition_order=partition_order)
+
+    @property
+    def num_shards(self) -> int:
+        return len(self.partition_order)
+
+
+class Spark(datasets.DatasetBuilder):
+    BUILDER_CONFIG_CLASS = SparkConfig
+
+    def __init__(
+        self,
+        df: "pyspark.sql.DataFrame",
+        cache_dir: str = None,
+        working_dir: str = None,
+        **config_kwargs,
+    ):
+        import pyspark
+
+        self._spark = pyspark.sql.SparkSession.builder.getOrCreate()
+        self.df = df
+        self._working_dir = working_dir
+
+        super().__init__(
+            cache_dir=cache_dir,
+            config_name=str(self.df.semanticHash()),
+            **config_kwargs,
+        )
+
+    def _validate_cache_dir(self):
+        # Define this so that we don't reference self in create_cache_and_write_probe, which will result in a pickling
+        # error due to pickling the SparkContext.
+        cache_dir = self._cache_dir
+
+        # Returns the path of the created file.
+        def create_cache_and_write_probe(context):
+            # makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
+            # already exist.
+            os.makedirs(cache_dir, exist_ok=True)
+            probe_file = os.path.join(cache_dir, "fs_test" + uuid.uuid4().hex)
+            # Opening the file in append mode will create a new file unless it already exists, in which case it will not
+            # change the file contents.
+            open(probe_file, "a")
+            return [probe_file]
+
+        if self._spark.conf.get("spark.master", "").startswith("local"):
+            return
+
+        # If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
+        # accessible to the driver.
+        # TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
+        if self._cache_dir:
+            probe = (
+                self._spark.sparkContext.parallelize(range(1), 1).mapPartitions(create_cache_and_write_probe).collect()
+            )
+            if os.path.isfile(probe[0]):
+                return
+
+        raise ValueError(
+            "When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir"
+        )
+
+    def _info(self):
+        return datasets.DatasetInfo(features=self.config.features)
+
+    def _split_generators(self, dl_manager: datasets.download.download_manager.DownloadManager):
+        return [datasets.SplitGenerator(name=datasets.Split.TRAIN)]
+
+    def _repartition_df_if_needed(self, max_shard_size):
+        import pyspark
+
+        def get_arrow_batch_size(it):
+            for batch in it:
+                yield pa.RecordBatch.from_pydict({"batch_bytes": [batch.nbytes]})
+
+        df_num_rows = self.df.count()
+        sample_num_rows = df_num_rows if df_num_rows <= 100 else 100
+        # Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
+        approx_bytes_per_row = (
+            self.df.limit(sample_num_rows)
+            .repartition(1)
+            .mapInArrow(get_arrow_batch_size, "batch_bytes: long")
+            .agg(pyspark.sql.functions.sum("batch_bytes").alias("sample_bytes"))
+            .collect()[0]
+            .sample_bytes
+            / sample_num_rows
+        )
+        approx_total_size = approx_bytes_per_row * df_num_rows
+        if approx_total_size > max_shard_size:
+            # Make sure there is at least one row per partition.
+            new_num_partitions = min(df_num_rows, int(approx_total_size / max_shard_size))
+            self.df = self.df.repartition(new_num_partitions)
+
+    def _prepare_split_single(
+        self,
+        fpath: str,
+        file_format: str,
+        max_shard_size: int,
+    ) -> Iterable[tuple[int, bool, Union[int, tuple]]]:
+        import pyspark
+
+        writer_class = ParquetWriter if file_format == "parquet" else ArrowWriter
+        working_fpath = os.path.join(self._working_dir, os.path.basename(fpath)) if self._working_dir else fpath
+        embed_local_files = file_format == "parquet"
+
+        # Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
+        # pickling the SparkContext.
+        features = self.config.features
+        writer_batch_size = self._writer_batch_size
+        storage_options = self._fs.storage_options
+
+        def write_arrow(it):
+            # Within the same SparkContext, no two task attempts will share the same attempt ID.
+            task_id = pyspark.TaskContext().taskAttemptId()
+            first_batch = next(it, None)
+            if first_batch is None:
+                # Some partitions might not receive any data.
+                return pa.RecordBatch.from_arrays(
+                    [[task_id], [0], [0]],
+                    names=["task_id", "num_examples", "num_bytes"],
+                )
+            shard_id = 0
+            writer = writer_class(
+                features=features,
+                path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+                writer_batch_size=writer_batch_size,
+                storage_options=storage_options,
+                embed_local_files=embed_local_files,
+            )
+            table = pa.Table.from_batches([first_batch])
+            writer.write_table(table)
+            for batch in it:
+                if max_shard_size is not None and writer._num_bytes >= max_shard_size:
+                    num_examples, num_bytes = writer.finalize()
+                    writer.close()
+                    yield pa.RecordBatch.from_arrays(
+                        [[task_id], [num_examples], [num_bytes]],
+                        names=["task_id", "num_examples", "num_bytes"],
+                    )
+                    shard_id += 1
+                    writer = writer_class(
+                        features=writer._features,
+                        path=working_fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+                        writer_batch_size=writer_batch_size,
+                        storage_options=storage_options,
+                        embed_local_files=embed_local_files,
+                    )
+                table = pa.Table.from_batches([batch])
+                writer.write_table(table)
+
+            if writer._num_bytes > 0:
+                num_examples, num_bytes = writer.finalize()
+                writer.close()
+                yield pa.RecordBatch.from_arrays(
+                    [[task_id], [num_examples], [num_bytes]],
+                    names=["task_id", "num_examples", "num_bytes"],
+                )
+
+            if working_fpath != fpath:
+                for file in os.listdir(os.path.dirname(working_fpath)):
+                    dest = os.path.join(os.path.dirname(fpath), os.path.basename(file))
+                    shutil.move(file, dest)
+
+        stats = (
+            self.df.mapInArrow(write_arrow, "task_id: long, num_examples: long, num_bytes: long")
+            .groupBy("task_id")
+            .agg(
+                pyspark.sql.functions.sum("num_examples").alias("total_num_examples"),
+                pyspark.sql.functions.sum("num_bytes").alias("total_num_bytes"),
+                pyspark.sql.functions.count("num_bytes").alias("num_shards"),
+                pyspark.sql.functions.collect_list("num_examples").alias("shard_lengths"),
+            )
+            .collect()
+        )
+        for row in stats:
+            yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)
+
+    def _prepare_split(
+        self,
+        split_generator: "datasets.SplitGenerator",
+        file_format: str = "arrow",
+        max_shard_size: Optional[Union[str, int]] = None,
+        num_proc: Optional[int] = None,
+        **kwargs,
+    ):
+        self._validate_cache_dir()
+
+        max_shard_size = convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE)
+        self._repartition_df_if_needed(max_shard_size)
+        is_local = not is_remote_filesystem(self._fs)
+        path_join = os.path.join if is_local else posixpath.join
+
+        SUFFIX = "-TTTTT-SSSSS-of-NNNNN"
+        fname = f"{self.name}-{split_generator.name}{SUFFIX}.{file_format}"
+        fpath = path_join(self._output_dir, fname)
+
+        total_num_examples = 0
+        total_num_bytes = 0
+        total_shards = 0
+        task_id_and_num_shards = []
+        all_shard_lengths = []
+
+        for task_id, content in self._prepare_split_single(fpath, file_format, max_shard_size):
+            (
+                num_examples,
+                num_bytes,
+                num_shards,
+                shard_lengths,
+            ) = content
+            if num_bytes > 0:
+                total_num_examples += num_examples
+                total_num_bytes += num_bytes
+                total_shards += num_shards
+                task_id_and_num_shards.append((task_id, num_shards))
+                all_shard_lengths.extend(shard_lengths)
+
+        split_generator.split_info.num_examples = total_num_examples
+        split_generator.split_info.num_bytes = total_num_bytes
+
+        # should rename everything at the end
+        logger.debug(f"Renaming {total_shards} shards.")
+        if total_shards > 1:
+            split_generator.split_info.shard_lengths = all_shard_lengths
+
+            # Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
+            # pickling error due to pickling the SparkContext.
+            fs = self._fs
+
+            # use the -SSSSS-of-NNNNN pattern
+            def _rename_shard(
+                task_id: int,
+                shard_id: int,
+                global_shard_id: int,
+            ):
+                rename(
+                    fs,
+                    fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+                    fpath.replace("TTTTT-SSSSS", f"{global_shard_id:05d}").replace("NNNNN", f"{total_shards:05d}"),
+                )
+
+            args = []
+            global_shard_id = 0
+            for i in range(len(task_id_and_num_shards)):
+                task_id, num_shards = task_id_and_num_shards[i]
+                for shard_id in range(num_shards):
+                    args.append([task_id, shard_id, global_shard_id])
+                    global_shard_id += 1
+            self._spark.sparkContext.parallelize(args, len(args)).map(lambda args: _rename_shard(*args)).collect()
+        else:
+            # don't use any pattern
+            shard_id = 0
+            task_id = task_id_and_num_shards[0][0]
+            self._rename(
+                fpath.replace("SSSSS", f"{shard_id:05d}").replace("TTTTT", f"{task_id:05d}"),
+                fpath.replace(SUFFIX, ""),
+            )
+
+    def _get_examples_iterable_for_split(
+        self,
+        split_generator: "datasets.SplitGenerator",
+    ) -> SparkExamplesIterable:
+        return SparkExamplesIterable(self.df)

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/text/text.py

@@ -0,0 +1,137 @@
+from dataclasses import dataclass
+from io import StringIO
+from typing import Literal, Optional
+
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.features.features import require_storage_cast
+from datasets.table import table_cast
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+@dataclass
+class TextConfig(datasets.BuilderConfig):
+    """BuilderConfig for text files.
+
+    Args:
+        features: (`Features`, *optional*):
+            Cast the data to `features`.
+        encoding: (`str`, defaults to "utf-8"):
+            Encoding to decode the file.
+        encoding_errors: (`str`, *optional*):
+            Argument to define what to do in case of encoding error.
+            This is the same as the `error` argument in `open()`.
+        chunksize: (`Features`, *optional*, defaults to "10MB"):
+            Chunk size to read the data.
+        keep_linebreaks: (`bool`, defaults to False):
+            Whether to keep line breaks.
+        sample_by (`Literal["line", "paragraph", "document"]`, defaults to "line"):
+            Whether to load data per line, praragraph or document.
+            By default one row in the dataset = one line.
+    """
+
+    features: Optional[datasets.Features] = None
+    encoding: str = "utf-8"
+    encoding_errors: Optional[str] = None
+    chunksize: int = 10 << 20  # 10MB
+    keep_linebreaks: bool = False
+    sample_by: Literal["line", "paragraph", "document"] = "line"
+
+
+class Text(datasets.ArrowBasedBuilder):
+    BUILDER_CONFIG_CLASS = TextConfig
+
+    def _info(self):
+        return datasets.DatasetInfo(features=self.config.features)
+
+    def _split_generators(self, dl_manager):
+        """The `data_files` kwarg in load_dataset() can be a str, List[str], Dict[str,str], or Dict[str,List[str]].
+
+        If str or List[str], then the dataset returns only the 'train' split.
+        If dict, then keys should be from the `datasets.Split` enum.
+        """
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+        dl_manager.download_config.extract_on_the_fly = True
+        base_data_files = dl_manager.download(self.config.data_files)
+        extracted_data_files = dl_manager.extract(base_data_files)
+        splits = []
+        for split_name, files in extracted_data_files.items():
+            files_iterables = [dl_manager.iter_files(file) for file in files]
+            splits.append(
+                datasets.SplitGenerator(
+                    name=split_name,
+                    gen_kwargs={"files_iterables": files_iterables, "base_files": base_data_files[split_name]},
+                )
+            )
+        return splits
+
+    def _cast_table(self, pa_table: pa.Table) -> pa.Table:
+        if self.config.features is not None:
+            schema = self.config.features.arrow_schema
+            if all(not require_storage_cast(feature) for feature in self.config.features.values()):
+                # cheaper cast
+                pa_table = pa_table.cast(schema)
+            else:
+                # more expensive cast; allows str <-> int/float or str to Audio for example
+                pa_table = table_cast(pa_table, schema)
+            return pa_table
+        else:
+            return pa_table.cast(pa.schema({"text": pa.string()}))
+
+    def _generate_shards(self, base_files, files_iterables):
+        yield from base_files
+
+    def _generate_tables(self, base_files, files_iterables):
+        pa_table_names = list(self.config.features) if self.config.features is not None else ["text"]
+        for shard_idx, files_iterable in enumerate(files_iterables):
+            for file in files_iterable:
+                # open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
+                with open(file, encoding=self.config.encoding, errors=self.config.encoding_errors) as f:
+                    if self.config.sample_by == "line":
+                        batch_idx = 0
+                        while True:
+                            batch = f.read(self.config.chunksize)
+                            if not batch:
+                                break
+                            batch += f.readline()  # finish current line
+                            # StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
+                            batch = StringIO(batch).readlines()
+                            if not self.config.keep_linebreaks:
+                                batch = [line.rstrip("\n") for line in batch]
+                            pa_table = pa.Table.from_arrays([pa.array(batch)], names=pa_table_names)
+                            # Uncomment for debugging (will print the Arrow table size and elements)
+                            # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+                            # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+                            yield Key(shard_idx, batch_idx), self._cast_table(pa_table)
+                            batch_idx += 1
+                    elif self.config.sample_by == "paragraph":
+                        batch_idx = 0
+                        batch = ""
+                        while True:
+                            new_batch = f.read(self.config.chunksize)
+                            if not new_batch:
+                                break
+                            batch += new_batch
+                            batch += f.readline()  # finish current line
+                            batch = batch.split("\n\n")
+                            pa_table = pa.Table.from_arrays(
+                                [pa.array([example for example in batch[:-1] if example])], names=pa_table_names
+                            )
+                            # Uncomment for debugging (will print the Arrow table size and elements)
+                            # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
+                            # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
+                            yield Key(shard_idx, batch_idx), self._cast_table(pa_table)
+                            batch_idx += 1
+                            batch = batch[-1]
+                        if batch:
+                            pa_table = pa.Table.from_arrays([pa.array([batch])], names=pa_table_names)
+                            yield (shard_idx, batch_idx), self._cast_table(pa_table)
+                    elif self.config.sample_by == "document":
+                        text = f.read()
+                        pa_table = pa.Table.from_arrays([pa.array([text])], names=pa_table_names)
+                        yield Key(shard_idx, 0), self._cast_table(pa_table)

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/webdataset/_tenbin.py

@@ -0,0 +1,285 @@
+#
+# Copyright (c) 2017-2021 NVIDIA CORPORATION. All rights reserved.
+# This file coems from the WebDataset library.
+# See the LICENSE file for licensing terms (BSD-style).
+#
+
+"""
+Binary tensor encodings for PyTorch and NumPy.
+
+This defines efficient binary encodings for tensors. The format is 8 byte
+aligned and can be used directly for computations when transmitted, say,
+via RDMA. The format is supported by WebDataset with the `.ten` filename
+extension. It is also used by Tensorcom, Tensorcom RDMA, and can be used
+for fast tensor storage with LMDB and in disk files (which can be memory
+mapped)
+
+Data is encoded as a series of chunks:
+
+- magic number (int64)
+- length in bytes (int64)
+- bytes (multiple of 64 bytes long)
+
+Arrays are a header chunk followed by a data chunk.
+Header chunks have the following structure:
+
+- dtype (int64)
+- 8 byte array name
+- ndim (int64)
+- dim[0]
+- dim[1]
+- ...
+"""
+
+import struct
+import sys
+
+import numpy as np
+
+
+def bytelen(a):
+    """Determine the length of a in bytes."""
+    if hasattr(a, "nbytes"):
+        return a.nbytes
+    elif isinstance(a, (bytearray, bytes)):
+        return len(a)
+    else:
+        raise ValueError(a, "cannot determine nbytes")
+
+
+def bytedata(a):
+    """Return a the raw data corresponding to a."""
+    if isinstance(a, (bytearray, bytes, memoryview)):
+        return a
+    elif hasattr(a, "data"):
+        return a.data
+    else:
+        raise ValueError(a, "cannot return bytedata")
+
+
+# tables for converting between long/short NumPy dtypes
+
+long_to_short = """
+float16 f2
+float32 f4
+float64 f8
+int8 i1
+int16 i2
+int32 i4
+int64 i8
+uint8 u1
+uint16 u2
+unit32 u4
+uint64 u8
+""".strip()
+long_to_short = [x.split() for x in long_to_short.split("\n")]
+long_to_short = {x[0]: x[1] for x in long_to_short}
+short_to_long = {v: k for k, v in long_to_short.items()}
+
+
+def check_acceptable_input_type(data, allow64):
+    """Check that the data has an acceptable type for tensor encoding.
+
+    :param data: array
+    :param allow64: allow 64 bit types
+    """
+    for a in data:
+        if a.dtype.name not in long_to_short:
+            raise ValueError("unsupported dataypte")
+        if not allow64 and a.dtype.name not in ["float64", "int64", "uint64"]:
+            raise ValueError("64 bit datatypes not allowed unless explicitly enabled")
+
+
+def str64(s):
+    """Convert a string to an int64."""
+    s = s + "\0" * (8 - len(s))
+    s = s.encode("ascii")
+    return struct.unpack("@q", s)[0]
+
+
+def unstr64(i):
+    """Convert an int64 to a string."""
+    b = struct.pack("@q", i)
+    return b.decode("ascii").strip("\0")
+
+
+def check_infos(data, infos, required_infos=None):
+    """Verify the info strings."""
+    if required_infos is False or required_infos is None:
+        return data
+    if required_infos is True:
+        return data, infos
+    if not isinstance(required_infos, (tuple, list)):
+        raise ValueError("required_infos must be tuple or list")
+    for required, actual in zip(required_infos, infos):
+        raise ValueError(f"actual info {actual} doesn't match required info {required}")
+    return data
+
+
+def encode_header(a, info=""):
+    """Encode an array header as a byte array."""
+    if a.ndim >= 10:
+        raise ValueError("too many dimensions")
+    if a.nbytes != np.prod(a.shape) * a.itemsize:
+        raise ValueError("mismatch between size and shape")
+    if a.dtype.name not in long_to_short:
+        raise ValueError("unsupported array type")
+    header = [str64(long_to_short[a.dtype.name]), str64(info), len(a.shape)] + list(a.shape)
+    return bytedata(np.array(header, dtype="i8"))
+
+
+def decode_header(h):
+    """Decode a byte array into an array header."""
+    h = np.frombuffer(h, dtype="i8")
+    if unstr64(h[0]) not in short_to_long:
+        raise ValueError("unsupported array type")
+    dtype = np.dtype(short_to_long[unstr64(h[0])])
+    info = unstr64(h[1])
+    rank = int(h[2])
+    shape = tuple(h[3 : 3 + rank])
+    return shape, dtype, info
+
+
+def encode_list(l, infos=None):  # noqa: E741
+    """Given a list of arrays, encode them into a list of byte arrays."""
+    if infos is None:
+        infos = [""]
+    else:
+        if len(l) != len(infos):
+            raise ValueError(f"length of list {l} must muatch length of infos {infos}")
+    result = []
+    for i, a in enumerate(l):
+        header = encode_header(a, infos[i % len(infos)])
+        result += [header, bytedata(a)]
+    return result
+
+
+def decode_list(l, infos=False):  # noqa: E741
+    """Given a list of byte arrays, decode them into arrays."""
+    result = []
+    infos0 = []
+    for header, data in zip(l[::2], l[1::2]):
+        shape, dtype, info = decode_header(header)
+        a = np.frombuffer(data, dtype=dtype, count=np.prod(shape)).reshape(*shape)
+        result += [a]
+        infos0 += [info]
+    return check_infos(result, infos0, infos)
+
+
+magic_str = "~TenBin~"
+magic = str64(magic_str)
+magic_bytes = unstr64(magic).encode("ascii")
+
+
+def roundup(n, k=64):
+    """Round up to the next multiple of 64."""
+    return k * ((n + k - 1) // k)
+
+
+def encode_chunks(l):  # noqa: E741
+    """Encode a list of chunks into a single byte array, with lengths and magics.."""
+    size = sum(16 + roundup(b.nbytes) for b in l)
+    result = bytearray(size)
+    offset = 0
+    for b in l:
+        result[offset : offset + 8] = magic_bytes
+        offset += 8
+        result[offset : offset + 8] = struct.pack("@q", b.nbytes)
+        offset += 8
+        result[offset : offset + bytelen(b)] = b
+        offset += roundup(bytelen(b))
+    return result
+
+
+def decode_chunks(buf):
+    """Decode a byte array into a list of chunks."""
+    result = []
+    offset = 0
+    total = bytelen(buf)
+    while offset < total:
+        if magic_bytes != buf[offset : offset + 8]:
+            raise ValueError("magic bytes mismatch")
+        offset += 8
+        nbytes = struct.unpack("@q", buf[offset : offset + 8])[0]
+        offset += 8
+        b = buf[offset : offset + nbytes]
+        offset += roundup(nbytes)
+        result.append(b)
+    return result
+
+
+def encode_buffer(l, infos=None):  # noqa: E741
+    """Encode a list of arrays into a single byte array."""
+    if not isinstance(l, list):
+        raise ValueError("requires list")
+    return encode_chunks(encode_list(l, infos=infos))
+
+
+def decode_buffer(buf, infos=False):
+    """Decode a byte array into a list of arrays."""
+    return decode_list(decode_chunks(buf), infos=infos)
+
+
+def write_chunk(stream, buf):
+    """Write a byte chunk to the stream with magics, length, and padding."""
+    nbytes = bytelen(buf)
+    stream.write(magic_bytes)
+    stream.write(struct.pack("@q", nbytes))
+    stream.write(bytedata(buf))
+    padding = roundup(nbytes) - nbytes
+    if padding > 0:
+        stream.write(b"\0" * padding)
+
+
+def read_chunk(stream):
+    """Read a byte chunk from a stream with magics, length, and padding."""
+    magic = stream.read(8)
+    if magic == b"":
+        return None
+    if magic != magic_bytes:
+        raise ValueError("magic number does not match")
+    nbytes = stream.read(8)
+    nbytes = struct.unpack("@q", nbytes)[0]
+    if nbytes < 0:
+        raise ValueError("negative nbytes")
+    data = stream.read(nbytes)
+    padding = roundup(nbytes) - nbytes
+    if padding > 0:
+        stream.read(padding)
+    return data
+
+
+def write(stream, l, infos=None):  # noqa: E741
+    """Write a list of arrays to a stream, with magics, length, and padding."""
+    for chunk in encode_list(l, infos=infos):
+        write_chunk(stream, chunk)
+
+
+def read(stream, n=sys.maxsize, infos=False):
+    """Read a list of arrays from a stream, with magics, length, and padding."""
+    chunks = []
+    for _ in range(n):
+        header = read_chunk(stream)
+        if header is None:
+            break
+        data = read_chunk(stream)
+        if data is None:
+            raise ValueError("premature EOF")
+        chunks += [header, data]
+    return decode_list(chunks, infos=infos)
+
+
+def save(fname, *args, infos=None, nocheck=False):
+    """Save a list of arrays to a file, with magics, length, and padding."""
+    if not nocheck and not fname.endswith(".ten"):
+        raise ValueError("file name should end in .ten")
+    with open(fname, "wb") as stream:
+        write(stream, args, infos=infos)
+
+
+def load(fname, infos=False, nocheck=False):
+    """Read a list of arrays from a file, with magics, length, and padding."""
+    if not nocheck and not fname.endswith(".ten"):
+        raise ValueError("file name should end in .ten")
+    with open(fname, "rb") as stream:
+        return read(stream, infos=infos)

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/packaged_modules/webdataset/webdataset.py

@@ -0,0 +1,349 @@
+import io
+import json
+import re
+from itertools import islice
+from typing import Any, Callable
+
+import fsspec
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from datasets.builder import Key
+from datasets.features.features import cast_to_python_objects
+from datasets.utils.file_utils import SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL, xbasename
+
+
+logger = datasets.utils.logging.get_logger(__name__)
+
+
+class WebDataset(datasets.GeneratorBasedBuilder):
+    DEFAULT_WRITER_BATCH_SIZE = 100
+    IMAGE_EXTENSIONS: list[str]  # definition at the bottom of the script
+    AUDIO_EXTENSIONS: list[str]  # definition at the bottom of the script
+    VIDEO_EXTENSIONS: list[str]  # definition at the bottom of the script
+    DECODERS: dict[str, Callable[[Any], Any]]  # definition at the bottom of the script
+    NUM_EXAMPLES_FOR_FEATURES_INFERENCE = 5
+
+    @classmethod
+    def _get_pipeline_from_tar(cls, tar_path, tar_iterator):
+        current_example = {}
+        fs: fsspec.AbstractFileSystem = fsspec.filesystem("memory")
+        streaming_download_manager = datasets.StreamingDownloadManager()
+        for filename, f in tar_iterator:
+            example_key, field_name = base_plus_ext(filename)
+            if example_key is None:
+                continue
+            if current_example and current_example["__key__"] != example_key:
+                # reposition some keys in last position
+                current_example["__key__"] = current_example.pop("__key__")
+                current_example["__url__"] = current_example.pop("__url__")
+                yield current_example
+                current_example = {}
+            current_example["__key__"] = example_key
+            current_example["__url__"] = tar_path
+            current_example[field_name] = f.read()
+            if field_name.split(".")[-1].lower() in SINGLE_FILE_COMPRESSION_EXTENSION_TO_PROTOCOL:
+                fs.write_bytes(filename, current_example[field_name])
+                extracted_file_path = streaming_download_manager.extract(f"memory://{filename}")
+                with fsspec.open(extracted_file_path) as f:
+                    current_example[field_name] = f.read()
+                fs.delete(filename)
+                data_extension = xbasename(extracted_file_path).split(".")[-1].lower()
+            else:
+                data_extension = field_name.split(".")[-1].lower()
+            if data_extension in cls.DECODERS:
+                current_example[field_name] = cls.DECODERS[data_extension](current_example[field_name])
+        if current_example:
+            yield current_example
+
+    def _info(self) -> datasets.DatasetInfo:
+        return datasets.DatasetInfo()
+
+    def _split_generators(self, dl_manager):
+        """We handle string, list and dicts in datafiles"""
+        # Download the data files
+        if not self.config.data_files:
+            raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}")
+        data_files = dl_manager.download(self.config.data_files)
+        splits = []
+        for split_name, tar_paths in data_files.items():
+            tar_iterators = [dl_manager.iter_archive(tar_path) for tar_path in tar_paths]
+            splits.append(
+                datasets.SplitGenerator(
+                    name=split_name, gen_kwargs={"tar_paths": tar_paths, "tar_iterators": tar_iterators}
+                )
+            )
+        if not self.info.features:
+            # Get one example to get the feature types
+            pipeline = self._get_pipeline_from_tar(tar_paths[0], tar_iterators[0])
+            first_examples = list(islice(pipeline, self.NUM_EXAMPLES_FOR_FEATURES_INFERENCE))
+            if any(example.keys() != first_examples[0].keys() for example in first_examples):
+                raise ValueError(
+                    "The TAR archives of the dataset should be in WebDataset format, "
+                    "but the files in the archive don't share the same prefix or the same types."
+                )
+            pa_tables = [
+                pa.Table.from_pylist(cast_to_python_objects([example], only_1d_for_numpy=True))
+                for example in first_examples
+            ]
+            inferred_arrow_schema = pa.concat_tables(pa_tables, promote_options="default").schema
+            features = datasets.Features.from_arrow_schema(inferred_arrow_schema)
+
+            for field_name in first_examples[0]:
+                extension = field_name.rsplit(".", 1)[-1].lower()
+                # Set Image types
+                if extension in self.IMAGE_EXTENSIONS:
+                    features[field_name] = datasets.Image()
+                # Set Audio types
+                if extension in self.AUDIO_EXTENSIONS:
+                    features[field_name] = datasets.Audio()
+                # Set Video types
+                if extension in self.VIDEO_EXTENSIONS:
+                    features[field_name] = datasets.Video()
+            self.info.features = features
+
+        return splits
+
+    def _generate_shards(self, tar_paths, tar_iterators):
+        yield from tar_paths
+
+    def _generate_examples(self, tar_paths, tar_iterators):
+        image_field_names = [
+            field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Image)
+        ]
+        audio_field_names = [
+            field_name for field_name, feature in self.info.features.items() if isinstance(feature, datasets.Audio)
+        ]
+        all_field_names = list(self.info.features.keys())
+        for tar_idx, (tar_path, tar_iterator) in enumerate(zip(tar_paths, tar_iterators)):
+            for example_idx, example in enumerate(self._get_pipeline_from_tar(tar_path, tar_iterator)):
+                for field_name in all_field_names:
+                    if field_name not in example:
+                        example[field_name] = None
+                for field_name in image_field_names + audio_field_names:
+                    if example[field_name] is not None:
+                        example[field_name] = {
+                            "path": example["__key__"] + "." + field_name,
+                            "bytes": example[field_name],
+                        }
+                yield Key(tar_idx, example_idx), example
+
+
+# Source: https://github.com/webdataset/webdataset/blob/87bd5aa41602d57f070f65a670893ee625702f2f/webdataset/tariterators.py#L25
+def base_plus_ext(path):
+    """Split off all file extensions.
+
+    Returns base, allext.
+    """
+    match = re.match(r"^((?:.*/|)[^.]+)[.]([^/]*)$", path)
+    if not match:
+        return None, None
+    return match.group(1), match.group(2)
+
+
+# Obtained with:
+# ```
+# import PIL.Image
+# IMAGE_EXTENSIONS = []
+# PIL.Image.init()
+# for ext, format in PIL.Image.EXTENSION.items():
+#     if format in PIL.Image.OPEN:
+#         IMAGE_EXTENSIONS.append(ext[1:])
+# ```
+# We intentionally do not run this code on launch because:
+# (1) Pillow is an optional dependency, so importing Pillow in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+IMAGE_EXTENSIONS = [
+    "blp",
+    "bmp",
+    "dib",
+    "bufr",
+    "cur",
+    "pcx",
+    "dcx",
+    "dds",
+    "ps",
+    "eps",
+    "fit",
+    "fits",
+    "fli",
+    "flc",
+    "ftc",
+    "ftu",
+    "gbr",
+    "gif",
+    "grib",
+    "h5",
+    "hdf",
+    "png",
+    "apng",
+    "jp2",
+    "j2k",
+    "jpc",
+    "jpf",
+    "jpx",
+    "j2c",
+    "icns",
+    "ico",
+    "im",
+    "iim",
+    "tif",
+    "tiff",
+    "jfif",
+    "jpe",
+    "jpg",
+    "jpeg",
+    "mpg",
+    "mpeg",
+    "msp",
+    "pcd",
+    "pxr",
+    "pbm",
+    "pgm",
+    "ppm",
+    "pnm",
+    "psd",
+    "bw",
+    "rgb",
+    "rgba",
+    "sgi",
+    "ras",
+    "tga",
+    "icb",
+    "vda",
+    "vst",
+    "webp",
+    "wmf",
+    "emf",
+    "xbm",
+    "xpm",
+]
+WebDataset.IMAGE_EXTENSIONS = IMAGE_EXTENSIONS
+
+
+# Obtained with:
+# ```
+# import soundfile as sf
+#
+# AUDIO_EXTENSIONS = [f".{format.lower()}" for format in sf.available_formats().keys()]
+#
+# # .opus decoding is supported if libsndfile >= 1.0.31:
+# AUDIO_EXTENSIONS.extend([".mp3", ".opus"])
+# ```
+# We intentionally did not run this code on launch because:
+# (1) Soundfile was an optional dependency, so importing it in global namespace is not allowed
+# (2) To ensure the list of supported extensions is deterministic
+# (3) We use TorchCodec now anyways instead of Soundfile
+AUDIO_EXTENSIONS = [
+    "aiff",
+    "au",
+    "avr",
+    "caf",
+    "flac",
+    "htk",
+    "svx",
+    "mat4",
+    "mat5",
+    "mpc2k",
+    "ogg",
+    "paf",
+    "pvf",
+    "raw",
+    "rf64",
+    "sd2",
+    "sds",
+    "ircam",
+    "voc",
+    "w64",
+    "wav",
+    "nist",
+    "wavex",
+    "wve",
+    "xi",
+    "mp3",
+    "opus",
+]
+WebDataset.AUDIO_EXTENSIONS = AUDIO_EXTENSIONS
+
+
+# TODO: initial list, we should check the compatibility of other formats
+VIDEO_EXTENSIONS = [
+    "mkv",
+    "mp4",
+    "avi",
+    "mpeg",
+    "mov",
+]
+WebDataset.VIDEO_EXTENSIONS = VIDEO_EXTENSIONS
+
+
+def text_loads(data: bytes):
+    return data.decode("utf-8")
+
+
+def tenbin_loads(data: bytes):
+    from . import _tenbin
+
+    return _tenbin.decode_buffer(data)
+
+
+def msgpack_loads(data: bytes):
+    import msgpack
+
+    return msgpack.unpackb(data)
+
+
+def npy_loads(data: bytes):
+    import numpy.lib.format
+
+    stream = io.BytesIO(data)
+    return numpy.lib.format.read_array(stream, allow_pickle=False)
+
+
+def npz_loads(data: bytes):
+    return np.load(io.BytesIO(data), allow_pickle=False)
+
+
+def cbor_loads(data: bytes):
+    import cbor
+
+    return cbor.loads(data)
+
+
+def torch_loads(data: bytes):
+    import torch
+
+    return torch.load(io.BytesIO(data), weights_only=True)
+
+
+# Obtained by checking `decoders` in `webdataset.autodecode`
+# and removing unsafe extension decoders.
+# Removed Pickle decoders:
+# - "pyd": lambda data: pickle.loads(data)
+# - "pickle": lambda data: pickle.loads(data)
+# Modified NumPy decoders to fix CVE-2019-6446 (add allow_pickle=False and weights_only=True):
+# - "npy": npy_loads,
+# - "npz": lambda data: np.load(io.BytesIO(data)),
+# - "pth": lambda data: torch_loads(data)
+DECODERS = {
+    "txt": text_loads,
+    "text": text_loads,
+    "transcript": text_loads,
+    "cls": int,
+    "cls2": int,
+    "index": int,
+    "inx": int,
+    "id": int,
+    "json": json.loads,
+    "jsn": json.loads,
+    "ten": tenbin_loads,
+    "tb": tenbin_loads,
+    "mp": msgpack_loads,
+    "msg": msgpack_loads,
+    "npy": npy_loads,
+    "npz": npz_loads,
+    "cbor": cbor_loads,
+    "pth": torch_loads,
+}
+WebDataset.DECODERS = DECODERS

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/parallel/__init__.py

@@ -0,0 +1 @@
+from .parallel import ParallelBackendConfig, parallel_backend, parallel_map

bioagent-bench/.venv/lib/python3.14/site-packages/datasets/parallel/parallel.py

@@ -0,0 +1,120 @@
+import contextlib
+from multiprocessing import Pool, RLock
+
+from tqdm.auto import tqdm
+
+from ..utils import experimental, logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class ParallelBackendConfig:
+    backend_name = None
+
+
+@experimental
+def parallel_map(function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func):
+    """
+    **Experimental.** Apply a function to iterable elements in parallel, where the implementation uses either
+    multiprocessing.Pool or joblib for parallelization.
+
+    Args:
+        function (`Callable[[Any], Any]`): Function to be applied to `iterable`.
+        iterable (`list`, `tuple` or `np.ndarray`): Iterable elements to apply function to.
+        num_proc (`int`): Number of processes (if no backend specified) or jobs (using joblib).
+        types (`tuple`): Additional types (besides `dict` values) to apply `function` recursively to their elements.
+        disable_tqdm (`bool`): Whether to disable the tqdm progressbar.
+        desc (`str`): Prefix for the tqdm progressbar.
+        single_map_nested_func (`Callable`): Map function that applies `function` to an element from `iterable`.
+            Takes a tuple of function, data_struct, types, rank, disable_tqdm, desc as input, where data_struct is an
+            element of `iterable`, and `rank` is used for progress bar.
+    """
+    if ParallelBackendConfig.backend_name is None:
+        return _map_with_multiprocessing_pool(
+            function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+        )
+
+    return _map_with_joblib(
+        function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+    )
+
+
+def _map_with_multiprocessing_pool(
+    function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+):
+    num_proc = num_proc if num_proc <= len(iterable) else len(iterable)
+    split_kwds = []  # We organize the splits ourselve (contiguous splits)
+    for index in range(num_proc):
+        div = len(iterable) // num_proc
+        mod = len(iterable) % num_proc
+        start = div * index + min(index, mod)
+        end = start + div + (1 if index < mod else 0)
+        split_kwds.append((function, iterable[start:end], batched, batch_size, types, index, disable_tqdm, desc))
+
+    if len(iterable) != sum(len(i[1]) for i in split_kwds):
+        raise ValueError(
+            f"Error dividing inputs iterable among processes. "
+            f"Total number of objects {len(iterable)}, "
+            f"length: {sum(len(i[1]) for i in split_kwds)}"
+        )
+
+    logger.info(
+        f"Spawning {num_proc} processes for {len(iterable)} objects in slices of {[len(i[1]) for i in split_kwds]}"
+    )
+    initargs, initializer = None, None
+    if not disable_tqdm:
+        initargs, initializer = (RLock(),), tqdm.set_lock
+    with Pool(num_proc, initargs=initargs, initializer=initializer) as pool:
+        mapped = pool.map(single_map_nested_func, split_kwds)
+    logger.info(f"Finished {num_proc} processes")
+    mapped = [obj for proc_res in mapped for obj in proc_res]
+    logger.info(f"Unpacked {len(mapped)} objects")
+
+    return mapped
+
+
+def _map_with_joblib(
+    function, iterable, num_proc, batched, batch_size, types, disable_tqdm, desc, single_map_nested_func
+):
+    # progress bar is not yet supported for _map_with_joblib, because tqdm couldn't accurately be applied to joblib,
+    # and it requires monkey-patching joblib internal classes which is subject to change
+    import joblib
+
+    with joblib.parallel_backend(ParallelBackendConfig.backend_name, n_jobs=num_proc):
+        return joblib.Parallel()(
+            joblib.delayed(single_map_nested_func)((function, obj, batched, batch_size, types, None, True, None))
+            for obj in iterable
+        )
+
+
+@experimental
+@contextlib.contextmanager
+def parallel_backend(backend_name: str):
+    """
+    **Experimental.**  Configures the parallel backend for parallelized dataset loading, which uses the parallelization
+    implemented by joblib.
+
+    Args:
+        backend_name (str): Name of backend for parallelization implementation, has to be supported by joblib.
+
+     Example usage:
+     ```py
+     with parallel_backend('spark'):
+       dataset = load_dataset(..., num_proc=2)
+     ```
+    """
+    ParallelBackendConfig.backend_name = backend_name
+
+    if backend_name == "spark":
+        from joblibspark import register_spark
+
+        register_spark()
+
+        # TODO: call create_cache_and_write_probe if "download" in steps
+        # TODO: raise NotImplementedError when Dataset.map etc is called
+
+    try:
+        yield
+    finally:
+        ParallelBackendConfig.backend_name = None

bioagent-bench/.venv/lib/python3.14/site-packages/dill-0.4.1.dist-info/INSTALLER

@@ -0,0 +1 @@
+uv

bioagent-bench/.venv/lib/python3.14/site-packages/dill-0.4.1.dist-info/LICENSE

@@ -0,0 +1,35 @@
+Copyright (c) 2004-2016 California Institute of Technology.
+Copyright (c) 2016-2026 The Uncertainty Quantification Foundation.
+All rights reserved.
+
+This software is available subject to the conditions and terms laid
+out below. By downloading and using this software you are agreeing
+to the following conditions.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+    - Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    - Redistributions in binary form must reproduce the above copyright
+      notice, this list of conditions and the following disclaimer in the
+      documentation and/or other materials provided with the distribution.
+
+    - Neither the names of the copyright holders nor the names of any of
+      the contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

bioagent-bench/.venv/lib/python3.14/site-packages/dill-0.4.1.dist-info/METADATA

@@ -0,0 +1,281 @@
+Metadata-Version: 2.1
+Name: dill
+Version: 0.4.1
+Summary: serialize all of Python
+Home-page: https://github.com/uqfoundation/dill
+Download-URL: https://pypi.org/project/dill/#files
+Author: Mike McKerns
+Author-email: mmckerns@uqfoundation.org
+Maintainer: Mike McKerns
+Maintainer-email: mmckerns@uqfoundation.org
+License: BSD-3-Clause
+Project-URL: Documentation, http://dill.rtfd.io
+Project-URL: Source Code, https://github.com/uqfoundation/dill
+Project-URL: Bug Tracker, https://github.com/uqfoundation/dill/issues
+Platform: Linux
+Platform: Windows
+Platform: Mac
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Software Development
+Requires-Python: >=3.9
+License-File: LICENSE
+Provides-Extra: readline
+Provides-Extra: graph
+Requires-Dist: objgraph>=1.7.2; extra == "graph"
+Provides-Extra: profile
+Requires-Dist: gprof2dot>=2022.7.29; extra == "profile"
+
+-----------------------------
+dill: serialize all of Python
+-----------------------------
+
+About Dill
+==========
+
+``dill`` extends Python's ``pickle`` module for serializing and de-serializing
+Python objects to the majority of the built-in Python types. Serialization
+is the process of converting an object to a byte stream, and the inverse
+of which is converting a byte stream back to a Python object hierarchy.
+
+``dill`` provides the user the same interface as the ``pickle`` module, and
+also includes some additional features. In addition to pickling Python
+objects, ``dill`` provides the ability to save the state of an interpreter
+session in a single command.  Hence, it would be feasible to save an
+interpreter session, close the interpreter, ship the pickled file to
+another computer, open a new interpreter, unpickle the session and
+thus continue from the 'saved' state of the original interpreter
+session.
+
+``dill`` can be used to store Python objects to a file, but the primary
+usage is to send Python objects across the network as a byte stream.
+``dill`` is quite flexible, and allows arbitrary user defined classes
+and functions to be serialized.  Thus ``dill`` is not intended to be
+secure against erroneously or maliciously constructed data. It is
+left to the user to decide whether the data they unpickle is from
+a trustworthy source.
+
+``dill`` is part of ``pathos``, a Python framework for heterogeneous computing.
+``dill`` is in active development, so any user feedback, bug reports, comments,
+or suggestions are highly appreciated.  A list of issues is located at
+https://github.com/uqfoundation/dill/issues, with a legacy list maintained at
+https://uqfoundation.github.io/project/pathos/query.
+
+
+Major Features
+==============
+
+``dill`` can pickle the following standard types:
+
+    - none, type, bool, int, float, complex, bytes, str,
+    - tuple, list, dict, file, buffer, builtin,
+    - Python classes, namedtuples, dataclasses, metaclasses,
+    - instances of classes,
+    - set, frozenset, array, functions, exceptions
+
+``dill`` can also pickle more 'exotic' standard types:
+
+    - functions with yields, nested functions, lambdas,
+    - cell, method, unboundmethod, module, code, methodwrapper,
+    - methoddescriptor, getsetdescriptor, memberdescriptor, wrapperdescriptor,
+    - dictproxy, slice, notimplemented, ellipsis, quit
+
+``dill`` cannot yet pickle these standard types:
+
+    - frame, generator, traceback
+
+``dill`` also provides the capability to:
+
+    - save and load Python interpreter sessions
+    - save and extract the source code from functions and classes
+    - interactively diagnose pickling errors
+
+
+Current Release
+===============
+
+The latest released version of ``dill`` is available from:
+
+    https://pypi.org/project/dill
+
+``dill`` is distributed under a 3-clause BSD license.
+
+
+Development Version
+===================
+
+You can get the latest development version with all the shiny new features at:
+
+    https://github.com/uqfoundation
+
+If you have a new contribution, please submit a pull request.
+
+
+Installation
+============
+
+``dill`` can be installed with ``pip``::
+
+    $ pip install dill
+
+To optionally include the ``objgraph`` diagnostic tool in the install::
+
+    $ pip install dill[graph]
+
+To optionally include the ``gprof2dot`` diagnostic tool in the install::
+
+    $ pip install dill[profile]
+
+For windows users, to optionally install session history tools::
+
+    $ pip install dill[readline]
+
+
+Requirements
+============
+
+``dill`` requires:
+
+    - ``python`` (or ``pypy``), **>=3.9**
+    - ``setuptools``, **>=42**
+
+Optional requirements:
+
+    - ``objgraph``, **>=1.7.2**
+    - ``gprof2dot``, **>=2022.7.29**
+    - ``pyreadline``, **>=1.7.1** (on windows)
+
+
+Basic Usage
+===========
+
+``dill`` is a drop-in replacement for ``pickle``. Existing code can be
+updated to allow complete pickling using::
+
+    >>> import dill as pickle
+
+or::
+
+    >>> from dill import dumps, loads
+
+``dumps`` converts the object to a unique byte string, and ``loads`` performs
+the inverse operation::
+
+    >>> squared = lambda x: x**2
+    >>> loads(dumps(squared))(3)
+    9
+
+There are a number of options to control serialization which are provided
+as keyword arguments to several ``dill`` functions:
+
+* with *protocol*, the pickle protocol level can be set. This uses the
+  same value as the ``pickle`` module, *DEFAULT_PROTOCOL*.
+* with *byref=True*, ``dill`` to behave a lot more like pickle with
+  certain objects (like modules) pickled by reference as opposed to
+  attempting to pickle the object itself.
+* with *recurse=True*, objects referred to in the global dictionary are
+  recursively traced and pickled, instead of the default behavior of
+  attempting to store the entire global dictionary.
+* with *fmode*, the contents of the file can be pickled along with the file
+  handle, which is useful if the object is being sent over the wire to a
+  remote system which does not have the original file on disk. Options are
+  *HANDLE_FMODE* for just the handle, *CONTENTS_FMODE* for the file content
+  and *FILE_FMODE* for content and handle.
+* with *ignore=False*, objects reconstructed with types defined in the
+  top-level script environment use the existing type in the environment
+  rather than a possibly different reconstructed type.
+
+The default serialization can also be set globally in *dill.settings*.
+Thus, we can modify how ``dill`` handles references to the global dictionary
+locally or globally::
+
+    >>> import dill.settings
+    >>> dumps(absolute) == dumps(absolute, recurse=True)
+    False
+    >>> dill.settings['recurse'] = True
+    >>> dumps(absolute) == dumps(absolute, recurse=True)
+    True
+
+``dill`` also includes source code inspection, as an alternate to pickling::
+
+    >>> import dill.source
+    >>> print(dill.source.getsource(squared))
+    squared = lambda x:x**2
+
+To aid in debugging pickling issues, use *dill.detect* which provides
+tools like pickle tracing::
+
+    >>> import dill.detect
+    >>> with dill.detect.trace():
+    >>>     dumps(squared)
+    ┬ F1: <function <lambda> at 0x7fe074f8c280>
+    ├┬ F2: <function _create_function at 0x7fe074c49c10>
+    │└ # F2 [34 B]
+    ├┬ Co: <code object <lambda> at 0x7fe07501eb30, file "<stdin>", line 1>
+    │├┬ F2: <function _create_code at 0x7fe074c49ca0>
+    ││└ # F2 [19 B]
+    │└ # Co [87 B]
+    ├┬ D1: <dict object at 0x7fe0750d4680>
+    │└ # D1 [22 B]
+    ├┬ D2: <dict object at 0x7fe074c5a1c0>
+    │└ # D2 [2 B]
+    ├┬ D2: <dict object at 0x7fe074f903c0>
+    │├┬ D2: <dict object at 0x7fe074f8ebc0>
+    ││└ # D2 [2 B]
+    │└ # D2 [23 B]
+    └ # F1 [180 B]
+
+With trace, we see how ``dill`` stored the lambda (``F1``) by first storing
+``_create_function``, the underlying code object (``Co``) and ``_create_code``
+(which is used to handle code objects), then we handle the reference to
+the global dict (``D2``) plus other dictionaries (``D1`` and ``D2``) that
+save the lambda object's state. A ``#`` marks when the object is actually stored.
+
+
+More Information
+================
+
+Probably the best way to get started is to look at the documentation at
+http://dill.rtfd.io. Also see ``dill.tests`` for a set of scripts that
+demonstrate how ``dill`` can serialize different Python objects. You can
+run the test suite with ``python -m dill.tests``. The contents of any
+pickle file can be examined with ``undill``.  As ``dill`` conforms to
+the ``pickle`` interface, the examples and documentation found at
+http://docs.python.org/library/pickle.html also apply to ``dill``
+if one will ``import dill as pickle``. The source code is also generally
+well documented, so further questions may be resolved by inspecting the
+code itself. Please feel free to submit a ticket on github, or ask a
+question on stackoverflow (**@Mike McKerns**).
+If you would like to share how you use ``dill`` in your work, please send
+an email (to **mmckerns at uqfoundation dot org**).
+
+
+Citation
+========
+
+If you use ``dill`` to do research that leads to publication, we ask that you
+acknowledge use of ``dill`` by citing the following in your publication::
+
+    M.M. McKerns, L. Strand, T. Sullivan, A. Fang, M.A.G. Aivazis,
+    "Building a framework for predictive science", Proceedings of
+    the 10th Python in Science Conference, 2011;
+    http://arxiv.org/pdf/1202.1056
+
+    Michael McKerns and Michael Aivazis,
+    "pathos: a framework for heterogeneous computing", 2010- ;
+    https://uqfoundation.github.io/project/pathos
+
+Please see https://uqfoundation.github.io/project/pathos or
+http://arxiv.org/pdf/1202.1056 for further information.

bioagent-bench/.venv/lib/python3.14/site-packages/dill-0.4.1.dist-info/RECORD

@@ -0,0 +1,56 @@
+../../../bin/get_gprof,sha256=lpXb6xvSUtU9lKE9Dw4Zy35hBLg0RsQ-01l9KNrgDuQ,2491
+../../../bin/get_objgraph,sha256=U9yru_IE03LwChLwS6_nOZovQ29eQEb_4T92bzSYi7c,1685
+../../../bin/undill,sha256=-J0asuqKiqJjosezK0OAgF4PX5gNaUjMfLQvxciY0iA,621
+dill-0.4.1.dist-info/INSTALLER,sha256=5hhM4Q4mYTT9z6QB6PGpUAW81PGNFrYrdXMj4oM_6ak,2
+dill-0.4.1.dist-info/LICENSE,sha256=9mnWO-v3ZbqqCyDYsMEZMpan4OoMIsUKMIt012qzk5w,1790
+dill-0.4.1.dist-info/METADATA,sha256=BA1shO3fKcjBTcKu8hSWKpGYA_VXJDgk3U-Ko7SeOGg,10171
+dill-0.4.1.dist-info/RECORD,,
+dill-0.4.1.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+dill-0.4.1.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+dill-0.4.1.dist-info/top_level.txt,sha256=HLSIyYIjQzJiBvs3_-16ntezE3j6mWGTW0DT1xDd7X0,5
+dill/__diff.py,sha256=7XCqWIelTWSLjieOIu3Xtr1mUui9YBmltP7-tvtrG-Q,7146
+dill/__info__.py,sha256=EGg2Nh84NhPnYq1J2j6EAPixBb3psVxY-5rAWzr8prk,10756
+dill/__init__.py,sha256=Y8usnzpfsclJxT6TvRYGarEXkWEXprv4lC2ZJ0bGM2Y,3798
+dill/_dill.py,sha256=NdaVyGzOb5S57abYN0eFXcogocHQrTBkUru7cqw2fwM,91730
+dill/_objects.py,sha256=_Xc7OqnZesDmKBmtkbkrBBMDzH3TIFy_vtMRtc7lsSI,20015
+dill/_shims.py,sha256=kkIa7muWHHp9uItpNffqmbhBO4MEKPIdeyHOKxx2VF4,6635
+dill/detect.py,sha256=VbCyNMaIVudiqM1nLmGwS-yXW6M9CkdpbkVarAmnJqU,11207
+dill/logger.py,sha256=Z1wgU8BbSAEWBKoucYRHYrFhVLqqqTTi-BHcfsebi88,11143
+dill/objtypes.py,sha256=3hDEnRS040mVyA3nMJp0seIiAw9SafxbuvG5tFvthfE,736
+dill/pointers.py,sha256=zeX8XqlLBIW9vL-XH_er9HUWmInlbfWjYtva56eTmww,4467
+dill/session.py,sha256=88DJFIQrA8733P-VKkbCq0d1lyJzckQYCrGxcPeSKeE,23541
+dill/settings.py,sha256=DRFp2QC8xspPZ5Sb3eIB6abPp1wNuBNtCEjZ8DLY-x4,630
+dill/source.py,sha256=GsmYf6bEPLM9rivz1Ec7G2E-tEgAoXI95CisqdcQGGA,45710
+dill/temp.py,sha256=eFyMgZIJRUVgdRi2a4I2tZd5g3HSUFnfm5K3LV2Z93k,8027
+dill/tests/__init__.py,sha256=8OmXwJHyWK1axs4XwCD4TimAE1U_oF6S1ZIPJ8tI8Dg,479
+dill/tests/__main__.py,sha256=6jkJVIcrIHqHjyKOoDr95npJz4eASssTGKbuhWEqMow,899
+dill/tests/test_abc.py,sha256=QJ5rW4-HPG0mj-K-96zwASOC-LqGa0XnzDTNW43Cnr4,4227
+dill/tests/test_check.py,sha256=h567c9d3L-O5lc4H8lBsN5St4NkQr5Em9eAU-0l8xYA,1396
+dill/tests/test_classdef.py,sha256=UTctS9spat_KwiubR2-05hVbMvmT5_BqZJWReDC34lM,8600
+dill/tests/test_dataclasses.py,sha256=btxL2-lqjFOLSDf8imLZAAQ4bkm9xQwvS2VlPiIrlQY,890
+dill/tests/test_detect.py,sha256=pm55LV1oi3xr6zmawc2pBlzSMSMkh4TwV7IuoAh3r8M,4329
+dill/tests/test_dictviews.py,sha256=aRqVTK5DOuIoKzxWQcPenSRnvCqLyYeM2jIQsn32Tqo,1337
+dill/tests/test_diff.py,sha256=cvfUtLcJOqlH0VcEmuIHQf69194Jy7vWoEk8pO-GLKM,2667
+dill/tests/test_extendpickle.py,sha256=kYtoA2gr_CLo7NRdyu2fwOdPMt8yp4kVzd9hki7pZBk,1315
+dill/tests/test_fglobals.py,sha256=dSyPOYarpytrnb723m3nAJqGp3zeTFovA7wfoZr6Z_o,1676
+dill/tests/test_file.py,sha256=OsM189KoskuN84bag6GbYJ4OYcTy5WlHSlMWj0j0BhQ,13578
+dill/tests/test_functions.py,sha256=40QuhQIqV-jaDvuR5pq0_Haf7Pmd9M3DYJDs2oSxjiU,4267
+dill/tests/test_functors.py,sha256=iLv95xcaTasuqZh5LQBLa6AWrPeCxFRb-TLzhNzrMB4,930
+dill/tests/test_logger.py,sha256=W0QJMVMQOUOizGb1sNsKInNVS76XqrM7UFyEX-Ei_bs,2385
+dill/tests/test_mixins.py,sha256=tlCp5-NYObO7VIiK0YkciB7yRqepRJPd_RYAc79Zcvw,4007
+dill/tests/test_module.py,sha256=LVdwwdkdbRofoEHhOfszi7oVEUoE__YZyyJzRyza_Ws,1943
+dill/tests/test_moduledict.py,sha256=s5siniJxn78G83lPiQnx4eYV6Z6isCU3m2phgRTtlVA,1182
+dill/tests/test_nested.py,sha256=EuyAWkXGuERTaUaPQDOxhO1cvDhCDWPML3vYOQ0HpFg,3146
+dill/tests/test_objects.py,sha256=6Gy-YuVDuX1efV_pQEbCen7ZpMxTazY9Ihq7y1otesQ,1931
+dill/tests/test_properties.py,sha256=ex9gCfebSawz6n5RGLS4NK5tyC7zjULJBYqPSl33Yqs,1346
+dill/tests/test_pycapsule.py,sha256=EJFsKZ0kTWi17YOQ0wwaoglXP9XqDS83EYIr1-ZsNUk,1417
+dill/tests/test_recursive.py,sha256=5zwJ62LHCtd3lFsFX7__XqflV78QKwuv-hnvBISoiVs,4182
+dill/tests/test_registered.py,sha256=fO2J8t1VlNFd6ecUtd8akgjl4nRy5TldZbnBEzymtWE,1573
+dill/tests/test_restricted.py,sha256=MG6gWcyn1X2kMNL7hX40iTZNbtre8-6QwneTt6EW7eE,783
+dill/tests/test_selected.py,sha256=1lr05NaRqy27XV-xj5A93Mh8vuOEVXbZZ_ELgjyBPl0,3069
+dill/tests/test_session.py,sha256=fILaOGfRHdMd7wED2fqkRHkHlRLY13TaTnvWIcbP8ic,10161
+dill/tests/test_source.py,sha256=3QnPpuQovYz0wAxzH7U4LWaGWI0sejOra8qPRgGj8W0,7311
+dill/tests/test_sources.py,sha256=QWX_Oc3NjpiMZqLLIJsXpK7kGGOYFDv47rCt7HIX_7s,8672
+dill/tests/test_temp.py,sha256=81i8qP5zFbX7bBlXBiYMDlm_p5GTvze6zVtAPifaD_g,2619
+dill/tests/test_threads.py,sha256=a8GRhk81_gE-TsLX21BG8Jb85vRaDd8All2AafqKzHA,1257
+dill/tests/test_weakref.py,sha256=vKT130ADMeqzhD3vx8KZnF5zvvkhTAaq8WD1tyACX2Y,1602

bioagent-bench/.venv/lib/python3.14/site-packages/dill-0.4.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (75.6.0)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

bioagent-bench/.venv/lib/python3.14/site-packages/dill-0.4.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+dill

bioagent-bench/.venv/lib/python3.14/site-packages/pandas/__init__.py

@@ -0,0 +1,353 @@
+from __future__ import annotations
+
+__docformat__ = "restructuredtext"
+
+# Let users know if they're missing any of our hard dependencies
+# except tzdata (see https://github.com/pandas-dev/pandas/issues/63264)
+_hard_dependencies = ("numpy", "dateutil")
+
+for _dependency in _hard_dependencies:
+    try:
+        __import__(_dependency)
+    except ImportError as _e:  # pragma: no cover
+        raise ImportError(
+            f"Unable to import required dependency {_dependency}. "
+            "Please see the traceback for details."
+        ) from _e
+
+del _hard_dependencies, _dependency
+
+try:
+    # numpy compat
+    from pandas.compat import (
+        is_numpy_dev as _is_numpy_dev,  # pyright: ignore[reportUnusedImport] # noqa: F401
+    )
+except ImportError as _err:  # pragma: no cover
+    _module = _err.name
+    raise ImportError(
+        f"C extension: {_module} not built. If you want to import "
+        "pandas from the source directory, you may need to run "
+        "'python -m pip install -ve . --no-build-isolation -Ceditable-verbose=true' "
+        "to build the C extensions first."
+    ) from _err
+
+from pandas._config import (
+    get_option,
+    set_option,
+    reset_option,
+    describe_option,
+    option_context,
+    options,
+)
+
+# let init-time option registration happen
+import pandas.core.config_init  # pyright: ignore[reportUnusedImport] # noqa: F401
+
+from pandas.core.api import (
+    # dtype
+    ArrowDtype,
+    Int8Dtype,
+    Int16Dtype,
+    Int32Dtype,
+    Int64Dtype,
+    UInt8Dtype,
+    UInt16Dtype,
+    UInt32Dtype,
+    UInt64Dtype,
+    Float32Dtype,
+    Float64Dtype,
+    CategoricalDtype,
+    PeriodDtype,
+    IntervalDtype,
+    DatetimeTZDtype,
+    StringDtype,
+    BooleanDtype,
+    # missing
+    NA,
+    isna,
+    isnull,
+    notna,
+    notnull,
+    # indexes
+    Index,
+    CategoricalIndex,
+    RangeIndex,
+    MultiIndex,
+    IntervalIndex,
+    TimedeltaIndex,
+    DatetimeIndex,
+    PeriodIndex,
+    IndexSlice,
+    # tseries
+    NaT,
+    Period,
+    period_range,
+    Timedelta,
+    timedelta_range,
+    Timestamp,
+    date_range,
+    bdate_range,
+    Interval,
+    interval_range,
+    DateOffset,
+    # conversion
+    to_numeric,
+    to_datetime,
+    to_timedelta,
+    # misc
+    Flags,
+    Grouper,
+    factorize,
+    unique,
+    NamedAgg,
+    array,
+    Categorical,
+    set_eng_float_format,
+    Series,
+    DataFrame,
+)
+from pandas.core.col import col
+
+from pandas.core.dtypes.dtypes import SparseDtype
+
+from pandas.tseries.api import infer_freq
+from pandas.tseries import offsets
+
+from pandas.core.computation.api import eval
+
+from pandas.core.reshape.api import (
+    concat,
+    lreshape,
+    melt,
+    wide_to_long,
+    merge,
+    merge_asof,
+    merge_ordered,
+    crosstab,
+    pivot,
+    pivot_table,
+    get_dummies,
+    from_dummies,
+    cut,
+    qcut,
+)
+
+from pandas import api, arrays, errors, io, plotting, tseries
+from pandas import testing
+from pandas.util._print_versions import show_versions
+
+from pandas.io.api import (
+    # excel
+    ExcelFile,
+    ExcelWriter,
+    read_excel,
+    # parsers
+    read_csv,
+    read_fwf,
+    read_table,
+    # pickle
+    read_pickle,
+    to_pickle,
+    # pytables
+    HDFStore,
+    read_hdf,
+    # sql
+    read_sql,
+    read_sql_query,
+    read_sql_table,
+    # misc
+    read_clipboard,
+    read_parquet,
+    read_orc,
+    read_feather,
+    read_html,
+    read_xml,
+    read_json,
+    read_stata,
+    read_sas,
+    read_spss,
+    read_iceberg,
+)
+
+from pandas.io.json._normalize import json_normalize
+
+from pandas.util._tester import test
+
+# use the closest tagged version if possible
+_built_with_meson = False
+try:
+    from pandas._version_meson import (  # pyright: ignore [reportMissingImports]
+        __version__,
+        __git_version__,
+    )
+
+    _built_with_meson = True
+except ImportError:
+    from pandas._version import get_versions
+
+    v = get_versions()
+    __version__ = v.get("closest-tag", v["version"])
+    __git_version__ = v.get("full-revisionid")
+    del get_versions, v
+
+
+# module level doc-string
+__doc__ = """
+pandas - a powerful data analysis and manipulation library for Python
+=====================================================================
+
+**pandas** is a Python package providing fast, flexible, and expressive data
+structures designed to make working with "relational" or "labeled" data both
+easy and intuitive. It aims to be the fundamental high-level building block for
+doing practical, **real world** data analysis in Python. Additionally, it has
+the broader goal of becoming **the most powerful and flexible open source data
+analysis / manipulation tool available in any language**. It is already well on
+its way toward this goal.
+
+Main Features
+-------------
+Here are just a few of the things that pandas does well:
+
+  - Easy handling of missing data in floating point as well as non-floating
+    point data.
+  - Size mutability: columns can be inserted and deleted from DataFrame and
+    higher dimensional objects
+  - Automatic and explicit data alignment: objects can be explicitly aligned
+    to a set of labels, or the user can simply ignore the labels and let
+    `Series`, `DataFrame`, etc. automatically align the data for you in
+    computations.
+  - Powerful, flexible group by functionality to perform split-apply-combine
+    operations on data sets, for both aggregating and transforming data.
+  - Make it easy to convert ragged, differently-indexed data in other Python
+    and NumPy data structures into DataFrame objects.
+  - Intelligent label-based slicing, fancy indexing, and subsetting of large
+    data sets.
+  - Intuitive merging and joining data sets.
+  - Flexible reshaping and pivoting of data sets.
+  - Hierarchical labeling of axes (possible to have multiple labels per tick).
+  - Robust IO tools for loading data from flat files (CSV and delimited),
+    Excel files, databases, and saving/loading data from the ultrafast HDF5
+    format.
+  - Time series-specific functionality: date range generation and frequency
+    conversion, moving window statistics, date shifting and lagging.
+"""
+
+# Use __all__ to let type checkers know what is part of the public API.
+# Pandas is not (yet) a py.typed library: the public API is determined
+# based on the documentation.
+__all__ = [
+    "NA",
+    "ArrowDtype",
+    "BooleanDtype",
+    "Categorical",
+    "CategoricalDtype",
+    "CategoricalIndex",
+    "DataFrame",
+    "DateOffset",
+    "DatetimeIndex",
+    "DatetimeTZDtype",
+    "ExcelFile",
+    "ExcelWriter",
+    "Flags",
+    "Float32Dtype",
+    "Float64Dtype",
+    "Grouper",
+    "HDFStore",
+    "Index",
+    "IndexSlice",
+    "Int8Dtype",
+    "Int16Dtype",
+    "Int32Dtype",
+    "Int64Dtype",
+    "Interval",
+    "IntervalDtype",
+    "IntervalIndex",
+    "MultiIndex",
+    "NaT",
+    "NamedAgg",
+    "Period",
+    "PeriodDtype",
+    "PeriodIndex",
+    "RangeIndex",
+    "Series",
+    "SparseDtype",
+    "StringDtype",
+    "Timedelta",
+    "TimedeltaIndex",
+    "Timestamp",
+    "UInt8Dtype",
+    "UInt16Dtype",
+    "UInt32Dtype",
+    "UInt64Dtype",
+    "api",
+    "array",
+    "arrays",
+    "bdate_range",
+    "col",
+    "concat",
+    "crosstab",
+    "cut",
+    "date_range",
+    "describe_option",
+    "errors",
+    "eval",
+    "factorize",
+    "from_dummies",
+    "get_dummies",
+    "get_option",
+    "infer_freq",
+    "interval_range",
+    "io",
+    "isna",
+    "isnull",
+    "json_normalize",
+    "lreshape",
+    "melt",
+    "merge",
+    "merge_asof",
+    "merge_ordered",
+    "notna",
+    "notnull",
+    "offsets",
+    "option_context",
+    "options",
+    "period_range",
+    "pivot",
+    "pivot_table",
+    "plotting",
+    "qcut",
+    "read_clipboard",
+    "read_csv",
+    "read_excel",
+    "read_feather",
+    "read_fwf",
+    "read_hdf",
+    "read_html",
+    "read_iceberg",
+    "read_json",
+    "read_orc",
+    "read_parquet",
+    "read_pickle",
+    "read_sas",
+    "read_spss",
+    "read_sql",
+    "read_sql_query",
+    "read_sql_table",
+    "read_stata",
+    "read_table",
+    "read_xml",
+    "reset_option",
+    "set_eng_float_format",
+    "set_option",
+    "show_versions",
+    "test",
+    "testing",
+    "timedelta_range",
+    "to_datetime",
+    "to_numeric",
+    "to_pickle",
+    "to_timedelta",
+    "tseries",
+    "unique",
+    "wide_to_long",
+]

bioagent-bench/.venv/lib/python3.14/site-packages/pandas/_typing.py

@@ -0,0 +1,578 @@
+from __future__ import annotations
+
+from builtins import type as type_t  # pyright: ignore[reportUnusedImport]
+from collections.abc import (
+    Callable,
+    Hashable,
+    Iterator,
+    Mapping,
+    MutableMapping,
+    Sequence,
+)
+from datetime import (
+    date,
+    datetime,
+    timedelta,
+    tzinfo,
+)
+from os import PathLike
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    Literal,
+    ParamSpec,
+    Protocol,
+    SupportsIndex,
+    TypeAlias,
+    TypeVar,
+    Union,
+    overload,
+)
+
+import numpy as np
+import numpy.typing as npt
+
+# To prevent import cycles place any internal imports in the branch below
+# and use a string literal forward reference to it in subsequent types
+# https://mypy.readthedocs.io/en/latest/common_issues.html#import-cycles
+
+# Note that Union is needed when a Union includes a pandas type
+
+if TYPE_CHECKING:
+    from pandas._libs import (
+        NaTType,
+        Period,
+        Timedelta,
+        Timestamp,
+    )
+    from pandas._libs.tslibs import BaseOffset
+
+    from pandas.core.dtypes.dtypes import ExtensionDtype
+
+    from pandas import (
+        DatetimeIndex,
+        Interval,
+        PeriodIndex,
+        TimedeltaIndex,
+    )
+    from pandas.arrays import (
+        DatetimeArray,
+        TimedeltaArray,
+    )
+    from pandas.core.arrays.base import ExtensionArray
+    from pandas.core.frame import DataFrame
+    from pandas.core.generic import NDFrame
+    from pandas.core.groupby.generic import (
+        DataFrameGroupBy,
+        GroupBy,
+        SeriesGroupBy,
+    )
+    from pandas.core.indexes.base import Index
+    from pandas.core.internals import (
+        BlockManager,
+        SingleBlockManager,
+    )
+    from pandas.core.resample import Resampler
+    from pandas.core.series import Series
+    from pandas.core.window.rolling import BaseWindow
+
+    from pandas.io.formats.format import EngFormatter
+    from pandas.tseries.holiday import AbstractHolidayCalendar
+
+    ScalarLike_co: TypeAlias = int | float | complex | str | bytes | np.generic
+
+    # numpy compatible types
+    NumpyValueArrayLike: TypeAlias = ScalarLike_co | npt.ArrayLike
+    NumpySorter: TypeAlias = npt.NDArray[np.integer] | None
+
+
+P = ParamSpec("P")
+
+HashableT = TypeVar("HashableT", bound=Hashable)
+HashableT2 = TypeVar("HashableT2", bound=Hashable)
+MutableMappingT = TypeVar("MutableMappingT", bound=MutableMapping)
+
+# array-like
+
+ArrayLike: TypeAlias = Union["ExtensionArray", np.ndarray]
+ArrayLikeT = TypeVar("ArrayLikeT", "ExtensionArray", np.ndarray)
+AnyArrayLike: TypeAlias = Union[ArrayLike, "Index", "Series"]
+TimeArrayLike: TypeAlias = Union["DatetimeArray", "TimedeltaArray"]
+
+# list-like
+
+# from https://github.com/hauntsaninja/useful_types
+# includes Sequence-like objects but excludes str and bytes
+_T_co = TypeVar("_T_co", covariant=True)
+
+
+class SequenceNotStr(Protocol[_T_co]):
+    __module__: str = "pandas.api.typing.aliases"
+
+    @overload
+    def __getitem__(self, index: SupportsIndex, /) -> _T_co: ...
+
+    @overload
+    def __getitem__(self, index: slice, /) -> Sequence[_T_co]: ...
+
+    def __contains__(self, value: object, /) -> bool: ...
+
+    def __len__(self) -> int: ...
+
+    def __iter__(self) -> Iterator[_T_co]: ...
+
+    def index(self, value: Any, start: int = ..., stop: int = ..., /) -> int: ...
+
+    def count(self, value: Any, /) -> int: ...
+
+    def __reversed__(self) -> Iterator[_T_co]: ...
+
+
+ListLike: TypeAlias = AnyArrayLike | SequenceNotStr | range
+
+# scalars
+
+PythonScalar: TypeAlias = str | float | bool
+DatetimeLikeScalar: TypeAlias = Union["Period", "Timestamp", "Timedelta"]
+
+# aligned with pandas-stubs - typical scalars found in Series.  Explicitly leaves
+# out object
+_IndexIterScalar: TypeAlias = Union[
+    str,
+    bytes,
+    date,
+    datetime,
+    timedelta,
+    np.datetime64,
+    np.timedelta64,
+    bool,
+    int,
+    float,
+    "Timestamp",
+    "Timedelta",
+]
+Scalar: TypeAlias = Union[
+    _IndexIterScalar, "Interval", complex, np.integer, np.floating, np.complexfloating
+]
+
+IntStrT = TypeVar("IntStrT", bound=int | str)
+
+# timestamp and timedelta convertible types
+
+TimestampConvertibleTypes: TypeAlias = Union[
+    "Timestamp", date, np.datetime64, np.int64, float, str
+]
+TimestampNonexistent: TypeAlias = (
+    Literal["shift_forward", "shift_backward", "NaT", "raise"] | timedelta
+)
+
+TimedeltaConvertibleTypes: TypeAlias = Union[
+    "Timedelta", timedelta, np.timedelta64, np.int64, float, str
+]
+Timezone: TypeAlias = str | tzinfo
+
+ToTimestampHow: TypeAlias = Literal["s", "e", "start", "end"]
+
+# NDFrameT is stricter and ensures that the same subclass of NDFrame always is
+# used. E.g. `def func(a: NDFrameT) -> NDFrameT: ...` means that if a
+# Series is passed into a function, a Series is always returned and if a DataFrame is
+# passed in, a DataFrame is always returned.
+NDFrameT = TypeVar("NDFrameT", bound="NDFrame")
+
+IndexT = TypeVar("IndexT", bound="Index")
+FreqIndexT = TypeVar("FreqIndexT", "DatetimeIndex", "PeriodIndex", "TimedeltaIndex")
+NumpyIndexT = TypeVar("NumpyIndexT", np.ndarray, "Index")
+
+AxisInt: TypeAlias = int
+Axis: TypeAlias = AxisInt | Literal["index", "columns", "rows"]
+IndexLabel: TypeAlias = Hashable | Sequence[Hashable]
+Level: TypeAlias = Hashable
+Shape: TypeAlias = tuple[int, ...]
+Suffixes: TypeAlias = Sequence[str | None]
+Ordered: TypeAlias = bool | None
+JSONSerializable: TypeAlias = PythonScalar | list | dict | None
+Frequency: TypeAlias = Union[str, "BaseOffset"]
+Axes: TypeAlias = ListLike
+
+RandomState: TypeAlias = (
+    int
+    | np.ndarray
+    | np.random.Generator
+    | np.random.BitGenerator
+    | np.random.RandomState
+)
+
+
+# dtypes
+NpDtype: TypeAlias = str | np.dtype | type[str | complex | bool | object]
+Dtype: TypeAlias = Union["ExtensionDtype", NpDtype]
+AstypeArg: TypeAlias = Union["ExtensionDtype", npt.DTypeLike]
+# DtypeArg specifies all allowable dtypes in a functions its dtype argument
+DtypeArg: TypeAlias = Dtype | Mapping[Hashable, Dtype]
+DtypeObj: TypeAlias = Union[np.dtype, "ExtensionDtype"]
+
+# converters
+ConvertersArg: TypeAlias = dict[Hashable, Callable[[Dtype], Dtype]]
+
+# parse_dates
+ParseDatesArg: TypeAlias = (
+    bool | list[Hashable] | list[list[Hashable]] | dict[Hashable, list[Hashable]]
+)
+
+# For functions like rename that convert one label to another
+Renamer: TypeAlias = Mapping[Any, Hashable] | Callable[[Any], Hashable]
+
+# to maintain type information across generic functions and parametrization
+T = TypeVar("T")
+
+# used in decorators to preserve the signature of the function it decorates
+# see https://mypy.readthedocs.io/en/stable/generics.html#declaring-decorators
+FuncType: TypeAlias = Callable[..., Any]
+F = TypeVar("F", bound=FuncType)
+TypeT = TypeVar("TypeT", bound=type)
+
+# types of vectorized key functions for DataFrame::sort_values and
+# DataFrame::sort_index, among others
+ValueKeyFunc: TypeAlias = Callable[["Series"], Union["Series", AnyArrayLike]] | None
+IndexKeyFunc: TypeAlias = Callable[["Index"], Union["Index", AnyArrayLike]] | None
+
+# types of `func` kwarg for DataFrame.aggregate and Series.aggregate
+AggFuncTypeBase: TypeAlias = Callable | str
+AggFuncTypeDict: TypeAlias = MutableMapping[
+    Hashable, AggFuncTypeBase | list[AggFuncTypeBase]
+]
+AggFuncType: TypeAlias = AggFuncTypeBase | list[AggFuncTypeBase] | AggFuncTypeDict
+AggObjType: TypeAlias = Union[
+    "Series",
+    "DataFrame",
+    "GroupBy",
+    "SeriesGroupBy",
+    "DataFrameGroupBy",
+    "BaseWindow",
+    "Resampler",
+]
+
+PythonFuncType: TypeAlias = Callable[[Any], Any]
+
+# filenames and file-like-objects
+AnyStr_co = TypeVar("AnyStr_co", str, bytes, covariant=True)
+AnyStr_contra = TypeVar("AnyStr_contra", str, bytes, contravariant=True)
+
+
+class BaseBuffer(Protocol):
+    @property
+    def mode(self) -> str:
+        # for _get_filepath_or_buffer
+        ...
+
+    def seek(self, offset: int, whence: int = ..., /) -> int:
+        # with one argument: gzip.GzipFile, bz2.BZ2File
+        # with two arguments: zip.ZipFile, read_sas
+        ...
+
+    def seekable(self) -> bool:
+        # for bz2.BZ2File
+        ...
+
+    def tell(self) -> int:
+        # for zip.ZipFile, read_stata, to_stata
+        ...
+
+
+class ReadBuffer(BaseBuffer, Protocol[AnyStr_co]):
+    __module__: str = "pandas.api.typing.aliases"
+
+    def read(self, n: int = ..., /) -> AnyStr_co:
+        # for BytesIOWrapper, gzip.GzipFile, bz2.BZ2File
+        ...
+
+
+class WriteBuffer(BaseBuffer, Protocol[AnyStr_contra]):
+    __module__: str = "pandas.api.typing.aliases"
+
+    def write(self, b: AnyStr_contra, /) -> Any:
+        # for gzip.GzipFile, bz2.BZ2File
+        ...
+
+    def flush(self) -> Any:
+        # for gzip.GzipFile, bz2.BZ2File
+        ...
+
+
+class ReadPickleBuffer(ReadBuffer[bytes], Protocol):
+    __module__: str = "pandas.api.typing.aliases"
+
+    def readline(self) -> bytes: ...
+
+
+class WriteExcelBuffer(WriteBuffer[bytes], Protocol):
+    __module__: str = "pandas.api.typing.aliases"
+
+    def truncate(self, size: int | None = ..., /) -> int: ...
+
+
+class ReadCsvBuffer(ReadBuffer[AnyStr_co], Protocol):
+    __module__: str = "pandas.api.typing.aliases"
+
+    def __iter__(self) -> Iterator[AnyStr_co]:
+        # for engine=python
+        ...
+
+    def fileno(self) -> int:
+        # for _MMapWrapper
+        ...
+
+    def readline(self) -> AnyStr_co:
+        # for engine=python
+        ...
+
+    @property
+    def closed(self) -> bool:
+        # for engine=pyarrow
+        ...
+
+
+FilePath: TypeAlias = str | PathLike[str]
+
+# for arbitrary kwargs passed during reading/writing files
+StorageOptions: TypeAlias = dict[str, Any] | None
+
+# compression keywords and compression
+CompressionDict: TypeAlias = dict[str, Any]
+CompressionOptions: TypeAlias = (
+    Literal["infer", "gzip", "bz2", "zip", "xz", "zstd", "tar"] | CompressionDict | None
+)
+ParquetCompressionOptions: TypeAlias = (
+    Literal["snappy", "gzip", "brotli", "lz4", "zstd"] | None
+)
+
+# types in DataFrameFormatter
+FormattersType: TypeAlias = (
+    list[Callable] | tuple[Callable, ...] | Mapping[str | int, Callable]
+)
+ColspaceType: TypeAlias = Mapping[Hashable, str | int]
+FloatFormatType: TypeAlias = Union[str, Callable, "EngFormatter"]
+ColspaceArgType: TypeAlias = (
+    str | int | Sequence[str | int] | Mapping[Hashable, str | int]
+)
+
+# Arguments for fillna()
+FillnaOptions: TypeAlias = Literal["backfill", "bfill", "ffill", "pad"]
+InterpolateOptions: TypeAlias = Literal[
+    "linear",
+    "time",
+    "index",
+    "values",
+    "nearest",
+    "zero",
+    "slinear",
+    "quadratic",
+    "cubic",
+    "barycentric",
+    "polynomial",
+    "krogh",
+    "piecewise_polynomial",
+    "spline",
+    "pchip",
+    "akima",
+    "cubicspline",
+    "from_derivatives",
+]
+
+# internals
+Manager: TypeAlias = Union["BlockManager", "SingleBlockManager"]
+
+# indexing
+# PositionalIndexer -> valid 1D positional indexer, e.g. can pass
+# to ndarray.__getitem__
+# ScalarIndexer is for a single value as the index
+# SequenceIndexer is for list like or slices (but not tuples)
+# PositionalIndexerTuple is extends the PositionalIndexer for 2D arrays
+# These are used in various __getitem__ overloads
+# TODO(typing#684): add Ellipsis, see
+# https://github.com/python/typing/issues/684#issuecomment-548203158
+# https://bugs.python.org/issue41810
+# Using List[int] here rather than Sequence[int] to disallow tuples.
+ScalarIndexer: TypeAlias = int | np.integer
+SequenceIndexer: TypeAlias = slice | list[int] | np.ndarray
+PositionalIndexer: TypeAlias = ScalarIndexer | SequenceIndexer
+PositionalIndexerTuple: TypeAlias = tuple[PositionalIndexer, PositionalIndexer]
+PositionalIndexer2D: TypeAlias = PositionalIndexer | PositionalIndexerTuple
+TakeIndexer: TypeAlias = Sequence[int] | Sequence[np.integer] | npt.NDArray[np.integer]
+
+# Shared by functions such as drop and astype
+IgnoreRaise: TypeAlias = Literal["ignore", "raise"]
+
+# Windowing rank methods
+WindowingRankType: TypeAlias = Literal["average", "min", "max"]
+
+# read_csv engines
+CSVEngine: TypeAlias = Literal["c", "python", "pyarrow", "python-fwf"]
+
+# read_json engines
+JSONEngine: TypeAlias = Literal["ujson", "pyarrow"]
+
+# read_xml parsers
+XMLParsers: TypeAlias = Literal["lxml", "etree"]
+
+# read_html flavors
+HTMLFlavors: TypeAlias = Literal["lxml", "html5lib", "bs4"]
+
+# Interval closed type
+IntervalLeftRight: TypeAlias = Literal["left", "right"]
+IntervalClosedType: TypeAlias = IntervalLeftRight | Literal["both", "neither"]
+
+# datetime and NaTType
+DatetimeNaTType: TypeAlias = Union[datetime, "NaTType"]
+DateTimeErrorChoices: TypeAlias = Literal["raise", "coerce"]
+
+# sort_index
+SortKind: TypeAlias = Literal["quicksort", "mergesort", "heapsort", "stable"]
+NaPosition: TypeAlias = Literal["first", "last"]
+
+# Arguments for nsmallest and nlargest
+NsmallestNlargestKeep: TypeAlias = Literal["first", "last", "all"]
+
+# quantile interpolation
+QuantileInterpolation: TypeAlias = Literal[
+    "linear", "lower", "higher", "midpoint", "nearest"
+]
+
+# plotting
+PlottingOrientation: TypeAlias = Literal["horizontal", "vertical"]
+
+# dropna
+AnyAll: TypeAlias = Literal["any", "all"]
+
+# merge
+MergeHow: TypeAlias = Literal[
+    "left", "right", "inner", "outer", "cross", "left_anti", "right_anti"
+]
+MergeValidate: TypeAlias = Literal[
+    "one_to_one",
+    "1:1",
+    "one_to_many",
+    "1:m",
+    "many_to_one",
+    "m:1",
+    "many_to_many",
+    "m:m",
+]
+
+# join
+JoinHow: TypeAlias = Literal["left", "right", "inner", "outer"]
+JoinValidate: TypeAlias = Literal[
+    "one_to_one",
+    "1:1",
+    "one_to_many",
+    "1:m",
+    "many_to_one",
+    "m:1",
+    "many_to_many",
+    "m:m",
+]
+
+# reindex
+ReindexMethod: TypeAlias = FillnaOptions | Literal["nearest"]
+
+MatplotlibColor: TypeAlias = str | Sequence[float]
+TimeGrouperOrigin: TypeAlias = Union[
+    "Timestamp", Literal["epoch", "start", "start_day", "end", "end_day"]
+]
+TimeAmbiguous: TypeAlias = (
+    Literal["infer", "NaT", "raise"] | bool | npt.NDArray[np.bool_]
+)
+TimeNonexistent: TypeAlias = (
+    Literal["shift_forward", "shift_backward", "NaT", "raise"] | timedelta
+)
+
+DropKeep: TypeAlias = Literal["first", "last", False]
+CorrelationMethod: TypeAlias = (
+    Literal["pearson", "kendall", "spearman"]
+    | Callable[[np.ndarray, np.ndarray], float]
+)
+
+AlignJoin: TypeAlias = Literal["outer", "inner", "left", "right"]
+DtypeBackend: TypeAlias = Literal["pyarrow", "numpy_nullable"]
+
+TimeUnit: TypeAlias = Literal["s", "ms", "us", "ns"]
+OpenFileErrors: TypeAlias = Literal[
+    "strict",
+    "ignore",
+    "replace",
+    "surrogateescape",
+    "xmlcharrefreplace",
+    "backslashreplace",
+    "namereplace",
+]
+
+# update
+UpdateJoin: TypeAlias = Literal["left"]
+
+# applymap
+NaAction: TypeAlias = Literal["ignore"]
+
+# from_dict
+FromDictOrient: TypeAlias = Literal["columns", "index", "tight"]
+
+# to_stata
+ToStataByteorder: TypeAlias = Literal[">", "<", "little", "big"]
+
+# ExcelWriter
+ExcelWriterIfSheetExists: TypeAlias = Literal["error", "new", "replace", "overlay"]
+ExcelWriterMergeCells: TypeAlias = bool | Literal["columns"]
+
+# Offsets
+OffsetCalendar: TypeAlias = Union[np.busdaycalendar, "AbstractHolidayCalendar"]
+
+# read_csv: usecols
+UsecolsArgType: TypeAlias = (
+    SequenceNotStr[Hashable] | range | AnyArrayLike | Callable[[HashableT], bool] | None
+)
+
+# maintain the sub-type of any hashable sequence
+SequenceT = TypeVar("SequenceT", bound=Sequence[Hashable])
+
+SliceType: TypeAlias = Hashable | None
+
+
+# Arrow PyCapsule Interface
+# from https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html#protocol-typehints
+
+
+class ArrowArrayExportable(Protocol):
+    """
+    An object with an ``__arrow_c_array__`` method.
+
+    This method indicates the object is an Arrow-compatible object implementing
+    the `Arrow PyCapsule Protocol`_ (exposing the `Arrow C Data Interface`_ in
+    Python), enabling zero-copy Arrow data interchange across libraries.
+
+    .. _Arrow PyCapsule Protocol: https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html
+    .. _Arrow C Data Interface: https://arrow.apache.org/docs/format/CDataInterface.html
+
+    """
+
+    def __arrow_c_array__(
+        self, requested_schema: object | None = None
+    ) -> tuple[object, object]: ...
+
+
+class ArrowStreamExportable(Protocol):
+    """
+    An object with an ``__arrow_c_stream__`` method.
+
+    This method indicates the object is an Arrow-compatible object implementing
+    the `Arrow PyCapsule Protocol`_ (exposing the `Arrow C Data Interface`_
+    for streams in Python), enabling zero-copy Arrow data interchange across
+    libraries.
+
+    .. _Arrow PyCapsule Protocol: https://arrow.apache.org/docs/format/CDataInterface/PyCapsuleInterface.html
+    .. _Arrow C Stream Interface: https://arrow.apache.org/docs/format/CStreamInterface.html
+
+    """
+
+    def __arrow_c_stream__(self, requested_schema: object | None = None) -> object: ...
+
+
+__all__ = ["type_t"]

bioagent-bench/.venv/lib/python3.14/site-packages/pandas/_version.py

@@ -0,0 +1,692 @@
+# This file helps to compute a version number in source trees obtained from
+# git-archive tarball (such as those provided by github's download-from-tag
+# feature). Distribution tarballs (built by setup.py sdist) and build
+# directories (produced by setup.py build) will contain a much shorter file
+# that just contains the computed version number.
+
+# This file is released into the public domain.
+# Generated by versioneer-0.28
+# https://github.com/python-versioneer/python-versioneer
+
+"""Git implementation of _version.py."""
+
+from collections.abc import Callable
+import errno
+import functools
+import os
+import re
+import subprocess
+import sys
+
+
+def get_keywords():
+    """Get the keywords needed to look up the version information."""
+    # these strings will be replaced by git during git-archive.
+    # setup.py/versioneer.py will grep for the variable names, so they must
+    # each be defined on a line of their own. _version.py will just call
+    # get_keywords().
+    git_refnames = " (HEAD, tag: v3.0.2, origin/3.0.x)"
+    git_full = "ab90747e3dae0e69b1bdbf083820b8075689b34b"
+    git_date = "2026-03-30 21:34:07 +0200"
+    keywords = {"refnames": git_refnames, "full": git_full, "date": git_date}
+    return keywords
+
+
+class VersioneerConfig:
+    """Container for Versioneer configuration parameters."""
+
+
+def get_config():
+    """Create, populate and return the VersioneerConfig() object."""
+    # these strings are filled in when 'setup.py versioneer' creates
+    # _version.py
+    cfg = VersioneerConfig()
+    cfg.VCS = "git"
+    cfg.style = "pep440"
+    cfg.tag_prefix = "v"
+    cfg.parentdir_prefix = "pandas-"
+    cfg.versionfile_source = "pandas/_version.py"
+    cfg.verbose = False
+    return cfg
+
+
+class NotThisMethod(Exception):
+    """Exception raised if a method is not valid for the current scenario."""
+
+
+LONG_VERSION_PY: dict[str, str] = {}
+HANDLERS: dict[str, dict[str, Callable]] = {}
+
+
+def register_vcs_handler(vcs, method):  # decorator
+    """Create decorator to mark a method as the handler of a VCS."""
+
+    def decorate(f):
+        """Store f in HANDLERS[vcs][method]."""
+        if vcs not in HANDLERS:
+            HANDLERS[vcs] = {}
+        HANDLERS[vcs][method] = f
+        return f
+
+    return decorate
+
+
+def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None):
+    """Call the given command(s)."""
+    assert isinstance(commands, list)
+    process = None
+
+    popen_kwargs = {}
+    if sys.platform == "win32":
+        # This hides the console window if pythonw.exe is used
+        startupinfo = subprocess.STARTUPINFO()
+        startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+        popen_kwargs["startupinfo"] = startupinfo
+
+    for command in commands:
+        dispcmd = str([command, *args])
+        try:
+            # remember shell=False, so use git.cmd on windows, not just git
+            process = subprocess.Popen(
+                [command, *args],
+                cwd=cwd,
+                env=env,
+                stdout=subprocess.PIPE,
+                stderr=(subprocess.PIPE if hide_stderr else None),
+                **popen_kwargs,
+            )
+            break
+        except OSError:
+            e = sys.exc_info()[1]
+            if e.errno == errno.ENOENT:
+                continue
+            if verbose:
+                print(f"unable to run {dispcmd}")
+                print(e)
+            return None, None
+    else:
+        if verbose:
+            print(f"unable to find command, tried {commands}")
+        return None, None
+    stdout = process.communicate()[0].strip().decode()
+    if process.returncode != 0:
+        if verbose:
+            print(f"unable to run {dispcmd} (error)")
+            print(f"stdout was {stdout}")
+        return None, process.returncode
+    return stdout, process.returncode
+
+
+def versions_from_parentdir(parentdir_prefix, root, verbose):
+    """Try to determine the version from the parent directory name.
+
+    Source tarballs conventionally unpack into a directory that includes both
+    the project name and a version string. We will also support searching up
+    two directory levels for an appropriately named parent directory
+    """
+    rootdirs = []
+
+    for _ in range(3):
+        dirname = os.path.basename(root)
+        if dirname.startswith(parentdir_prefix):
+            return {
+                "version": dirname[len(parentdir_prefix) :],
+                "full-revisionid": None,
+                "dirty": False,
+                "error": None,
+                "date": None,
+            }
+        rootdirs.append(root)
+        root = os.path.dirname(root)  # up a level
+
+    if verbose:
+        print(
+            f"Tried directories {rootdirs!s} \
+            but none started with prefix {parentdir_prefix}"
+        )
+    raise NotThisMethod("rootdir doesn't start with parentdir_prefix")
+
+
+@register_vcs_handler("git", "get_keywords")
+def git_get_keywords(versionfile_abs):
+    """Extract version information from the given file."""
+    # the code embedded in _version.py can just fetch the value of these
+    # keywords. When used from setup.py, we don't want to import _version.py,
+    # so we do it with a regexp instead. This function is not used from
+    # _version.py.
+    keywords = {}
+    try:
+        with open(versionfile_abs, encoding="utf-8") as fobj:
+            for line in fobj:
+                if line.strip().startswith("git_refnames ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["refnames"] = mo.group(1)
+                if line.strip().startswith("git_full ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["full"] = mo.group(1)
+                if line.strip().startswith("git_date ="):
+                    mo = re.search(r'=\s*"(.*)"', line)
+                    if mo:
+                        keywords["date"] = mo.group(1)
+    except OSError:
+        pass
+    return keywords
+
+
+@register_vcs_handler("git", "keywords")
+def git_versions_from_keywords(keywords, tag_prefix, verbose):
+    """Get version information from git keywords."""
+    if "refnames" not in keywords:
+        raise NotThisMethod("Short version file found")
+    date = keywords.get("date")
+    if date is not None:
+        # Use only the last line.  Previous lines may contain GPG signature
+        # information.
+        date = date.splitlines()[-1]
+
+        # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant
+        # datestamp. However we prefer "%ci" (which expands to an "ISO-8601
+        # -like" string, which we must then edit to make compliant), because
+        # it's been around since git-1.5.3, and it's too difficult to
+        # discover which version we're using, or to work around using an
+        # older one.
+        date = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+    refnames = keywords["refnames"].strip()
+    if refnames.startswith("$Format"):
+        if verbose:
+            print("keywords are unexpanded, not using")
+        raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
+    refs = {r.strip() for r in refnames.strip("()").split(",")}
+    # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
+    # just "foo-1.0". If we see a "tag: " prefix, prefer those.
+    TAG = "tag: "
+    tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)}
+    if not tags:
+        # Either we're using git < 1.8.3, or there really are no tags. We use
+        # a heuristic: assume all version tags have a digit. The old git %d
+        # expansion behaves like git log --decorate=short and strips out the
+        # refs/heads/ and refs/tags/ prefixes that would let us distinguish
+        # between branches and tags. By ignoring refnames without digits, we
+        # filter out many common branch names like "release" and
+        # "stabilization", as well as "HEAD" and "master".
+        tags = {r for r in refs if re.search(r"\d", r)}
+        if verbose:
+            print(f"discarding '{','.join(refs - tags)}', no digits")
+    if verbose:
+        print(f"likely tags: {','.join(sorted(tags))}")
+    for ref in sorted(tags):
+        # sorting will prefer e.g. "2.0" over "2.0rc1"
+        if ref.startswith(tag_prefix):
+            r = ref[len(tag_prefix) :]
+            # Filter out refs that exactly match prefix or that don't start
+            # with a number once the prefix is stripped (mostly a concern
+            # when prefix is '')
+            if not re.match(r"\d", r):
+                continue
+            if verbose:
+                print(f"picking {r}")
+            return {
+                "version": r,
+                "full-revisionid": keywords["full"].strip(),
+                "dirty": False,
+                "error": None,
+                "date": date,
+            }
+    # no suitable tags, so version is "0+unknown", but full hex is still there
+    if verbose:
+        print("no suitable tags, using unknown + full revision id")
+    return {
+        "version": "0+unknown",
+        "full-revisionid": keywords["full"].strip(),
+        "dirty": False,
+        "error": "no suitable tags",
+        "date": None,
+    }
+
+
+@register_vcs_handler("git", "pieces_from_vcs")
+def git_pieces_from_vcs(tag_prefix, root, verbose, runner=run_command):
+    """Get version from 'git describe' in the root of the source tree.
+
+    This only gets called if the git-archive 'subst' keywords were *not*
+    expanded, and _version.py hasn't already been rewritten with a short
+    version string, meaning we're inside a checked out source tree.
+    """
+    GITS = ["git"]
+    if sys.platform == "win32":
+        GITS = ["git.cmd", "git.exe"]
+
+    # GIT_DIR can interfere with correct operation of Versioneer.
+    # It may be intended to be passed to the Versioneer-versioned project,
+    # but that should not change where we get our version from.
+    env = os.environ.copy()
+    env.pop("GIT_DIR", None)
+    runner = functools.partial(runner, env=env)
+
+    _, rc = runner(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=not verbose)
+    if rc != 0:
+        if verbose:
+            print(f"Directory {root} not under git control")
+        raise NotThisMethod("'git rev-parse --git-dir' returned error")
+
+    # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty]
+    # if there isn't one, this yields HEX[-dirty] (no NUM)
+    describe_out, rc = runner(
+        GITS,
+        [
+            "describe",
+            "--tags",
+            "--dirty",
+            "--always",
+            "--long",
+            "--match",
+            f"{tag_prefix}[[:digit:]]*",
+        ],
+        cwd=root,
+    )
+    # --long was added in git-1.5.5
+    if describe_out is None:
+        raise NotThisMethod("'git describe' failed")
+    describe_out = describe_out.strip()
+    full_out, rc = runner(GITS, ["rev-parse", "HEAD"], cwd=root)
+    if full_out is None:
+        raise NotThisMethod("'git rev-parse' failed")
+    full_out = full_out.strip()
+
+    pieces = {}
+    pieces["long"] = full_out
+    pieces["short"] = full_out[:7]  # maybe improved later
+    pieces["error"] = None
+
+    branch_name, rc = runner(GITS, ["rev-parse", "--abbrev-ref", "HEAD"], cwd=root)
+    # --abbrev-ref was added in git-1.6.3
+    if rc != 0 or branch_name is None:
+        raise NotThisMethod("'git rev-parse --abbrev-ref' returned error")
+    branch_name = branch_name.strip()
+
+    if branch_name == "HEAD":
+        # If we aren't exactly on a branch, pick a branch which represents
+        # the current commit. If all else fails, we are on a branchless
+        # commit.
+        branches, rc = runner(GITS, ["branch", "--contains"], cwd=root)
+        # --contains was added in git-1.5.4
+        if rc != 0 or branches is None:
+            raise NotThisMethod("'git branch --contains' returned error")
+        branches = branches.split("\n")
+
+        # Remove the first line if we're running detached
+        if "(" in branches[0]:
+            branches.pop(0)
+
+        # Strip off the leading "* " from the list of branches.
+        branches = [branch[2:] for branch in branches]
+        if "master" in branches:
+            branch_name = "master"
+        elif not branches:
+            branch_name = None
+        else:
+            # Pick the first branch that is returned. Good or bad.
+            branch_name = branches[0]
+
+    pieces["branch"] = branch_name
+
+    # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty]
+    # TAG might have hyphens.
+    git_describe = describe_out
+
+    # look for -dirty suffix
+    dirty = git_describe.endswith("-dirty")
+    pieces["dirty"] = dirty
+    if dirty:
+        git_describe = git_describe[: git_describe.rindex("-dirty")]
+
+    # now we have TAG-NUM-gHEX or HEX
+
+    if "-" in git_describe:
+        # TAG-NUM-gHEX
+        mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe)
+        if not mo:
+            # unparsable. Maybe git-describe is misbehaving?
+            pieces["error"] = f"unable to parse git-describe output: '{describe_out}'"
+            return pieces
+
+        # tag
+        full_tag = mo.group(1)
+        if not full_tag.startswith(tag_prefix):
+            if verbose:
+                fmt = "tag '%s' doesn't start with prefix '%s'"
+                print(fmt % (full_tag, tag_prefix))
+            pieces["error"] = (
+                f"tag '{full_tag}' doesn't start with prefix '{tag_prefix}'"
+            )
+            return pieces
+        pieces["closest-tag"] = full_tag[len(tag_prefix) :]
+
+        # distance: number of commits since tag
+        pieces["distance"] = int(mo.group(2))
+
+        # commit: short hex revision ID
+        pieces["short"] = mo.group(3)
+
+    else:
+        # HEX: no tags
+        pieces["closest-tag"] = None
+        out, rc = runner(GITS, ["rev-list", "HEAD", "--left-right"], cwd=root)
+        pieces["distance"] = len(out.split())  # total number of commits
+
+    # commit date: see ISO-8601 comment in git_versions_from_keywords()
+    date = runner(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[0].strip()
+    # Use only the last line.  Previous lines may contain GPG signature
+    # information.
+    date = date.splitlines()[-1]
+    pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1)
+
+    return pieces
+
+
+def plus_or_dot(pieces) -> str:
+    """Return a + if we don't already have one, else return a ."""
+    if "+" in pieces.get("closest-tag", ""):
+        return "."
+    return "+"
+
+
+def render_pep440(pieces):
+    """Build up version string, with post-release "local version identifier".
+
+    Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you
+    get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty
+
+    Exceptions:
+    1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += plus_or_dot(pieces)
+            rendered += f"{pieces['distance']}.g{pieces['short']}"
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = f"0+untagged.{pieces['distance']}.g{pieces['short']}"
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_branch(pieces):
+    """TAG[[.dev0]+DISTANCE.gHEX[.dirty]] .
+
+    The ".dev0" means not master branch. Note that .dev0 sorts backwards
+    (a feature branch will appear "older" than the master branch).
+
+    Exceptions:
+    1: no tags. 0[.dev0]+untagged.DISTANCE.gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            if pieces["branch"] != "master":
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += f"{pieces['distance']}.g{pieces['short']}"
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = "0"
+        if pieces["branch"] != "master":
+            rendered += ".dev0"
+        rendered += f"+untagged.{pieces['distance']}.g{pieces['short']}"
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def pep440_split_post(ver):
+    """Split pep440 version string at the post-release segment.
+
+    Returns the release segments before the post-release and the
+    post-release version number (or -1 if no post-release segment is present).
+    """
+    vc = str.split(ver, ".post")
+    return vc[0], int(vc[1] or 0) if len(vc) == 2 else None
+
+
+def render_pep440_pre(pieces):
+    """TAG[.postN.devDISTANCE] -- No -dirty.
+
+    Exceptions:
+    1: no tags. 0.post0.devDISTANCE
+    """
+    if pieces["closest-tag"]:
+        if pieces["distance"]:
+            # update the post release segment
+            tag_version, post_version = pep440_split_post(pieces["closest-tag"])
+            rendered = tag_version
+            if post_version is not None:
+                rendered += f".post{post_version + 1}.dev{pieces['distance']}"
+            else:
+                rendered += f".post0.dev{pieces['distance']}"
+        else:
+            # no commits, use the tag as the version
+            rendered = pieces["closest-tag"]
+    else:
+        # exception #1
+        rendered = f"0.post0.dev{pieces['distance']}"
+    return rendered
+
+
+def render_pep440_post(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX] .
+
+    The ".dev0" means dirty. Note that .dev0 sorts backwards
+    (a dirty tree will appear "older" than the corresponding clean one),
+    but you shouldn't be releasing software with -dirty anyways.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += f".post{pieces['distance']}"
+            if pieces["dirty"]:
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += f"g{pieces['short']}"
+    else:
+        # exception #1
+        rendered = f"0.post{pieces['distance']}"
+        if pieces["dirty"]:
+            rendered += ".dev0"
+        rendered += f"+g{pieces['short']}"
+    return rendered
+
+
+def render_pep440_post_branch(pieces):
+    """TAG[.postDISTANCE[.dev0]+gHEX[.dirty]] .
+
+    The ".dev0" means not master branch.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]+gHEX[.dirty]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += f".post{pieces['distance']}"
+            if pieces["branch"] != "master":
+                rendered += ".dev0"
+            rendered += plus_or_dot(pieces)
+            rendered += f"g{pieces['short']}"
+            if pieces["dirty"]:
+                rendered += ".dirty"
+    else:
+        # exception #1
+        rendered = f"0.post{pieces['distance']}"
+        if pieces["branch"] != "master":
+            rendered += ".dev0"
+        rendered += f"+g{pieces['short']}"
+        if pieces["dirty"]:
+            rendered += ".dirty"
+    return rendered
+
+
+def render_pep440_old(pieces):
+    """TAG[.postDISTANCE[.dev0]] .
+
+    The ".dev0" means dirty.
+
+    Exceptions:
+    1: no tags. 0.postDISTANCE[.dev0]
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"] or pieces["dirty"]:
+            rendered += f"0.post{pieces['distance']}"
+            if pieces["dirty"]:
+                rendered += ".dev0"
+    else:
+        # exception #1
+        rendered = f"0.post{pieces['distance']}"
+        if pieces["dirty"]:
+            rendered += ".dev0"
+    return rendered
+
+
+def render_git_describe(pieces):
+    """TAG[-DISTANCE-gHEX][-dirty].
+
+    Like 'git describe --tags --dirty --always'.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        if pieces["distance"]:
+            rendered += f"-{pieces['distance']}-g{pieces['short']}"
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render_git_describe_long(pieces):
+    """TAG-DISTANCE-gHEX[-dirty].
+
+    Like 'git describe --tags --dirty --always --long'.
+    The distance/hash is unconditional.
+
+    Exceptions:
+    1: no tags. HEX[-dirty]  (note: no 'g' prefix)
+    """
+    if pieces["closest-tag"]:
+        rendered = pieces["closest-tag"]
+        rendered += f"-{pieces['distance']}-g{pieces['short']}"
+    else:
+        # exception #1
+        rendered = pieces["short"]
+    if pieces["dirty"]:
+        rendered += "-dirty"
+    return rendered
+
+
+def render(pieces, style):
+    """Render the given version pieces into the requested style."""
+    if pieces["error"]:
+        return {
+            "version": "unknown",
+            "full-revisionid": pieces.get("long"),
+            "dirty": None,
+            "error": pieces["error"],
+            "date": None,
+        }
+
+    if not style or style == "default":
+        style = "pep440"  # the default
+
+    if style == "pep440":
+        rendered = render_pep440(pieces)
+    elif style == "pep440-branch":
+        rendered = render_pep440_branch(pieces)
+    elif style == "pep440-pre":
+        rendered = render_pep440_pre(pieces)
+    elif style == "pep440-post":
+        rendered = render_pep440_post(pieces)
+    elif style == "pep440-post-branch":
+        rendered = render_pep440_post_branch(pieces)
+    elif style == "pep440-old":
+        rendered = render_pep440_old(pieces)
+    elif style == "git-describe":
+        rendered = render_git_describe(pieces)
+    elif style == "git-describe-long":
+        rendered = render_git_describe_long(pieces)
+    else:
+        raise ValueError(f"unknown style '{style}'")
+
+    return {
+        "version": rendered,
+        "full-revisionid": pieces["long"],
+        "dirty": pieces["dirty"],
+        "error": None,
+        "date": pieces.get("date"),
+    }
+
+
+def get_versions() -> dict:
+    """Get version information or return default if unable to do so."""
+    # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have
+    # __file__, we can work backwards from there to the root. Some
+    # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which
+    # case we can only use expanded keywords.
+
+    cfg = get_config()
+    verbose = cfg.verbose
+
+    try:
+        return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose)
+    except NotThisMethod:
+        pass
+
+    try:
+        root = os.path.realpath(__file__)
+        # versionfile_source is the relative path from the top of the source
+        # tree (where the .git directory might live) to this file. Invert
+        # this to find the root from __file__.
+        for _ in cfg.versionfile_source.split("/"):
+            root = os.path.dirname(root)
+    except NameError:
+        return {
+            "version": "0+unknown",
+            "full-revisionid": None,
+            "dirty": None,
+            "error": "unable to find root of source tree",
+            "date": None,
+        }
+
+    try:
+        pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose)
+        return render(pieces, cfg.style)
+    except NotThisMethod:
+        pass
+
+    try:
+        if cfg.parentdir_prefix:
+            return versions_from_parentdir(cfg.parentdir_prefix, root, verbose)
+    except NotThisMethod:
+        pass
+
+    return {
+        "version": "0+unknown",
+        "full-revisionid": None,
+        "dirty": None,
+        "error": "unable to compute version",
+        "date": None,
+    }