backend/app/example_corpora.py

"""Bundled example corpora, so a visitor can run CCR without their own data.

Every entry is a file in sample_data/ plus the provenance a researcher needs to
cite it. The citation travels into run metadata (see jobs.py), because results
produced on someone else's corpus have to carry the attribution into whatever
the researcher publishes - a demo dataset is still data with authors.

Adding one: drop the CSV in sample_data/, add an entry here, and say in
`preprocessing` exactly what was done to it. A test checks every file exists,
parses, and has the declared text column.
"""

from __future__ import annotations

from dataclasses import asdict, dataclass
from pathlib import Path

SAMPLES_DIR = Path(__file__).resolve().parents[2] / "sample_data"


@dataclass(frozen=True)
class ExampleCorpus:
    """A corpus offered in Step 1.

    Small ones ship in sample_data/ (`filename`); large ones live in object
    storage under the `examples/` prefix (`storage_key`), because a 38 MB file
    in the repo would ride in the Docker image and the Space repo on every
    deploy. Exactly one of the two is set.
    """

    id: str
    name: str
    filename: str
    text_column: str
    language: str
    n_rows: int
    description: str
    citation: str
    source_url: str
    preprocessing: str
    # The corpus's OWN licence, which is not the platform's. A researcher has to
    # know the terms they are working under before they analyse someone else's
    # data, so it is shown wherever the corpus is described.
    license: str = ""
    # First in the list and selected on load. NOT a claim that it is 'the'
    # corpus: the PI's ask was for CAMEL to be available and ready to use.
    preselected: bool = False
    storage_key: str = ""
    # Optional mark for the dataset, served from sample_data/ at /samples.
    # Shown in the info dialog, where it identifies THIS dataset - not in the
    # picker row, which lists several and would imply they share a source.
    logo: str = ""
    # One line under the name in the sample-data list, so a researcher can judge
    # fit before running. The row already shows the count and language.
    detail: str = ""

    @property
    def path(self) -> Path:
        return SAMPLES_DIR / self.filename

    @property
    def bundled(self) -> bool:
        return not self.storage_key

    def available(self) -> bool:
        """Bundled files are checked on disk; stored ones are assumed present -
        a HEAD on every listing would add a network round trip to page load,
        and a missing object surfaces as a clear 404 on selection instead."""
        return self.path.exists() if self.bundled else True

    def blocked_reason(self, row_ceiling: int) -> str:
        """Why this instance cannot run this corpus, or "" if it can.

        A corpus larger than CCR_MAX_ROWS is refused at ingest, so offering it
        as a working option produces a dead end. Saying so up front, and why,
        beats an error after the click - and it corrects itself the moment the
        deployment raises the limit.
        """
        if not self.available():
            return "Not loaded on this instance."
        if row_ceiling and self.n_rows > row_ceiling:
            return (
                f"Too large for this instance: {self.n_rows:,} texts against a "
                f"{row_ceiling:,}-row limit."
            )
        return ""

    def public(self, row_ceiling: int = 0) -> dict:
        data = asdict(self)
        data["bundled"] = self.bundled
        data["logo_url"] = (
            f"/samples/{self.logo}" if self.logo and (SAMPLES_DIR / self.logo).exists() else ""
        )
        data.pop("logo", None)
        blocked = self.blocked_reason(row_ceiling)
        data["usable"] = not blocked
        data["blocked_reason"] = blocked
        data.pop("storage_key", None)  # internal location, not the user's business
        return data


EXAMPLES: list[ExampleCorpus] = [
    ExampleCorpus(
        id="camel_sample",
        name="CAMEL corpus (sample)",
        filename="camel_sample.csv",
        text_column="text",
        language="en",
        n_rows=999,
        description=(
            "The Cultural and Moral Expressions in Language (CAMEL) corpus consists of "
            "over 57 thousand texts taken from several platforms and annotated for 25 "
            "different cultural and moral constructs. This is a 999-text sample drawn "
            "from it, keeping the platform mix of the full corpus (Reddit, Twitter, "
            "IMDB, news, Wikipedia, and others). The full corpus, including the "
            "annotations, is on Hugging Face."
        ),
        citation=(
            "Zewail, A., Setia, A., Mohammadsadegh, R., Seker, F., Hajian, A., Sosa, H., "
            "Morhayim, L., Reddy, S., & Atari, M. (2026). The Cultural and Moral "
            "Expressions in Language (CAMEL) corpus [Preprint]."
        ),
        source_url="https://huggingface.co/datasets/Culture-and-Morality-Lab/CAMEL_Dataset",
        preprocessing=(
            "Usernames were removed by the corpus authors. For this sample: texts drawn "
            "proportionally by source platform with a fixed seed "
            "(scripts/build_camel_sample.py), whitespace trimmed, and only the text and "
            "source_platform columns kept. Text is otherwise verbatim."
        ),
        detail=(
            "Median 25 words, mean 69. Runs in seconds on any model. "
            "Best for a first look at how CCR works."
        ),
        license="CC BY-NC 4.0 (Creative Commons Attribution-NonCommercial 4.0). Free for non-commercial research with attribution; see the dataset page for the full terms.",
        preselected=True,
        logo="camel_logo.png",
    ),
    ExampleCorpus(
        id="camel_full",
        name="CAMEL corpus (full)",
        filename="",
        storage_key="examples/camel_full.csv",
        text_column="text",
        language="en",
        n_rows=57174,
        description=(
            "The complete Cultural and Moral Expressions in Language (CAMEL) corpus: "
            "57,174 texts from Reddit, Twitter, IMDB, news, Wikipedia, the Internet "
            "Archive and other sources, annotated by the lab for 25 cultural and moral "
            "constructs. The annotation columns come with it, so CCR scores can be "
            "compared against the human ratings."
        ),
        citation=(
            "Zewail, A., Setia, A., Mohammadsadegh, R., Seker, F., Hajian, A., Sosa, H., "
            "Morhayim, L., Reddy, S., & Atari, M. (2026). The Cultural and Moral "
            "Expressions in Language (CAMEL) corpus [Preprint]."
        ),
        source_url="https://huggingface.co/datasets/Culture-and-Morality-Lab/CAMEL_Dataset",
        preprocessing=(
            "Usernames were removed by the corpus authors. Otherwise the corpus exactly "
            "as published: all 64 columns, text verbatim, no sampling."
        ),
        detail=(
            "Median 26 words, up to 1,770. Expect a few minutes on MiniLM and "
            "considerably longer on the large models; pick MiniLM unless you have a "
            "reason not to."
        ),
        license="CC BY-NC 4.0 (Creative Commons Attribution-NonCommercial 4.0). Free for non-commercial research with attribution; see the dataset page for the full terms.",
        logo="camel_logo.png",
    ),
]

BY_ID = {e.id: e for e in EXAMPLES}


def get(example_id: str) -> ExampleCorpus | None:
    return BY_ID.get(example_id)


def listed(row_ceiling: int = 0) -> list[dict]:
    """Examples this instance has, preselected one first.

    Entries too large for the row ceiling are still listed but marked unusable
    with a reason, rather than hidden: a corpus quietly vanishing is harder to
    explain than one that says why it is unavailable.
    """
    return [
        e.public(row_ceiling)
        for e in sorted(EXAMPLES, key=lambda e: not e.preselected)
        if e.available()
    ]