Spaces:
Running
Running
| """Bundled example corpora, so a visitor can run CCR without their own data. | |
| Every entry is a file in sample_data/ plus the provenance a researcher needs to | |
| cite it. The citation travels into run metadata (see jobs.py), because results | |
| produced on someone else's corpus have to carry the attribution into whatever | |
| the researcher publishes - a demo dataset is still data with authors. | |
| Adding one: drop the CSV in sample_data/, add an entry here, and say in | |
| `preprocessing` exactly what was done to it. A test checks every file exists, | |
| parses, and has the declared text column. | |
| """ | |
| from __future__ import annotations | |
| from dataclasses import asdict, dataclass | |
| from pathlib import Path | |
| SAMPLES_DIR = Path(__file__).resolve().parents[2] / "sample_data" | |
| class ExampleCorpus: | |
| """A corpus offered in Step 1. | |
| Small ones ship in sample_data/ (`filename`); large ones live in object | |
| storage under the `examples/` prefix (`storage_key`), because a 38 MB file | |
| in the repo would ride in the Docker image and the Space repo on every | |
| deploy. Exactly one of the two is set. | |
| """ | |
| id: str | |
| name: str | |
| filename: str | |
| text_column: str | |
| language: str | |
| n_rows: int | |
| description: str | |
| citation: str | |
| source_url: str | |
| preprocessing: str | |
| # The corpus's OWN licence, which is not the platform's. A researcher has to | |
| # know the terms they are working under before they analyse someone else's | |
| # data, so it is shown wherever the corpus is described. | |
| license: str = "" | |
| # First in the list and selected on load. NOT a claim that it is 'the' | |
| # corpus: the PI's ask was for CAMEL to be available and ready to use. | |
| preselected: bool = False | |
| storage_key: str = "" | |
| # Optional mark for the dataset, served from sample_data/ at /samples. | |
| # Shown in the info dialog, where it identifies THIS dataset - not in the | |
| # picker row, which lists several and would imply they share a source. | |
| logo: str = "" | |
| # One line under the name in the sample-data list, so a researcher can judge | |
| # fit before running. The row already shows the count and language. | |
| detail: str = "" | |
| def path(self) -> Path: | |
| return SAMPLES_DIR / self.filename | |
| def bundled(self) -> bool: | |
| return not self.storage_key | |
| def available(self) -> bool: | |
| """Bundled files are checked on disk; stored ones are assumed present - | |
| a HEAD on every listing would add a network round trip to page load, | |
| and a missing object surfaces as a clear 404 on selection instead.""" | |
| return self.path.exists() if self.bundled else True | |
| def blocked_reason(self, row_ceiling: int) -> str: | |
| """Why this instance cannot run this corpus, or "" if it can. | |
| A corpus larger than CCR_MAX_ROWS is refused at ingest, so offering it | |
| as a working option produces a dead end. Saying so up front, and why, | |
| beats an error after the click - and it corrects itself the moment the | |
| deployment raises the limit. | |
| """ | |
| if not self.available(): | |
| return "Not loaded on this instance." | |
| if row_ceiling and self.n_rows > row_ceiling: | |
| return ( | |
| f"Too large for this instance: {self.n_rows:,} texts against a " | |
| f"{row_ceiling:,}-row limit." | |
| ) | |
| return "" | |
| def public(self, row_ceiling: int = 0) -> dict: | |
| data = asdict(self) | |
| data["bundled"] = self.bundled | |
| data["logo_url"] = ( | |
| f"/samples/{self.logo}" if self.logo and (SAMPLES_DIR / self.logo).exists() else "" | |
| ) | |
| data.pop("logo", None) | |
| blocked = self.blocked_reason(row_ceiling) | |
| data["usable"] = not blocked | |
| data["blocked_reason"] = blocked | |
| data.pop("storage_key", None) # internal location, not the user's business | |
| return data | |
| EXAMPLES: list[ExampleCorpus] = [ | |
| ExampleCorpus( | |
| id="camel_sample", | |
| name="CAMEL corpus (sample)", | |
| filename="camel_sample.csv", | |
| text_column="text", | |
| language="en", | |
| n_rows=999, | |
| description=( | |
| "The Cultural and Moral Expressions in Language (CAMEL) corpus consists of " | |
| "over 57 thousand texts taken from several platforms and annotated for 25 " | |
| "different cultural and moral constructs. This is a 999-text sample drawn " | |
| "from it, keeping the platform mix of the full corpus (Reddit, Twitter, " | |
| "IMDB, news, Wikipedia, and others). The full corpus, including the " | |
| "annotations, is on Hugging Face." | |
| ), | |
| citation=( | |
| "Zewail, A., Setia, A., Mohammadsadegh, R., Seker, F., Hajian, A., Sosa, H., " | |
| "Morhayim, L., Reddy, S., & Atari, M. (2026). The Cultural and Moral " | |
| "Expressions in Language (CAMEL) corpus [Preprint]." | |
| ), | |
| source_url="https://huggingface.co/datasets/Culture-and-Morality-Lab/CAMEL_Dataset", | |
| preprocessing=( | |
| "Usernames were removed by the corpus authors. For this sample: texts drawn " | |
| "proportionally by source platform with a fixed seed " | |
| "(scripts/build_camel_sample.py), whitespace trimmed, and only the text and " | |
| "source_platform columns kept. Text is otherwise verbatim." | |
| ), | |
| detail=( | |
| "Median 25 words, mean 69. Runs in seconds on any model. " | |
| "Best for a first look at how CCR works." | |
| ), | |
| license="CC BY-NC 4.0 (Creative Commons Attribution-NonCommercial 4.0). Free for non-commercial research with attribution; see the dataset page for the full terms.", | |
| preselected=True, | |
| logo="camel_logo.png", | |
| ), | |
| ExampleCorpus( | |
| id="camel_full", | |
| name="CAMEL corpus (full)", | |
| filename="", | |
| storage_key="examples/camel_full.csv", | |
| text_column="text", | |
| language="en", | |
| n_rows=57174, | |
| description=( | |
| "The complete Cultural and Moral Expressions in Language (CAMEL) corpus: " | |
| "57,174 texts from Reddit, Twitter, IMDB, news, Wikipedia, the Internet " | |
| "Archive and other sources, annotated by the lab for 25 cultural and moral " | |
| "constructs. The annotation columns come with it, so CCR scores can be " | |
| "compared against the human ratings." | |
| ), | |
| citation=( | |
| "Zewail, A., Setia, A., Mohammadsadegh, R., Seker, F., Hajian, A., Sosa, H., " | |
| "Morhayim, L., Reddy, S., & Atari, M. (2026). The Cultural and Moral " | |
| "Expressions in Language (CAMEL) corpus [Preprint]." | |
| ), | |
| source_url="https://huggingface.co/datasets/Culture-and-Morality-Lab/CAMEL_Dataset", | |
| preprocessing=( | |
| "Usernames were removed by the corpus authors. Otherwise the corpus exactly " | |
| "as published: all 64 columns, text verbatim, no sampling." | |
| ), | |
| detail=( | |
| "Median 26 words, up to 1,770. Expect a few minutes on MiniLM and " | |
| "considerably longer on the large models; pick MiniLM unless you have a " | |
| "reason not to." | |
| ), | |
| license="CC BY-NC 4.0 (Creative Commons Attribution-NonCommercial 4.0). Free for non-commercial research with attribution; see the dataset page for the full terms.", | |
| logo="camel_logo.png", | |
| ), | |
| ] | |
| BY_ID = {e.id: e for e in EXAMPLES} | |
| def get(example_id: str) -> ExampleCorpus | None: | |
| return BY_ID.get(example_id) | |
| def listed(row_ceiling: int = 0) -> list[dict]: | |
| """Examples this instance has, preselected one first. | |
| Entries too large for the row ceiling are still listed but marked unusable | |
| with a reason, rather than hidden: a corpus quietly vanishing is harder to | |
| explain than one that says why it is unavailable. | |
| """ | |
| return [ | |
| e.public(row_ceiling) | |
| for e in sorted(EXAMPLES, key=lambda e: not e.preselected) | |
| if e.available() | |
| ] | |