Text Classification
Transformers
Safetensors
English
modernbert
commitment-detection
decision-detection
experimental
vela
text-embeddings-inference
Instructions to use TwilightTechie/vela-commitment-classifier-307m with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use TwilightTechie/vela-commitment-classifier-307m with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="TwilightTechie/vela-commitment-classifier-307m")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("TwilightTechie/vela-commitment-classifier-307m") model = AutoModelForSequenceClassification.from_pretrained("TwilightTechie/vela-commitment-classifier-307m", device_map="auto") - Notebooks
- Google Colab
- Kaggle
File size: 3,284 Bytes
f0e7192 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 | """Reconstruct reviewed requests from pinned source data and text-free label sidecars."""
# ruff: noqa: PLC0415
import argparse
import hashlib
import json
from pathlib import Path
def sha256_text(text):
return hashlib.sha256(text.encode()).hexdigest()
def hydrate_records(sidecar, sources):
repository = sidecar["source"]["repository"]
result = []
if repository == "CohereLabs/aya_dataset":
lookup = {}
for row in sources:
if not row.get("user_id"):
continue
key = (
sha256_text(row["inputs"]),
row["language_code"],
"aya-author-" + sha256_text(row["user_id"]),
)
lookup[key] = row["inputs"]
for row in sidecar["records"]:
key = (row["source_text_sha256"], row["language"], row["group_id"])
if key not in lookup:
raise ValueError(
"Reviewed Aya request or author differs from the pinned source"
)
result.append({**row, "text": lookup[key]})
elif repository == "databricks/databricks-dolly-15k":
for row in sidecar["records"]:
source = sources[row["source_row"]]
instruction, context = (
source["instruction"].strip(),
source["context"].strip(),
)
text = instruction + ("\n\n" + context if context else "")
if sha256_text(text) != row["source_text_sha256"]:
raise ValueError(
"Reviewed Dolly request differs from the pinned source"
)
result.append(
{**row, "text": text, "instruction": instruction, "context": context}
)
else:
raise ValueError("Unknown reviewed source")
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--sidecar", type=Path, required=True)
parser.add_argument("--source", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
if args.output.exists():
raise ValueError("Refusing to overwrite reviewed annotations")
sidecar = json.loads(args.sidecar.read_text())
if (
hashlib.sha256(args.source.read_bytes()).hexdigest()
!= sidecar["source"]["file_sha256"]
):
raise ValueError("Source file hash differs from the reviewed revision")
if sidecar["source"]["repository"] == "CohereLabs/aya_dataset":
from pyarrow import parquet
sources = parquet.read_table(
args.source, columns=["inputs", "language_code", "user_id"]
).to_pylist()
else:
sources = [
json.loads(line)
for line in args.source.read_text().split("\n")
if line.strip()
]
rows = hydrate_records(sidecar, sources)
content = "".join(
json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n" for row in rows
)
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(content)
print(
json.dumps(
{"rows": len(rows), "sha256": hashlib.sha256(content.encode()).hexdigest()}
)
)
if __name__ == "__main__":
main()
|