Spaces:
Runtime error
Runtime error
File size: 2,200 Bytes
54cd75a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | """
Text cleaning utilities shared across all loaders.
Applied before tokenization — keeps text usable for both
classical NLP pipelines and transformer tokenizers.
"""
import re
from typing import List
from data.schema import Argument, Debate
# Reddit-specific noise
_REDDIT_QUOTE = re.compile(r"^>.*$", re.MULTILINE)
_URL = re.compile(r"https?://\S+|www\.\S+")
_SUBREDDIT_MENTION = re.compile(r"r/\w+")
_USER_MENTION = re.compile(r"u/\w+")
_EDIT_NOTE = re.compile(r"\*?edit\*?:.*", re.IGNORECASE | re.DOTALL)
_WHITESPACE = re.compile(r"\s+")
# Deleted/removed placeholder strings
_DELETED = {"[deleted]", "[removed]", ""}
def clean_text(text: str) -> str:
text = _REDDIT_QUOTE.sub("", text)
text = _URL.sub("", text)
text = _SUBREDDIT_MENTION.sub("", text)
text = _USER_MENTION.sub("", text)
text = _EDIT_NOTE.sub("", text)
text = _WHITESPACE.sub(" ", text)
return text.strip()
def is_valid(text: str, min_tokens: int = 5) -> bool:
"""Return False for deleted posts or suspiciously short text."""
if text in _DELETED:
return False
return len(text.split()) >= min_tokens
def clean_debate(debate: Debate, min_tokens: int = 5) -> Debate:
"""Return a new Debate with cleaned argument texts, dropping invalid ones."""
cleaned_args = []
for arg in debate.arguments:
text = clean_text(arg.text)
if is_valid(text, min_tokens):
cleaned_args.append(
Argument(
id=arg.id,
text=text,
arg_type=arg.arg_type,
parent_id=arg.parent_id,
author=arg.author,
score=arg.score,
metadata=arg.metadata,
)
)
return Debate(
id=debate.id,
title=debate.title,
source=debate.source,
arguments=cleaned_args,
metadata=debate.metadata,
)
def clean_debates(debates: List[Debate], min_tokens: int = 5) -> List[Debate]:
cleaned = [clean_debate(d, min_tokens) for d in debates]
# Drop debates that lost their root claim during cleaning
return [d for d in cleaned if d.root() is not None]
|