Spaces:
Build error
Build error
Download data_ingestion/preprocess_data.py from Proton-Datalabs-dev/Smart-Tasker: direct link, hf CLI and curl.
- Browser
- Download file 498 Bytes
-
https://huggingface.co/spaces/Proton-Datalabs-dev/Smart-Tasker/resolve/main/data_ingestion/preprocess_data.py
- Command line
-
hf download hf://spaces/Proton-Datalabs-dev/Smart-Tasker/data_ingestion/preprocess_data.py
-
curl -L -o preprocess_data.py https://huggingface.co/spaces/Proton-Datalabs-dev/Smart-Tasker/resolve/main/data_ingestion/preprocess_data.py
498 Bytes
| # data_ingestion/preprocess_data.py | |
| import re | |
| def preprocess_text(data): | |
| """Cleans extracted text data to retain only the author and their message.""" | |
| cleaned_data = [] | |
| for line in data: | |
| # Match pattern with author and text (e.g., "Author: Message") | |
| match = re.match(r"^(.*?):\s+(.*)$", line) | |
| if match: | |
| author, text = match.groups() | |
| cleaned_data.append({"author": author.strip(), "text": text.strip()}) | |
| return cleaned_data | |