Download data_loader.py from JetBrains-Research/commit-message-editing: direct link, hf CLI and curl.
- Browser
- Download file 837 Bytes
-
https://huggingface.co/spaces/JetBrains-Research/commit-message-editing/resolve/2735b5e4c0050fca23cebca8a54d7e1b763f1237/data_loader.py
- Command line
-
hf download hf://spaces/JetBrains-Research/commit-message-editing@2735b5e4c0050fca23cebca8a54d7e1b763f1237/data_loader.py
-
curl -L -o data_loader.py https://huggingface.co/spaces/JetBrains-Research/commit-message-editing/resolve/2735b5e4c0050fca23cebca8a54d7e1b763f1237/data_loader.py
837 Bytes
| from datasets import load_dataset | |
| MODEL = 'cmg_gpt_4_0613' | |
| CACHE_DIR = 'cache' | |
| def load_data(): | |
| dataset = load_dataset("JetBrains-Research/lca-cmg", | |
| "commitchronicle-py-long", | |
| split="test", | |
| cache_dir=CACHE_DIR).to_pandas().set_index(['hash', 'repo']).rename( | |
| columns={'message': 'reference'}) | |
| model_dataset = load_dataset("JetBrains-Research/lca-results", | |
| MODEL, | |
| split="test", | |
| cache_dir=CACHE_DIR).to_pandas().set_index(['hash', 'repo'])[["prediction"]] | |
| model_dataset = model_dataset[~model_dataset.index.duplicated(keep='first')] | |
| dataset = dataset.join(other=model_dataset) | |
| return dataset.reset_index().to_dict('records') | |