| { |
| "_comment": [ |
| "Declared sources for CORTEX training data.", |
| "Every dataset must state its origin and license. training/train.py refuses to run on", |
| "a dataset that is not declared here or whose license is unknown, absent or proprietary.", |
| "Licenses below were read from the Hugging Face Hub API, not from memory." |
| ], |
| "schema_version": 1, |
| "datasets": [ |
| { |
| "id": "cortex-code-sft", |
| "source": "sahil2801/CodeAlpaca-20k", |
| "config": "default", |
| "split": "train", |
| "origin": "Hugging Face Hub dataset repository sahil2801/CodeAlpaca-20k", |
| "license": "cc-by-4.0", |
| "license_url": "https://huggingface.co/datasets/sahil2801/CodeAlpaca-20k", |
| "redistribution": "yes", |
| "attribution": "CodeAlpaca-20k, distributed under CC BY 4.0. Attribution required: cite the dataset repository above when redistributing.", |
| "record_type": "code-sft", |
| "fields": { |
| "prompt": "instruction", |
| "completion": "output", |
| "instruction_field": "instruction", |
| "input_field": "input" |
| }, |
| "id_field": null, |
| "path": "datasets/cortex-code-sft.jsonl", |
| "note": "Instruction/response pairs for code generation. CC BY 4.0 permits redistribution and commercial use with attribution." |
| }, |
| { |
| "id": "cortex-code-tests", |
| "source": "google-research-datasets/mbpp", |
| "config": "full", |
| "split": "test", |
| "origin": "Hugging Face Hub dataset repository google-research-datasets/mbpp", |
| "license": "cc-by-4.0", |
| "license_url": "https://huggingface.co/datasets/google-research-datasets/mbpp", |
| "redistribution": "yes", |
| "attribution": "MBPP (Mostly Basic Python Problems), Google Research, distributed under CC BY 4.0.", |
| "record_type": "code-sft", |
| "fields": { |
| "prompt": "text", |
| "completion": "code" |
| }, |
| "id_field": "task_id", |
| "path": "datasets/cortex-code-tests.jsonl", |
| "note": "Python problem statements with reference solutions. Suitable for evaluation and instruction tuning." |
| }, |
| { |
| "id": "cortex-code-eval", |
| "source": "openai/openai_humaneval", |
| "config": "openai_humaneval", |
| "split": "test", |
| "origin": "Hugging Face Hub dataset repository openai/openai_humaneval", |
| "license": "mit", |
| "license_url": "https://huggingface.co/datasets/openai/openai_humaneval", |
| "redistribution": "yes", |
| "attribution": "HumanEval, OpenAI, distributed under the MIT license.", |
| "record_type": "code-sft", |
| "fields": { |
| "prompt": "prompt", |
| "completion": "canonical_solution" |
| }, |
| "id_field": "task_id", |
| "path": "datasets/cortex-code-eval.jsonl", |
| "note": "Held out for evaluation only. Must not be trained on: doing so would invalidate the evaluation." |
| } |
| ], |
| "rejected": [ |
| { |
| "source": "bigcode/the-stack-dedup", |
| "license": "other", |
| "reason": "License field is 'other' with gated access; the governing terms are not verifiable from the Hub metadata alone." |
| }, |
| { |
| "source": "bigcode/starcoderdata", |
| "license": "other", |
| "reason": "License field is 'other' and access is gated; per-file source licenses are not declared at dataset level." |
| }, |
| { |
| "source": "codeparrot/codeparrot-clean", |
| "license": null, |
| "reason": "No license declared on the Hub. Cannot be redistributed or used without an explicit grant." |
| }, |
| { |
| "source": "codeparrot/github-code", |
| "license": "other", |
| "reason": "License field is 'other'; terms are not verifiable from Hub metadata alone." |
| }, |
| { |
| "source": "iamtarun/python_code_instructions_18k_alpaca", |
| "license": null, |
| "reason": "No license declared on the Hub." |
| } |
| ] |
| } |
|
|
