{
  "_comment": [
    "Declared sources for CORTEX training data.",
    "Every dataset must state its origin and license. training/train.py refuses to run on",
    "a dataset that is not declared here or whose license is unknown, absent or proprietary.",
    "Licenses below were read from the Hugging Face Hub API, not from memory."
  ],
  "schema_version": 1,
  "datasets": [
    {
      "id": "cortex-code-sft",
      "source": "sahil2801/CodeAlpaca-20k",
      "config": "default",
      "split": "train",
      "origin": "Hugging Face Hub dataset repository sahil2801/CodeAlpaca-20k",
      "license": "cc-by-4.0",
      "license_url": "https://huggingface.co/datasets/sahil2801/CodeAlpaca-20k",
      "redistribution": "yes",
      "attribution": "CodeAlpaca-20k, distributed under CC BY 4.0. Attribution required: cite the dataset repository above when redistributing.",
      "record_type": "code-sft",
      "fields": {
        "prompt": "instruction",
        "completion": "output",
        "instruction_field": "instruction",
        "input_field": "input"
      },
      "id_field": null,
      "path": "datasets/cortex-code-sft.jsonl",
      "note": "Instruction/response pairs for code generation. CC BY 4.0 permits redistribution and commercial use with attribution."
    },
    {
      "id": "cortex-code-tests",
      "source": "google-research-datasets/mbpp",
      "config": "full",
      "split": "test",
      "origin": "Hugging Face Hub dataset repository google-research-datasets/mbpp",
      "license": "cc-by-4.0",
      "license_url": "https://huggingface.co/datasets/google-research-datasets/mbpp",
      "redistribution": "yes",
      "attribution": "MBPP (Mostly Basic Python Problems), Google Research, distributed under CC BY 4.0.",
      "record_type": "code-sft",
      "fields": {
        "prompt": "text",
        "completion": "code"
      },
      "id_field": "task_id",
      "path": "datasets/cortex-code-tests.jsonl",
      "note": "Python problem statements with reference solutions. Suitable for evaluation and instruction tuning."
    },
    {
      "id": "cortex-code-eval",
      "source": "openai/openai_humaneval",
      "config": "openai_humaneval",
      "split": "test",
      "origin": "Hugging Face Hub dataset repository openai/openai_humaneval",
      "license": "mit",
      "license_url": "https://huggingface.co/datasets/openai/openai_humaneval",
      "redistribution": "yes",
      "attribution": "HumanEval, OpenAI, distributed under the MIT license.",
      "record_type": "code-sft",
      "fields": {
        "prompt": "prompt",
        "completion": "canonical_solution"
      },
      "id_field": "task_id",
      "path": "datasets/cortex-code-eval.jsonl",
      "note": "Held out for evaluation only. Must not be trained on: doing so would invalidate the evaluation."
    }
  ],
  "rejected": [
    {
      "source": "bigcode/the-stack-dedup",
      "license": "other",
      "reason": "License field is 'other' with gated access; the governing terms are not verifiable from the Hub metadata alone."
    },
    {
      "source": "bigcode/starcoderdata",
      "license": "other",
      "reason": "License field is 'other' and access is gated; per-file source licenses are not declared at dataset level."
    },
    {
      "source": "codeparrot/codeparrot-clean",
      "license": null,
      "reason": "No license declared on the Hub. Cannot be redistributed or used without an explicit grant."
    },
    {
      "source": "codeparrot/github-code",
      "license": "other",
      "reason": "License field is 'other'; terms are not verifiable from Hub metadata alone."
    },
    {
      "source": "iamtarun/python_code_instructions_18k_alpaca",
      "license": null,
      "reason": "No license declared on the Hub."
    }
  ]
}