falcon-h1-tiny-cyber / training_config.json
sourcepirate's picture
Upload folder using huggingface_hub
6f494ad verified
Raw History Blame Contribute Delete
3.51 kB
{
"model": {
"id": "tiiuae/Falcon-H1-Tiny-R-90M",
"dtype": "auto",
"attn": "sdpa",
"quant": "4bit",
"bnb_skip_modules": [
"out_proj",
"lm_head"
]
},
"data": {
"sources": [
{
"name": "redteam",
"repo": "WNT3D/Ultimate-Offensive-Red-Team",
"splits": [
"train"
],
"subset": null,
"parquet": "default/train/0000.parquet",
"revision": "refs/convert/parquet",
"cap": 20000
},
{
"name": "kali",
"repo": "suryanshp1/kali-linux-pentesting-data",
"splits": [
"train"
],
"subset": null,
"parquet": null,
"revision": null,
"cap": 343
},
{
"name": "kimi",
"repo": "echel0nn1881/kimi-cyber-reasoning",
"splits": [
"train"
],
"subset": null,
"parquet": null,
"revision": null,
"cap": 997
},
{
"name": "cti",
"repo": "reloading0101/threat-intelligence-dataset",
"splits": [
"train",
"validation"
],
"subset": "default",
"parquet": null,
"revision": null,
"cap": 6000
},
{
"name": "exploitdb",
"repo": "Waiper/ExploitDB_DataSet",
"splits": [
"train"
],
"subset": null,
"parquet": "datasets/exploitdb_dataset.parquet",
"revision": null,
"cap": 8000
},
{
"name": "linuxcmd",
"repo": "Romit2004/LinuxCommands",
"splits": [
"train"
],
"subset": null,
"parquet": null,
"revision": null,
"cap": 6000
},
{
"name": "debian",
"repo": "2etatg/Debian_Hacking_Networking",
"splits": [
"train"
],
"subset": null,
"parquet": null,
"revision": null,
"cap": 965
}
],
"output_dir": "data/processed",
"max_len": 1024,
"eval_max_len": 512,
"eval_size": 300,
"seed": 42
},
"lora": {
"r": 8,
"alpha": 16,
"dropout": 0.05,
"target_modules": [
"q_proj",
"k_proj",
"v_proj",
"o_proj",
"in_proj",
"gate_proj",
"up_proj",
"down_proj"
]
},
"train": {
"output_dir": "outputs/falcon-h1-tiny-cyber",
"micro_batch": 1,
"grad_accum": 8,
"epochs": 1,
"max_steps": 100,
"lr": 0.0002,
"warmup_ratio": 0.03,
"max_grad_norm": 1.0,
"optim": "paged_adamw_8bit",
"gradient_checkpointing": true,
"logging_steps": 10,
"save_steps": 20,
"eval_steps": 20,
"save_total_limit": 2,
"report_to": [],
"num_workers": 2,
"seed": 42,
"resume": true,
"generation_gate": true,
"gate_prompts": [
"Write a tcpdump command for DNS only",
"Explain how SQL injection works.",
"List three Linux commands to inspect open ports."
],
"gate_max_new_tokens": 48
},
"export": {
"merged_dir": "outputs/falcon-h1-tiny-merged",
"gguf_dir": "outputs/gguf-tiny",
"quant": "Q8_0",
"llama_cpp_dir": "vendor/llama.cpp",
"llama_cpp_rev": null,
"hf_repo": "sourcepirate/falcon-h1-tiny-cyber"
},
"wandb": {
"enabled": "auto",
"project": "falcon-h1-cyber",
"entity": null,
"run_name": null,
"tags": [
"qlora",
"falcon-h1-tiny",
"cyber"
],
"mode": null
}
}