{ "model": { "id": "tiiuae/Falcon-H1-Tiny-R-90M", "dtype": "auto", "attn": "sdpa", "quant": "4bit", "bnb_skip_modules": [ "out_proj", "lm_head" ] }, "data": { "sources": [ { "name": "redteam", "repo": "WNT3D/Ultimate-Offensive-Red-Team", "splits": [ "train" ], "subset": null, "parquet": "default/train/0000.parquet", "revision": "refs/convert/parquet", "cap": 20000 }, { "name": "kali", "repo": "suryanshp1/kali-linux-pentesting-data", "splits": [ "train" ], "subset": null, "parquet": null, "revision": null, "cap": 343 }, { "name": "kimi", "repo": "echel0nn1881/kimi-cyber-reasoning", "splits": [ "train" ], "subset": null, "parquet": null, "revision": null, "cap": 997 }, { "name": "cti", "repo": "reloading0101/threat-intelligence-dataset", "splits": [ "train", "validation" ], "subset": "default", "parquet": null, "revision": null, "cap": 6000 }, { "name": "exploitdb", "repo": "Waiper/ExploitDB_DataSet", "splits": [ "train" ], "subset": null, "parquet": "datasets/exploitdb_dataset.parquet", "revision": null, "cap": 8000 }, { "name": "linuxcmd", "repo": "Romit2004/LinuxCommands", "splits": [ "train" ], "subset": null, "parquet": null, "revision": null, "cap": 6000 }, { "name": "debian", "repo": "2etatg/Debian_Hacking_Networking", "splits": [ "train" ], "subset": null, "parquet": null, "revision": null, "cap": 965 } ], "output_dir": "data/processed", "max_len": 1024, "eval_max_len": 512, "eval_size": 300, "seed": 42 }, "lora": { "r": 8, "alpha": 16, "dropout": 0.05, "target_modules": [ "q_proj", "k_proj", "v_proj", "o_proj", "in_proj", "gate_proj", "up_proj", "down_proj" ] }, "train": { "output_dir": "outputs/falcon-h1-tiny-cyber", "micro_batch": 1, "grad_accum": 8, "epochs": 1, "max_steps": 100, "lr": 0.0002, "warmup_ratio": 0.03, "max_grad_norm": 1.0, "optim": "paged_adamw_8bit", "gradient_checkpointing": true, "logging_steps": 10, "save_steps": 20, "eval_steps": 20, "save_total_limit": 2, "report_to": [], "num_workers": 2, "seed": 42, "resume": true, "generation_gate": true, "gate_prompts": [ "Write a tcpdump command for DNS only", "Explain how SQL injection works.", "List three Linux commands to inspect open ports." ], "gate_max_new_tokens": 48 }, "export": { "merged_dir": "outputs/falcon-h1-tiny-merged", "gguf_dir": "outputs/gguf-tiny", "quant": "Q8_0", "llama_cpp_dir": "vendor/llama.cpp", "llama_cpp_rev": null, "hf_repo": "sourcepirate/falcon-h1-tiny-cyber" }, "wandb": { "enabled": "auto", "project": "falcon-h1-cyber", "entity": null, "run_name": null, "tags": [ "qlora", "falcon-h1-tiny", "cyber" ], "mode": null } }