| # BITS Phishing Detector — all dials in one place. | |
| # Verified pins and the reasoning behind every number: docs/research/*.md (2026-10-07). | |
| seed: 42 | |
| paths: | |
| data_raw: data/raw | |
| data_processed: data/processed | |
| tokenizer: data/processed/tokenizer.json | |
| checkpoints: checkpoints | |
| reports: reports | |
| logs: logs | |
| app_dir: app | |
| # ---------------------------------------------------------------- model (Step 3) | |
| model: | |
| # "Large" BERT-style pre-LN encoder. 24L/1024/16/4096/32768/512 = 337,476,608 params with the | |
| # tied MLM head, 337,443,842 as the deployed classifier (+2.3% over the ~330M target). | |
| # Switch n_layers to 23 (324,880,384) only if ~330M must be a hard ceiling. | |
| n_layers: 24 | |
| d_model: 1024 | |
| n_heads: 16 # head_dim 64 -> Flash-SDPA eligible on sm_86 | |
| d_ff: 4096 | |
| vocab_size: 32768 # 2^15, tensor-core aligned | |
| max_seq_len: 512 | |
| dropout: 0.1 | |
| attention_dropout: 0.1 | |
| norm: pre_ln | |
| activation: gelu | |
| tie_mlm_head: true | |
| grad_checkpointing: false # B=16 fits without it; checkpointing costs ~1.33x compute | |
| # ---------------------------------------------------------------- tokenizer | |
| tokenizer: | |
| type: byte_level_bpe | |
| vocab_size: 32768 | |
| min_frequency: 2 | |
| special_tokens: ["[PAD]", "[UNK]", "[CLS]", "[SEP]", "[MASK]"] | |
| # Trained ONLY on the pretraining corpus + fine-tune train split. Never on val/test. | |
| train_on: [pretrain, finetune_train] | |
| # ---------------------------------------------------------------- data (Step 2 / data_prep) | |
| data: | |
| labelled_primary: puyang2025/seven-phishing-email-datasets # 203,017 rows; text/subject/label/dataset_name | |
| labelled_excluded: | |
| - it4lia/PhishingEmailCuratedDatasets_Cleaned # no body text (subjects + 53 numeric features only) | |
| - zefang-liu/phishing-email-dataset # 91.9% exact subset of the primary set, lossy text | |
| pretrain_corpus: corbt/enron-emails # 517,401 rows -> ~244k after exact dedup | |
| holdout_test_source: TREC-06 # never seen in tokenizer, pretraining, or fine-tuning | |
| second_choice_holdout: CEAS-08 # only after stripping mailing-list footers | |
| # Positive class = label 1 of the primary set ("phishing or spam / unwanted"). All label-1 rows | |
| # come from spam-track corpora with no spam-vs-phishing distinction; mapping spam->0 would leave | |
| # ~0 positives. PENDING Nathan's confirmation (see RUN_LOG.md entry 0). | |
| spam_policy: positive_class | |
| min_alnum_chars: 20 # labelled rows: min alphanumerics in subject+body | |
| pretrain_min_alnum_chars: 50 # corbt bodies shorter than this (after quote stripping) are dropped | |
| quote_strip_min_alnum: 200 # cut a quoted/forwarded tail only if this much text survives above it | |
| dedup: | |
| normalize: "NFKC; lowercase; keep [a-z0-9] only (common.normalize_text)" | |
| keys: [full_sha1, prefix300_sha1, subject_body_prefix300_sha1, subject_body_full_sha1] | |
| key_prefix_chars: 300 | |
| key_prefix_min_len: 80 | |
| near_dup_shingle_chars: 50 | |
| near_dup_stride: 25 | |
| near_dup_min_shared: 3 | |
| near_dup_window_chars: 1500 | |
| near_dup_shingle_cap: 2000 # shingles shared by more rows than this are boilerplate; ignored and counted | |
| finetune_val_fraction: 0.10 # grouped by near-dup cluster, stratified by source x label | |
| model_input: "subject + \"\\n\\n\" + body" # never sender/receiver (TREC headers are synthetic) | |
| synthetic: | |
| enabled: false # <= 2,000 rows, source=synthetic, training-only; results reported with/without | |
| max_rows: 2000 | |
| # Three-way labels (RUN_LOG entry 6; data_prep.py build3 / check3). label3: 0 legitimate / 1 phishing / 2 spam / | |
| # -1 unwanted-but-unsplit; label_set: legit / phish / spam / unwanted. The binary `label` column is unchanged. | |
| threeway: | |
| # Provenance of a binary-positive row -> label3. TREC-05/06/07 and CEAS-08 never separated spam from phishing | |
| # (-1: trains on P(phishing)+P(spam)); SpamAssassin / Enron-Spam / Ling label-1 rows are spam folders (2); | |
| # Nazario and Nigerian_Fraud are phishing corpora by construction (1). Every label-0 row is 0. | |
| positive_label3_by_source: | |
| Assassin: 2 | |
| Enron: 2 | |
| Ling: 2 | |
| TREC-05: -1 | |
| TREC-06: -1 | |
| TREC-07: -1 | |
| CEAS-08: -1 | |
| Nazario: 1 | |
| Nigerian_Fraud: 1 | |
| champa: # Champa et al. raw corpus mirror (CC-BY-4.0, byte-identical to Zenodo 8339691) | |
| repo: kudzaiprichard/aura-phishing-email-corpus | |
| files: [Nazario.csv, Nigerian_Fraud.csv] | |
| local_dir: data/raw/aura | |
| label3: 1 | |
| bigshp: # locuoco 300k set: research verdict DO_NOT_USE (docs/research/bigshp_research.md) | |
| enabled: false # set true only on a USE / USE_PARTIALLY verdict; the filter chain below then applies | |
| verdict: DO_NOT_USE | |
| repo: locuoco/the-biggest-spam-ham-phish-email-dataset-300000 | |
| file: df.csv | |
| local_dir: data/raw/bigshp | |
| drop_phish_class: true # its phish label is source-derived (87% spam-folder / spam-track mail) and stopword-stripped | |
| min_chars: 160 # SMS-like fragments | |
| min_english_fraction: 0.7 # label-2 rows: share of word tokens in our legitimate-mail vocabulary | |
| near_dup_token_key_len: 30 # within-file near-dup key: first 30 word tokens | |
| new_source_test_fraction: 0.15 # Nazario + Nigerian_Fraud: cluster-grouped slice -> test_holdout3 | |
| new_source_val_fraction: 0.10 # of the remainder -> finetune_val | |
| # A new near-dup cluster that links to BOTH train (or the MLM corpus) AND val is a 419 template the primary set | |
| # already holds on both sides; dropping it keeps train_vs_val_shingle_links at the binary build's 0 (costs ~424 | |
| # Nigerian_Fraud rows). false = pin such clusters to train (adds ~226 info-only train/val near-dup links). | |
| drop_new_clusters_linked_to_both_train_and_val: true | |
| holdout3_from_val: # clean-provenance legit / spam rows moved from finetune_val into test_holdout3 | |
| legit_sources: [CEAS-08, TREC-07, Assassin] | |
| spam_sources: [Assassin, Enron] | |
| legit_rows: 2000 | |
| spam_rows: 700 | |
| # exclusions: keys or >=3-shingle links into the pretraining corpus, clusters also in finetune_train, | |
| # clusters touched by a new row, clusters with an ineligible member (whole clusters move or none) | |
| display_names: {0: Legitimate, 1: Phishing, 2: Spam} | |
| binary_rollup_name: "Unwanted (phishing or spam)" | |
| # ---------------------------------------------------------------- VRAM policy | |
| vram: | |
| target_gb_low: 14 | |
| target_gb_high: 18 | |
| hard_cap_gb: 20 | |
| alloc_conf: "expandable_segments:True" | |
| log_peak_every_steps: 20 | |
| # ---------------------------------------------------------------- Phase 1: MLM pretraining | |
| pretrain: | |
| precision: bf16_autocast | |
| sequence_packing: true # no attn_mask -> Flash SDPA backend | |
| effective_batch_sequences: 256 # 256 x 512 = 131,072 tokens/step (BERT setting) | |
| micro_batch: auto # batch finder picks the largest candidate whose reserved peak is in the VRAM band | |
| micro_batch_start: 24 # Nathan's choice 2026-10-08 (17.65 GiB reserved, 25.2k tok/s); 256/24 -> accumulation 11 = effective 264 | |
| micro_batch_candidates: [16, 24, 12, 8] | |
| epochs: 12 # Nathan's choice 2026-10-08: ~14 h at 25k tok/s, ~9,550 steps; val MLM loss watched for a rise | |
| optimizer: adamw | |
| lr: 1.5e-4 | |
| betas: [0.9, 0.98] | |
| eps: 1.0e-6 | |
| weight_decay: 0.01 # excluded for biases and LayerNorm | |
| grad_clip: 1.0 | |
| warmup_steps: 1000 # cap; actual warmup = min(warmup_steps, warmup_fraction x total steps) | |
| warmup_fraction: 0.06 # 3 epochs (~2,460 steps) -> ~148 warmup steps; 6 epochs -> ~295 | |
| schedule: cosine_to_10pct | |
| mlm_probability: 0.15 | |
| mask_replace_probs: [0.8, 0.1, 0.1] # [MASK] / random token / unchanged | |
| loss_on_masked_only: true # gather masked positions before the MLM head | |
| checkpoint_every_steps: 500 | |
| log_every_steps: 20 | |
| eval_every_steps: 500 | |
| # ---------------------------------------------------------------- Phase 2: fine-tuning (the instructor's dials) | |
| finetune: | |
| # Three-way head (RUN_LOG entry 6): 0 Legitimate / 1 Phishing / 2 Spam. Lives here, NOT under model:, because the | |
| # live pretraining run refuses to resume if its model snapshot changes. 2 reproduces the binary baseline runs. | |
| # Rows with label_set "unwanted" (TREC-05/07, CEAS-08 positives) train on P(phishing)+P(spam) (partial-label NLL). | |
| num_labels: 3 | |
| epochs: 3 | |
| batch_size: 32 # examples per optimizer step (= micro_batch x accumulation) | |
| micro_batch: 16 # rows per forward/backward; measured ~11.8 GiB reserved at 16 x 512 | |
| eval_batch_size: 64 | |
| lr: 3.0e-5 | |
| weight_decay: 0.01 | |
| betas: [0.9, 0.98] | |
| eps: 1.0e-6 | |
| grad_clip: 1.0 | |
| warmup_fraction: 0.06 | |
| schedule: linear_decay | |
| eval_every_steps: 500 | |
| log_every_steps: 20 | |
| loss: cross_entropy # experiment set: cross_entropy | weighted_cross_entropy | focal (all over the partial-label NLL) | |
| class_weights: auto # inverse-frequency over fully-labelled rows when loss == weighted_cross_entropy; | |
| # "unwanted" rows get mean(w_phishing, w_spam) | |
| focal_gamma: 2.0 | |
| early_stopping: | |
| # val_f1_unwanted = F1 of the binary roll-up P(phishing)+P(spam) >= 0.5 (identical to the binary runs' val F1, | |
| # so results stay comparable); alternative: val_macro_f1_3way (macro-F1 over the three classes on val rows | |
| # with a full label). | |
| metric: val_f1_unwanted | |
| patience: 2 | |
| pooling: cls | |
| freeze_encoder_layers: 0 # experiment dial (instructor: "number of layers") | |
| dynamic_padding_multiple: 8 | |
| checkpoint_every_steps: 500 | |
| # ---------------------------------------------------------------- evaluation | |
| eval: | |
| metrics: [accuracy, precision, recall, f1, roc_auc, pr_auc, confusion_matrix, recall_at_fpr_1pct, recall_at_fpr_0_1pct] | |
| credential_lure_slice: true # secondary slice of the holdout matching credential-lure phrases | |
| latency_samples: 100 # avg ms/email, GPU and CPU, like the instructor's step 5 | |
| latency_max_len: 512 | |
| threshold: 0.5 # decision threshold; evaluate.py also reports the best-F1 threshold | |
| # Three-way evaluation (RUN_LOG entry 6). The binary metrics above are unchanged: they are computed on the | |
| # roll-up P(unwanted) = P(phishing) + P(spam) (= P(class 1) for a 2-class head), so results stay comparable | |
| # with the binary-era runs. Three-way metrics use only rows with label3 in {0,1,2}; -1/"unwanted" rows | |
| # (TREC-05 / TREC-07 / CEAS-08 positives) enter the roll-up only. | |
| test3_file: test_holdout3.parquet # clean-provenance three-way holdout written by data_prep (--split test3); read by evaluate.py | |
| # The three keys below RESTATE the label contract that is fixed in code (evaluate.LABEL_NAMES / UNWANTED_NAME / | |
| # DECISION_RULE, mirrored in export.py and the app). evaluate.py asserts at startup that they still match and | |
| # refuses to run otherwise, so a drift is loud; editing them here does not rename anything. | |
| label_names: {0: Legitimate, 1: Phishing, 2: Spam} | |
| unwanted_name: "Unwanted (phishing or spam)" | |
| threeway_decision_rule: rollup_then_argmax # unwanted iff P(unwanted) >= threshold, then argmax(P(phishing), P(spam)); argmax-of-3 reported alongside | |
| # documentation only: the sections every evaluated three-way block contains (not read by any script) | |
| metrics_threeway: [confusion_matrix_3x3, per_class_precision_recall_f1, macro_f1, phishing_vs_spam_on_true_unwanted, per_source_counts] | |
| top_k_confusions: 25 # read by evaluate.py: size of the phishing->spam / spam->phishing preview tables in error_analysis_test3.md | |
| model_card_metrics3: reports/metrics_test3.json # read by export.py: three-way table source (binary table: reports/metrics.json); both files must name the exported checkpoint | |
| # ---------------------------------------------------------------- Hugging Face (ask before any push) | |
| hf: | |
| owner: BuildITSystems | |
| model_repo: BuildITSystems/phishing-detector-encoder | |
| space_repo: BuildITSystems/phishing-detector-demo | |
| private: true | |
| space_sdk: gradio | |
| space_sdk_version: "6.29.1" | |
| space_python_version: "3.12.12" # ZeroGPU supports exactly 3.12.12 / 3.10.13 | |
| space_hardware: zero-a10g # ZeroGPU flavor string (backing GPU: RTX Pro 6000 Blackwell slice) | |
| space_torch: "2.13.0" # highest version on the ZeroGPU allow-list (2.14.x not listed) | |
| weights_format: safetensors_bf16 | |