razz-123 commited on
Commit
042eb00
·
verified ·
1 Parent(s): 6f7374a

Upload folder using huggingface_hub

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +7 -0
  2. Dockerfile +18 -0
  3. __pycache__/attention.cpython-314.pyc +0 -0
  4. __pycache__/config.cpython-314.pyc +0 -0
  5. __pycache__/dataset.cpython-314.pyc +0 -0
  6. __pycache__/embedding.cpython-314.pyc +0 -0
  7. __pycache__/feedforward.cpython-314.pyc +0 -0
  8. __pycache__/inference.cpython-314.pyc +0 -0
  9. __pycache__/layernorm.cpython-314.pyc +0 -0
  10. __pycache__/loss.cpython-314.pyc +0 -0
  11. __pycache__/model.cpython-314.pyc +0 -0
  12. __pycache__/positional_encoding.cpython-314.pyc +0 -0
  13. __pycache__/retriever.cpython-314.pyc +0 -0
  14. __pycache__/tokenizer.cpython-314.pyc +0 -0
  15. __pycache__/trainer.cpython-314.pyc +0 -0
  16. __pycache__/transformer_block.cpython-314.pyc +0 -0
  17. __pycache__/utils.cpython-314.pyc +0 -0
  18. attention.py +48 -0
  19. chat_template.py +36 -0
  20. config.py +56 -0
  21. data/code_train.txt +276 -0
  22. data/code_valid.txt +73 -0
  23. data/cyber_train.txt +83 -0
  24. data/cyber_valid.txt +19 -0
  25. data/general_train.txt +65 -0
  26. data/general_valid.txt +35 -0
  27. data/tokenizer/tokenizer.json +169 -0
  28. data/train.txt +56 -0
  29. data/valid.txt +29 -0
  30. data_folder/.gitattributes +60 -0
  31. data_folder/CyberSec-Dataset_escaped.jsonl +3 -0
  32. data_folder/Fenrir.png +3 -0
  33. data_folder/README.md +156 -0
  34. data_preprocessor.py +63 -0
  35. dataset.py +181 -0
  36. dpo_trainer.py +99 -0
  37. embedding.py +11 -0
  38. eval_math.py +48 -0
  39. faiss_index.faiss +0 -0
  40. faiss_index_chunks.txt +1 -0
  41. feedforward.py +14 -0
  42. inference.py +107 -0
  43. kbidu_env/.gitignore +2 -0
  44. kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd +3 -0
  45. kbidu_env/Lib/site-packages/__pycache__/isympy.cpython-314.pyc +0 -0
  46. kbidu_env/Lib/site-packages/__pycache__/six.cpython-314.pyc +0 -0
  47. kbidu_env/Lib/site-packages/__pycache__/threadpoolctl.cpython-314.pyc +0 -0
  48. kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc +3 -0
  49. kbidu_env/Lib/site-packages/_distutils_hack/__init__.py +220 -0
  50. kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-314.pyc +0 -0
.gitattributes CHANGED
@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
 
 
 
 
 
 
 
 
33
  *.zip filter=lfs diff=lfs merge=lfs -text
34
  *.zst filter=lfs diff=lfs merge=lfs -text
35
  *tfevents* filter=lfs diff=lfs merge=lfs -text
36
+ data_folder/CyberSec-Dataset_escaped.jsonl filter=lfs diff=lfs merge=lfs -text
37
+ data_folder/Fenrir.png filter=lfs diff=lfs merge=lfs -text
38
+ kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
39
+ kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text
40
+ kbidu_env/Lib/site-packages/accelerate/__pycache__/accelerator.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text
41
+ kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/dataclasses.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text
42
+ kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/modeling.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text
Dockerfile ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM nvidia/cuda:12.4.0-runtime-ubuntu22.04
2
+
3
+ ENV DEBIAN_FRONTEND=noninteractive
4
+ ENV PYTHONUNBUFFERED=1
5
+
6
+ RUN apt-get update && apt-get install -y \
7
+ python3.10 python3-pip git wget \
8
+ && ln -s /usr/bin/python3.10 /usr/bin/python \
9
+ && rm -rf /var/lib/apt/lists/*
10
+
11
+ WORKDIR /workspace
12
+ COPY requirements.txt .
13
+ RUN pip install --no-cache-dir -r requirements.txt
14
+
15
+ COPY . ./KbiduAI/
16
+ WORKDIR /workspace/KbiduAI
17
+
18
+ ENTRYPOINT ["torchrun", "--nproc_per_node=1", "main.py"]
__pycache__/attention.cpython-314.pyc ADDED
Binary file (3.55 kB). View file
 
__pycache__/config.cpython-314.pyc ADDED
Binary file (1.29 kB). View file
 
__pycache__/dataset.cpython-314.pyc ADDED
Binary file (10.1 kB). View file
 
__pycache__/embedding.cpython-314.pyc ADDED
Binary file (1.07 kB). View file
 
__pycache__/feedforward.cpython-314.pyc ADDED
Binary file (1.53 kB). View file
 
__pycache__/inference.cpython-314.pyc ADDED
Binary file (6.02 kB). View file
 
__pycache__/layernorm.cpython-314.pyc ADDED
Binary file (1.36 kB). View file
 
__pycache__/loss.cpython-314.pyc ADDED
Binary file (1.36 kB). View file
 
__pycache__/model.cpython-314.pyc ADDED
Binary file (3.6 kB). View file
 
__pycache__/positional_encoding.cpython-314.pyc ADDED
Binary file (2.21 kB). View file
 
__pycache__/retriever.cpython-314.pyc ADDED
Binary file (5.71 kB). View file
 
__pycache__/tokenizer.cpython-314.pyc ADDED
Binary file (1.82 kB). View file
 
__pycache__/trainer.cpython-314.pyc ADDED
Binary file (6.11 kB). View file
 
__pycache__/transformer_block.cpython-314.pyc ADDED
Binary file (1.83 kB). View file
 
__pycache__/utils.cpython-314.pyc ADDED
Binary file (1.42 kB). View file
 
attention.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # attention.py
2
+ import torch
3
+ import torch.nn as nn
4
+ import torch.nn.functional as F
5
+
6
+ class MultiHeadAttention(nn.Module):
7
+ def __init__(self, d_model, num_heads, num_kv_heads=None, dropout=0.1):
8
+ super().__init__()
9
+ assert d_model % num_heads == 0
10
+ self.d_model = d_model
11
+ self.num_heads = num_heads
12
+ self.num_kv_heads = num_kv_heads or num_heads
13
+ self.head_dim = d_model // num_heads
14
+
15
+ self.W_q = nn.Linear(d_model, num_heads * self.head_dim, bias=False)
16
+ self.W_k = nn.Linear(d_model, self.num_kv_heads * self.head_dim, bias=False)
17
+ self.W_v = nn.Linear(d_model, self.num_kv_heads * self.head_dim, bias=False)
18
+ self.W_o = nn.Linear(num_heads * self.head_dim, d_model, bias=False)
19
+ self.dropout = nn.Dropout(dropout)
20
+
21
+ def forward(self, x, mask=None, rotary=None):
22
+ batch, seq_len, _ = x.size()
23
+
24
+ q = self.W_q(x).view(batch, seq_len, self.num_heads, self.head_dim)
25
+ k = self.W_k(x).view(batch, seq_len, self.num_kv_heads, self.head_dim)
26
+ v = self.W_v(x).view(batch, seq_len, self.num_kv_heads, self.head_dim)
27
+
28
+ if rotary is not None:
29
+ q = rotary(q, seq_len)
30
+ k = rotary(k, seq_len)
31
+
32
+ if self.num_kv_heads != self.num_heads:
33
+ k = k.repeat_interleave(self.num_heads // self.num_kv_heads, dim=2)
34
+ v = v.repeat_interleave(self.num_heads // self.num_kv_heads, dim=2)
35
+
36
+ # Only use is_causal if mask is None, otherwise use explicit mask
37
+ use_causal = mask is None
38
+
39
+ attn_output = F.scaled_dot_product_attention(
40
+ q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2),
41
+ attn_mask=mask,
42
+ dropout_p=self.dropout.p if self.training else 0.0,
43
+ is_causal=use_causal
44
+ )
45
+ attn_output = attn_output.transpose(1, 2).contiguous().view(batch, seq_len, -1)
46
+ return self.W_o(attn_output)
47
+
48
+
chat_template.py ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # chat_template.py
2
+
3
+ class ChatTemplate:
4
+ def __init__(self, system_prompt=None):
5
+ self.system_prompt = system_prompt or "You are Kbidu AI, a helpful, harmless, and honest assistant specialized in mathematics and cybersecurity."
6
+ self.history = []
7
+
8
+ def format_messages(self, messages):
9
+ """
10
+ messages: list of dicts [{"role": "system/user/assistant", "content": "..."}]
11
+ Returns: formatted prompt string
12
+ """
13
+ formatted = []
14
+ if self.system_prompt:
15
+ formatted.append(f"System: {self.system_prompt}")
16
+
17
+ for msg in messages:
18
+ role = msg["role"].capitalize()
19
+ content = msg["content"]
20
+ formatted.append(f"{role}: {content}")
21
+
22
+ # Add final assistant turn
23
+ formatted.append("Assistant:")
24
+ return "\n".join(formatted)
25
+
26
+ def build_prompt(self, query, history=None):
27
+ """Build prompt from conversation history and new query."""
28
+ messages = []
29
+ if history:
30
+ messages.extend(history)
31
+ messages.append({"role": "user", "content": query})
32
+ return self.format_messages(messages)
33
+
34
+ # Usage:
35
+ # template = ChatTemplate()
36
+ # prompt = template.build_prompt("What is RSA encryption?", history=[...])
config.py ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # config.py
2
+
3
+ class KbiduTestConfig:
4
+ """Quick test mode – trains in ~2 minutes on a single GPU."""
5
+ vocab_size = 1000
6
+ d_model = 128
7
+ num_layers = 4
8
+ num_heads = 4
9
+ num_kv_heads = 2
10
+ d_ff = 256
11
+ max_seq_len = 64
12
+ dropout = 0.1
13
+ layer_norm_eps = 1e-6
14
+ use_gradient_checkpointing = True
15
+
16
+ batch_size = 4
17
+ learning_rate = 3e-4
18
+ weight_decay = 0.1
19
+ warmup_steps = 10
20
+ total_steps = 50
21
+ gradient_accumulation_steps = 2
22
+
23
+ use_streaming = True
24
+
25
+ # --- Vision settings (for multimodal training) ---
26
+ vision_model = "openai/clip-vit-base-patch32"
27
+ freeze_vision = True
28
+ num_image_tokens = 8
29
+
30
+
31
+ class Kbidu30BConfig:
32
+ """32B parameter model – production configuration."""
33
+ vocab_size = 50000
34
+ d_model = 7168
35
+ num_layers = 60
36
+ num_heads = 56
37
+ num_kv_heads = 8
38
+ d_ff = 19200
39
+ max_seq_len = 2048
40
+ dropout = 0.1
41
+ layer_norm_eps = 1e-6
42
+ use_gradient_checkpointing = True
43
+
44
+ batch_size = 2
45
+ learning_rate = 3e-4
46
+ weight_decay = 0.1
47
+ warmup_steps = 2000
48
+ total_steps = 100000
49
+ gradient_accumulation_steps = 8
50
+
51
+ use_streaming = True
52
+
53
+ # --- Vision settings (for multimodal training) ---
54
+ vision_model = "openai/clip-vit-base-patch32"
55
+ freeze_vision = True
56
+ num_image_tokens = 64
data/code_train.txt ADDED
@@ -0,0 +1,276 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def fibonacci(n):
2
+ if n <= 1:
3
+ return n
4
+ return fibonacci(n-1) + fibonacci(n-2)
5
+
6
+ class Person:
7
+ def __init__(self, name, age):
8
+ self.name = name
9
+ self.age = age
10
+
11
+ def greet(self):
12
+ return f"Hello, my name is {self.name}"
13
+
14
+ def is_prime(n):
15
+ if n < 2:
16
+ return False
17
+ for i in range(2, int(n**0.5) + 1):
18
+ if n % i == 0:
19
+ return False
20
+ return True
21
+
22
+ import pandas as pd
23
+ df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
24
+ print(df.mean())
25
+
26
+ def quick_sort(arr):
27
+ if len(arr) <= 1:
28
+ return arr
29
+ pivot = arr[len(arr) // 2]
30
+ left = [x for x in arr if x < pivot]
31
+ middle = [x for x in arr if x == pivot]
32
+ right = [x for x in arr if x > pivot]
33
+ return quick_sort(left) + middle + quick_sort(right)
34
+
35
+ class Stack:
36
+ def __init__(self):
37
+ self.items = []
38
+ def push(self, item):
39
+ self.items.append(item)
40
+ def pop(self):
41
+ return self.items.pop() if self.items else None
42
+ def peek(self):
43
+ return self.items[-1] if self.items else None
44
+
45
+ import requests
46
+ response = requests.get("https://api.github.com")
47
+ data = response.json()
48
+
49
+ def read_file(filepath):
50
+ with open(filepath, 'r') as f:
51
+ return f.read()
52
+
53
+ def write_file(filepath, content):
54
+ with open(filepath, 'w') as f:
55
+ f.write(content)
56
+
57
+ class LinkedList:
58
+ def __init__(self):
59
+ self.head = None
60
+ def append(self, data):
61
+ new_node = Node(data)
62
+ if not self.head:
63
+ self.head = new_node
64
+ return
65
+ current = self.head
66
+ while current.next:
67
+ current = current.next
68
+ current.next = new_node
69
+
70
+ import numpy as np
71
+ arr = np.array([1, 2, 3, 4, 5])
72
+ print(arr.mean())
73
+
74
+ def binary_search(arr, target):
75
+ low, high = 0, len(arr) - 1
76
+ while low <= high:
77
+ mid = (low + high) // 2
78
+ if arr[mid] == target:
79
+ return mid
80
+ elif arr[mid] < target:
81
+ low = mid + 1
82
+ else:
83
+ high = mid - 1
84
+ return -1
85
+
86
+ def merge_sort(arr):
87
+ if len(arr) <= 1:
88
+ return arr
89
+ mid = len(arr) // 2
90
+ left = merge_sort(arr[:mid])
91
+ right = merge_sort(arr[mid:])
92
+ return merge(left, right)
93
+
94
+ def merge(left, right):
95
+ result = []
96
+ i = j = 0
97
+ while i < len(left) and j < len(right):
98
+ if left[i] <= right[j]:
99
+ result.append(left[i])
100
+ i += 1
101
+ else:
102
+ result.append(right[j])
103
+ j += 1
104
+ result.extend(left[i:])
105
+ result.extend(right[j:])
106
+ return result
107
+
108
+ class Node:
109
+ def __init__(self, data):
110
+ self.data = data
111
+ self.next = None
112
+
113
+ import re
114
+ pattern = re.compile(r'\d+')
115
+ matches = pattern.findall("There are 123 apples and 456 oranges")
116
+
117
+ def factorial(n):
118
+ if n <= 1:
119
+ return 1
120
+ return n * factorial(n - 1)
121
+
122
+ def decorator(func):
123
+ def wrapper(*args, **kwargs):
124
+ print("Before function call")
125
+ result = func(*args, **kwargs)
126
+ print("After function call")
127
+ return result
128
+ return wrapper
129
+
130
+ @decorator
131
+ def hello():
132
+ print("Hello, World!")
133
+
134
+ import json
135
+ data = {"name": "Alice", "age": 30}
136
+ json_str = json.dumps(data)
137
+ parsed = json.loads(json_str)
138
+
139
+ def count_words(text):
140
+ words = text.split()
141
+ return len(words)
142
+
143
+ class Queue:
144
+ def __init__(self):
145
+ self.items = []
146
+ def enqueue(self, item):
147
+ self.items.append(item)
148
+ def dequeue(self):
149
+ return self.items.pop(0) if self.items else None
150
+
151
+ def reverse_string(s):
152
+ return s[::-1]
153
+
154
+ def is_palindrome(s):
155
+ return s == s[::-1]
156
+
157
+ import sys
158
+ print(sys.version)
159
+
160
+ def map_function(func, arr):
161
+ return [func(x) for x in arr]
162
+
163
+ def filter_function(func, arr):
164
+ return [x for x in arr if func(x)]
165
+
166
+ def reduce_function(func, arr):
167
+ result = arr[0]
168
+ for x in arr[1:]:
169
+ result = func(result, x)
170
+ return result
171
+
172
+ import os
173
+ files = os.listdir(".")
174
+
175
+ def power(base, exp):
176
+ return base ** exp
177
+
178
+ def gcd(a, b):
179
+ while b:
180
+ a, b = b, a % b
181
+ return a
182
+
183
+ def lcm(a, b):
184
+ return a * b // gcd(a, b)
185
+
186
+ def is_even(n):
187
+ return n % 2 == 0
188
+
189
+ def is_odd(n):
190
+ return n % 2 != 0
191
+
192
+ def sum_of_squares(arr):
193
+ return sum(x**2 for x in arr)
194
+
195
+ def average(arr):
196
+ return sum(arr) / len(arr) if arr else 0
197
+
198
+ def median(arr):
199
+ sorted_arr = sorted(arr)
200
+ n = len(sorted_arr)
201
+ if n % 2 == 0:
202
+ return (sorted_arr[n//2 - 1] + sorted_arr[n//2]) / 2
203
+ return sorted_arr[n//2]
204
+
205
+ def mode(arr):
206
+ from collections import Counter
207
+ counter = Counter(arr)
208
+ return max(counter, key=counter.get)
209
+
210
+ def std_dev(arr):
211
+ mean = average(arr)
212
+ variance = sum((x - mean) ** 2 for x in arr) / len(arr)
213
+ return variance ** 0.5
214
+
215
+ import math
216
+ print(math.pi)
217
+ print(math.e)
218
+
219
+ def sqrt(n):
220
+ return n ** 0.5
221
+
222
+ def log(n):
223
+ import math
224
+ return math.log(n)
225
+
226
+ def exp(n):
227
+ import math
228
+ return math.exp(n)
229
+
230
+ import datetime
231
+ now = datetime.datetime.now()
232
+ print(now.strftime("%Y-%m-%d %H:%M:%S"))
233
+
234
+ def get_date():
235
+ import datetime
236
+ return datetime.date.today()
237
+
238
+ def get_time():
239
+ import datetime
240
+ return datetime.datetime.now().time()
241
+
242
+ class BankAccount:
243
+ def __init__(self, balance=0):
244
+ self.balance = balance
245
+ def deposit(self, amount):
246
+ self.balance += amount
247
+ def withdraw(self, amount):
248
+ if amount > self.balance:
249
+ raise ValueError("Insufficient funds")
250
+ self.balance -= amount
251
+ def get_balance(self):
252
+ return self.balance
253
+
254
+ class SavingsAccount(BankAccount):
255
+ def __init__(self, balance=0, interest_rate=0.01):
256
+ super().__init__(balance)
257
+ self.interest_rate = interest_rate
258
+ def add_interest(self):
259
+ self.balance *= (1 + self.interest_rate)
260
+
261
+ import random
262
+ print(random.randint(1, 100))
263
+ print(random.choice(["apple", "banana", "cherry"]))
264
+
265
+ def shuffle_list(arr):
266
+ import random
267
+ random.shuffle(arr)
268
+ return arr
269
+
270
+ def sample_list(arr, k):
271
+ import random
272
+ return random.sample(arr, k)
273
+
274
+ def random_choice(arr):
275
+ import random
276
+ return random.choice(arr)
data/code_valid.txt ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def quicksort(arr):
2
+ if len(arr) <= 1:
3
+ return arr
4
+ pivot = arr[len(arr)//2]
5
+ left = [x for x in arr if x < pivot]
6
+ middle = [x for x in arr if x == pivot]
7
+ right = [x for x in arr if x > pivot]
8
+ return quicksort(left) + middle + quicksort(right)
9
+
10
+ class Animal:
11
+ def __init__(self, name):
12
+ self.name = name
13
+ def speak(self):
14
+ return f"{self.name} makes a sound"
15
+
16
+ class Dog(Animal):
17
+ def speak(self):
18
+ return f"{self.name} barks"
19
+
20
+ class Cat(Animal):
21
+ def speak(self):
22
+ return f"{self.name} meows"
23
+
24
+ def parse_json(json_str):
25
+ import json
26
+ return json.loads(json_str)
27
+
28
+ def to_json(obj):
29
+ import json
30
+ return json.dumps(obj)
31
+
32
+ def read_csv(filepath):
33
+ import csv
34
+ with open(filepath, 'r') as f:
35
+ reader = csv.reader(f)
36
+ return list(reader)
37
+
38
+ def write_csv(filepath, data):
39
+ import csv
40
+ with open(filepath, 'w', newline='') as f:
41
+ writer = csv.writer(f)
42
+ writer.writerows(data)
43
+
44
+ class ListNode:
45
+ def __init__(self, val=0, next=None):
46
+ self.val = val
47
+ self.next = next
48
+
49
+ def reverse_linked_list(head):
50
+ prev = None
51
+ current = head
52
+ while current:
53
+ next_node = current.next
54
+ current.next = prev
55
+ prev = current
56
+ current = next_node
57
+ return prev
58
+
59
+ def detect_cycle(head):
60
+ slow = fast = head
61
+ while fast and fast.next:
62
+ slow = slow.next
63
+ fast = fast.next.next
64
+ if slow == fast:
65
+ return True
66
+ return False
67
+
68
+ def find_middle(head):
69
+ slow = fast = head
70
+ while fast and fast.next:
71
+ slow = slow.next
72
+ fast = fast.next.next
73
+ return slow
data/cyber_train.txt ADDED
@@ -0,0 +1,83 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ SQL injection is a code injection technique that attackers use to exploit vulnerabilities in web applications. It occurs when user input is not properly sanitized and is directly embedded into SQL queries.
2
+
3
+ Buffer overflow is a vulnerability that occurs when a program writes more data to a buffer than it can hold, causing the excess data to overflow into adjacent memory locations.
4
+
5
+ Cross-Site Scripting (XSS) is a security vulnerability that allows attackers to inject malicious scripts into web pages viewed by other users.
6
+
7
+ Ransomware is a type of malware that encrypts a victim's files and demands a ransom payment in exchange for the decryption key.
8
+
9
+ A zero-day vulnerability is a previously unknown security flaw that has not yet been patched by the software vendor. Attackers can exploit these vulnerabilities before the vendor releases a fix.
10
+
11
+ Phishing is a social engineering attack that attempts to trick victims into revealing sensitive information by masquerading as a trustworthy entity.
12
+
13
+ The CIA triad (Confidentiality, Integrity, Availability) is the foundation of information security.
14
+
15
+ Public-key cryptography uses a pair of keys: a public key for encryption and a private key for decryption. The public key can be shared openly, while the private key must be kept secret.
16
+
17
+ Symmetric encryption uses the same key for both encryption and decryption. Both the sender and receiver must share the same secret key.
18
+
19
+ A firewall is a network security device that monitors and filters incoming and outgoing network traffic based on predetermined security rules.
20
+
21
+ Multi-factor authentication (MFA) requires users to provide two or more verification factors to gain access to a resource, significantly improving security.
22
+
23
+ A Denial-of-Service (DoS) attack aims to make a machine or network resource unavailable by overwhelming it with a flood of illegitimate requests.
24
+
25
+ The MITRE ATT&CK framework is a globally accessible knowledge base of adversary tactics and techniques based on real-world observations.
26
+
27
+ Vulnerability management is the cyclical practice of identifying, classifying, remediating, and mitigating vulnerabilities.
28
+
29
+ Penetration testing is an authorized simulated cyberattack on a computer system performed to evaluate the security of the system.
30
+
31
+ Security Information and Event Management (SIEM) provides real-time analysis of security alerts generated by applications and network hardware.
32
+
33
+ The OWASP Top 10 is a standard awareness document for developers and web application security, representing a broad consensus about the most critical security risks to web applications.
34
+
35
+ Cryptography is the practice and study of techniques for secure communication in the presence of adversarial behavior.
36
+
37
+ A vulnerability scanner is a tool that automatically identifies security weaknesses in systems and applications.
38
+
39
+ Social engineering is the psychological manipulation of people to perform actions or divulge confidential information.
40
+
41
+ Digital forensics is the process of uncovering and interpreting electronic data for use in a court of law.
42
+
43
+ Endpoint Detection and Response (EDR) is a cybersecurity technology that continuously monitors and responds to advanced threats.
44
+
45
+ Zero Trust security model assumes that no user, device, or network traffic should be trusted by default, even if it originates from inside the organization's network perimeter.
46
+
47
+ Access control is the selective restriction of access to a resource, ensuring that only authorized users can access sensitive data.
48
+
49
+ Data encryption transforms readable data into an unreadable format using an algorithm and a key, ensuring that only authorized parties can decrypt and access the data.
50
+
51
+ Intrusion Detection System (IDS) monitors network traffic for suspicious activity and alerts administrators when potential threats are detected.
52
+
53
+ Intrusion Prevention System (IPS) actively blocks detected threats in real-time, preventing them from reaching the target.
54
+
55
+ Role-Based Access Control (RBAC) restricts system access to authorized users based on their roles within an organization.
56
+
57
+ A man-in-the-middle attack occurs when an attacker secretly intercepts and relays messages between two parties who believe they are communicating directly with each other.
58
+
59
+ Vulnerability exploitation is the process of taking advantage of a vulnerability to compromise a system.
60
+
61
+ Patch management is the process of distributing and applying updates to software to fix security vulnerabilities and other bugs.
62
+
63
+ Security audits evaluate an organization's security posture by assessing its policies, procedures, and technical controls against established standards.
64
+
65
+ The Principle of Least Privilege states that users should be given only the minimum levels of access necessary to perform their job functions.
66
+
67
+ Security awareness training educates employees about security risks and best practices to reduce human error and improve overall security posture.
68
+
69
+ Ransomware attacks often use phishing emails to deliver the malware, tricking users into opening malicious attachments or clicking on malicious links.
70
+
71
+ Two-factor authentication (2FA) is a security process that requires users to provide two different authentication factors to verify their identity.
72
+
73
+ Security Operations Center (SOC) is a centralized unit that deals with security issues on an organizational and technical level.
74
+
75
+ Threat intelligence is evidence-based knowledge about existing or emerging threats that can be used to inform security decisions.
76
+
77
+ A security policy is a document that outlines an organization's security requirements, controls, and procedures.
78
+
79
+ Breach and Attack Simulation (BAS) is a security testing methodology that continuously simulates attacks on an organization's infrastructure to identify vulnerabilities.
80
+
81
+ Container security focuses on securing containerized applications and the infrastructure that supports them, including vulnerabilities in container images and runtime environments.
82
+
83
+ Security orchestration, automation, and response (SOAR) is a suite of software solutions that allow organizations to collect data about security threats and respond to events without human assistance.
data/cyber_valid.txt ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ The OWASP Top 10 includes Injection, Broken Authentication, Sensitive Data Exposure, and other critical security risks.
2
+
3
+ Defense in depth is a security strategy that layers multiple defensive mechanisms to protect systems and data, so if one layer fails, others still provide protection.
4
+
5
+ Public Key Infrastructure (PKI) is a system that creates, manages, distributes, and revokes digital certificates.
6
+
7
+ A certificate authority (CA) is a trusted entity that issues digital certificates used to verify the identity of individuals or organizations.
8
+
9
+ Digital signatures provide authentication, integrity, and non-repudiation for digital documents and transactions.
10
+
11
+ Hashing algorithms like SHA-256 are used to verify data integrity by producing a unique fixed-size hash for any given input.
12
+
13
+ Secure Sockets Layer (SSL) and Transport Layer Security (TLS) are cryptographic protocols that provide secure communication over a computer network.
14
+
15
+ The principle of least privilege ensures that users only have the access rights necessary to perform their job functions.
16
+
17
+ Identity and Access Management (IAM) is the framework of policies and technologies that ensure the right individuals access the right resources at the right times for the right reasons.
18
+
19
+ Privileged Access Management (PAM) focuses on securing, controlling, and monitoring access to privileged accounts.
data/general_train.txt ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Artificial intelligence is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction.
2
+
3
+ Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed.
4
+
5
+ Deep learning is a type of machine learning that uses neural networks with many layers to analyze various factors of data.
6
+
7
+ Natural language processing (NLP) is a branch of artificial intelligence that gives computers the ability to understand text and spoken words in much the same way human beings can.
8
+
9
+ Computer vision is a field of artificial intelligence that trains computers to interpret and understand the visual world.
10
+
11
+ The Internet of Things (IoT) refers to the network of physical objects embedded with sensors, software, and other technologies for the purpose of connecting and exchanging data with other devices and systems over the internet.
12
+
13
+ Blockchain technology is a decentralized, distributed ledger that records the provenance of a digital asset.
14
+
15
+ Cloud computing is the on-demand availability of computer system resources, especially data storage and computing power, without direct active management by the user.
16
+
17
+ Quantum computing is a type of computation that harnesses the collective properties of quantum states, such as superposition, interference, and entanglement, to perform calculations.
18
+
19
+ Big data refers to extremely large data sets that may be analyzed computationally to reveal patterns, trends, and associations.
20
+
21
+ Data science is an interdisciplinary field that uses scientific methods, processes, algorithms, and systems to extract knowledge and insights from structured and unstructured data.
22
+
23
+ Reinforcement learning is an area of machine learning concerned with how software agents ought to take actions in an environment to maximize the cumulative reward.
24
+
25
+ Generative AI refers to artificial intelligence systems that can generate new content such as text, images, audio, and video.
26
+
27
+ Neural networks are computing systems inspired by the biological neural networks that constitute animal brains.
28
+
29
+ Large Language Models (LLMs) are a type of artificial intelligence model that uses deep learning techniques and massive data sets to understand, summarize, generate, and predict new content.
30
+
31
+ Transformer architecture is a neural network architecture that uses self-attention mechanisms to process sequential data, and is the foundation of modern LLMs.
32
+
33
+ The attention mechanism in deep learning allows models to focus on relevant parts of the input when generating output, significantly improving performance on tasks like translation and summarization.
34
+
35
+ Transfer learning is a machine learning technique where a model trained on one task is reused as the starting point for a model on a second task.
36
+
37
+ Fine-tuning is the process of taking a pre-trained model and training it further on a new dataset to adapt it to a specific task.
38
+
39
+ Overfitting occurs when a machine learning model learns the training data too well, including its noise and outliers, resulting in poor performance on new data.
40
+
41
+ Regularization is a technique used to prevent overfitting by adding a penalty term to the loss function to discourage complex models.
42
+
43
+ Gradient descent is an optimization algorithm used to minimize the loss function by iteratively moving in the direction of steepest descent.
44
+
45
+ Backpropagation is an algorithm used to train neural networks by calculating gradients of the loss function with respect to the weights.
46
+
47
+ The learning rate is a hyperparameter that controls how much to change the model weights in response to the estimated error each time the model weights are updated.
48
+
49
+ Batch size is the number of training examples utilized in one iteration of model training.
50
+
51
+ Epoch refers to one complete pass through the entire training dataset during model training.
52
+
53
+ Activation functions such as ReLU, sigmoid, and tanh introduce non-linearity into neural networks, enabling them to learn complex patterns.
54
+
55
+ Dropout is a regularization technique where randomly selected neurons are ignored during training to prevent overfitting.
56
+
57
+ Convolutional Neural Networks (CNNs) are a class of deep neural networks commonly applied to analyzing visual imagery.
58
+
59
+ Recurrent Neural Networks (RNNs) are a class of neural networks that are designed to handle sequential data, such as time series or natural language.
60
+
61
+ Attention mechanisms in neural networks allow the model to weigh the importance of different input elements, improving performance on tasks like machine translation and text summarization.
62
+
63
+ Word embeddings are dense vector representations of words that capture semantic meaning, allowing models to understand word relationships.
64
+
65
+ Tokenization is the process of breaking down text into smaller units called tokens, which are the fundamental building blocks for language models.
data/general_valid.txt ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Supervised learning is a type of machine learning where models are trained on labeled data, learning to map inputs to outputs.
2
+
3
+ Unsupervised learning is a type of machine learning that uses unlabeled data to discover hidden patterns or data groupings.
4
+
5
+ Semi-supervised learning uses both labeled and unlabeled data to train machine learning models, combining the strengths of supervised and unsupervised learning.
6
+
7
+ Reinforcement learning involves training agents to make sequences of decisions by rewarding desirable behaviors and punishing undesirable ones.
8
+
9
+ Natural language understanding (NLU) is a subset of NLP that focuses on machine comprehension of human language, enabling systems to grasp meaning, intent, and context.
10
+
11
+ Natural language generation (NLG) is a subset of NLP that focuses on generating natural language text or speech from structured data.
12
+
13
+ Sentiment analysis uses NLP to identify and extract subjective information from source materials, determining the emotional tone behind words.
14
+
15
+ Machine translation is the task of automatically translating text from one language to another using computational methods.
16
+
17
+ Text summarization is the process of creating a concise and coherent summary of a longer text document.
18
+
19
+ Information retrieval is the process of obtaining relevant information from a collection of resources based on a user's query.
20
+
21
+ Question answering systems automatically answer questions posed by humans in natural language.
22
+
23
+ Named Entity Recognition (NER) is a subtask of NLP that locates and classifies named entities in text into predefined categories such as person names, organizations, locations, and more.
24
+
25
+ Coreference resolution is the task of finding all expressions that refer to the same entity in a text.
26
+
27
+ Dependency parsing is the process of analyzing the grammatical structure of a sentence to establish relationships between words.
28
+
29
+ Part-of-speech tagging is the process of assigning a part-of-speech label (noun, verb, adjective, etc.) to each word in a text.
30
+
31
+ Semantic similarity measures how similar two pieces of text are in terms of their meaning, which is crucial for many NLP applications.
32
+
33
+ Text classification is the task of assigning predefined categories to text documents.
34
+
35
+ Topic modeling is a technique for discovering abstract topics in a collection of documents.
data/tokenizer/tokenizer.json ADDED
@@ -0,0 +1,169 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": null,
4
+ "padding": null,
5
+ "added_tokens": [
6
+ {
7
+ "id": 0,
8
+ "content": "<PAD>",
9
+ "single_word": false,
10
+ "lstrip": false,
11
+ "rstrip": false,
12
+ "normalized": false,
13
+ "special": true
14
+ },
15
+ {
16
+ "id": 1,
17
+ "content": "<UNK>",
18
+ "single_word": false,
19
+ "lstrip": false,
20
+ "rstrip": false,
21
+ "normalized": false,
22
+ "special": true
23
+ },
24
+ {
25
+ "id": 2,
26
+ "content": "<BOS>",
27
+ "single_word": false,
28
+ "lstrip": false,
29
+ "rstrip": false,
30
+ "normalized": false,
31
+ "special": true
32
+ },
33
+ {
34
+ "id": 3,
35
+ "content": "<EOS>",
36
+ "single_word": false,
37
+ "lstrip": false,
38
+ "rstrip": false,
39
+ "normalized": false,
40
+ "special": true
41
+ }
42
+ ],
43
+ "normalizer": null,
44
+ "pre_tokenizer": {
45
+ "type": "ByteLevel",
46
+ "add_prefix_space": true,
47
+ "trim_offsets": true,
48
+ "use_regex": true
49
+ },
50
+ "post_processor": {
51
+ "type": "ByteLevel",
52
+ "add_prefix_space": true,
53
+ "trim_offsets": true,
54
+ "use_regex": true
55
+ },
56
+ "decoder": {
57
+ "type": "ByteLevel",
58
+ "add_prefix_space": true,
59
+ "trim_offsets": true,
60
+ "use_regex": true
61
+ },
62
+ "model": {
63
+ "type": "BPE",
64
+ "dropout": null,
65
+ "unk_token": null,
66
+ "continuing_subword_prefix": null,
67
+ "end_of_word_suffix": null,
68
+ "fuse_unk": false,
69
+ "byte_fallback": false,
70
+ "ignore_merges": false,
71
+ "vocab": {
72
+ "<PAD>": 0,
73
+ "<UNK>": 1,
74
+ "<BOS>": 2,
75
+ "<EOS>": 3,
76
+ ".": 4,
77
+ "T": 5,
78
+ "a": 6,
79
+ "d": 7,
80
+ "e": 8,
81
+ "g": 9,
82
+ "h": 10,
83
+ "i": 11,
84
+ "l": 12,
85
+ "m": 13,
86
+ "n": 14,
87
+ "o": 15,
88
+ "p": 16,
89
+ "r": 17,
90
+ "s": 18,
91
+ "t": 19,
92
+ "v": 20,
93
+ "x": 21,
94
+ "»": 22,
95
+ "¿": 23,
96
+ "ï": 24,
97
+ "Ċ": 25,
98
+ "č": 26,
99
+ "Ġ": 27,
100
+ "is": 28,
101
+ "Ġt": 29,
102
+ "Th": 30,
103
+ "ex": 31,
104
+ "in": 32,
105
+ "»¿": 33,
106
+ "": 34,
107
+ "čĊ": 35,
108
+ "Ġis": 36,
109
+ "Ġ": 37,
110
+ "Ġtex": 38,
111
+ "This": 39,
112
+ "Ġtext": 40
113
+ },
114
+ "merges": [
115
+ [
116
+ "i",
117
+ "s"
118
+ ],
119
+ [
120
+ "Ġ",
121
+ "t"
122
+ ],
123
+ [
124
+ "T",
125
+ "h"
126
+ ],
127
+ [
128
+ "e",
129
+ "x"
130
+ ],
131
+ [
132
+ "i",
133
+ "n"
134
+ ],
135
+ [
136
+ "»",
137
+ "¿"
138
+ ],
139
+ [
140
+ "ï",
141
+ "»¿"
142
+ ],
143
+ [
144
+ "č",
145
+ "Ċ"
146
+ ],
147
+ [
148
+ "Ġ",
149
+ "is"
150
+ ],
151
+ [
152
+ "Ġ",
153
+ ""
154
+ ],
155
+ [
156
+ "Ġt",
157
+ "ex"
158
+ ],
159
+ [
160
+ "Th",
161
+ "is"
162
+ ],
163
+ [
164
+ "Ġtex",
165
+ "t"
166
+ ]
167
+ ]
168
+ }
169
+ }
data/train.txt ADDED
@@ -0,0 +1,56 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ def hello_world():
2
+ print("Hello, World!")
3
+
4
+ SQL injection is a code injection technique used to attack data-driven applications.
5
+
6
+ Explain the concept of a buffer overflow vulnerability.
7
+
8
+ class Calculator:
9
+ def add(self, a, b):
10
+ return a + b
11
+
12
+ CVE-2024-1234 is a critical vulnerability in the Linux kernel.
13
+
14
+ Write a Python function to check if a number is prime.
15
+
16
+ Cybersecurity is the practice of protecting systems, networks, and programs from digital attacks.
17
+
18
+ What is a zero-day vulnerability?
19
+
20
+ def factorial(n):
21
+ if n <= 1:
22
+ return 1
23
+ return n * factorial(n - 1)
24
+
25
+ Ransomware is a type of malicious software that threatens to publish the victim's data.
26
+
27
+ How does public-key cryptography work?
28
+
29
+ import requests
30
+ response = requests.get("https://api.example.com/data")
31
+
32
+ A firewall is a network security device that monitors and filters incoming and outgoing network traffic.
33
+
34
+ Explain the difference between symmetric and asymmetric encryption.
35
+
36
+ def binary_search(arr, target):
37
+ left, right = 0, len(arr) - 1
38
+ while left <= right:
39
+ mid = (left + right) // 2
40
+ if arr[mid] == target:
41
+ return mid
42
+ elif arr[mid] < target:
43
+ left = mid + 1
44
+ else:
45
+ right = mid - 1
46
+ return -1
47
+
48
+ What is phishing?
49
+
50
+ Malware analysis is the process of understanding the behavior and purpose of a suspicious file or URL.
51
+
52
+ Use the os module to list all files in a directory.
53
+
54
+ Cross-site scripting (XSS) is a security vulnerability that allows attackers to inject malicious scripts into web pages.
55
+
56
+ Explain the concept of defense in depth.
data/valid.txt ADDED
@@ -0,0 +1,29 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ What is a DDoS attack?
2
+
3
+ A DDoS attack is a malicious attempt to disrupt normal traffic of a targeted server.
4
+
5
+ Write a Python decorator that measures execution time.
6
+
7
+ Encryption is the process of converting plaintext into ciphertext.
8
+
9
+ What is the OWASP Top 10?
10
+
11
+ def merge_sort(arr):
12
+ if len(arr) <= 1:
13
+ return arr
14
+ mid = len(arr) // 2
15
+ left = merge_sort(arr[:mid])
16
+ right = merge_sort(arr[mid:])
17
+ return merge(left, right)
18
+
19
+ What is a man-in-the-middle attack?
20
+
21
+ Secure coding practices help prevent vulnerabilities in software.
22
+
23
+ Explain the concept of least privilege.
24
+
25
+ Hash functions are used to verify data integrity.
26
+
27
+ What is a vulnerability scanner?
28
+
29
+ Firewalls can be hardware-based or software-based.
data_folder/.gitattributes ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ *.7z filter=lfs diff=lfs merge=lfs -text
2
+ *.arrow filter=lfs diff=lfs merge=lfs -text
3
+ *.bin filter=lfs diff=lfs merge=lfs -text
4
+ *.bz2 filter=lfs diff=lfs merge=lfs -text
5
+ *.ckpt filter=lfs diff=lfs merge=lfs -text
6
+ *.ftz filter=lfs diff=lfs merge=lfs -text
7
+ *.gz filter=lfs diff=lfs merge=lfs -text
8
+ *.h5 filter=lfs diff=lfs merge=lfs -text
9
+ *.joblib filter=lfs diff=lfs merge=lfs -text
10
+ *.lfs.* filter=lfs diff=lfs merge=lfs -text
11
+ *.lz4 filter=lfs diff=lfs merge=lfs -text
12
+ *.mds filter=lfs diff=lfs merge=lfs -text
13
+ *.mlmodel filter=lfs diff=lfs merge=lfs -text
14
+ *.model filter=lfs diff=lfs merge=lfs -text
15
+ *.msgpack filter=lfs diff=lfs merge=lfs -text
16
+ *.npy filter=lfs diff=lfs merge=lfs -text
17
+ *.npz filter=lfs diff=lfs merge=lfs -text
18
+ *.onnx filter=lfs diff=lfs merge=lfs -text
19
+ *.ot filter=lfs diff=lfs merge=lfs -text
20
+ *.parquet filter=lfs diff=lfs merge=lfs -text
21
+ *.pb filter=lfs diff=lfs merge=lfs -text
22
+ *.pickle filter=lfs diff=lfs merge=lfs -text
23
+ *.pkl filter=lfs diff=lfs merge=lfs -text
24
+ *.pt filter=lfs diff=lfs merge=lfs -text
25
+ *.pth filter=lfs diff=lfs merge=lfs -text
26
+ *.rar filter=lfs diff=lfs merge=lfs -text
27
+ *.safetensors filter=lfs diff=lfs merge=lfs -text
28
+ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
29
+ *.tar.* filter=lfs diff=lfs merge=lfs -text
30
+ *.tar filter=lfs diff=lfs merge=lfs -text
31
+ *.tflite filter=lfs diff=lfs merge=lfs -text
32
+ *.tgz filter=lfs diff=lfs merge=lfs -text
33
+ *.wasm filter=lfs diff=lfs merge=lfs -text
34
+ *.xz filter=lfs diff=lfs merge=lfs -text
35
+ *.zip filter=lfs diff=lfs merge=lfs -text
36
+ *.zst filter=lfs diff=lfs merge=lfs -text
37
+ *tfevents* filter=lfs diff=lfs merge=lfs -text
38
+ # Audio files - uncompressed
39
+ *.pcm filter=lfs diff=lfs merge=lfs -text
40
+ *.sam filter=lfs diff=lfs merge=lfs -text
41
+ *.raw filter=lfs diff=lfs merge=lfs -text
42
+ # Audio files - compressed
43
+ *.aac filter=lfs diff=lfs merge=lfs -text
44
+ *.flac filter=lfs diff=lfs merge=lfs -text
45
+ *.mp3 filter=lfs diff=lfs merge=lfs -text
46
+ *.ogg filter=lfs diff=lfs merge=lfs -text
47
+ *.wav filter=lfs diff=lfs merge=lfs -text
48
+ # Image files - uncompressed
49
+ *.bmp filter=lfs diff=lfs merge=lfs -text
50
+ *.gif filter=lfs diff=lfs merge=lfs -text
51
+ *.png filter=lfs diff=lfs merge=lfs -text
52
+ *.tiff filter=lfs diff=lfs merge=lfs -text
53
+ # Image files - compressed
54
+ *.jpg filter=lfs diff=lfs merge=lfs -text
55
+ *.jpeg filter=lfs diff=lfs merge=lfs -text
56
+ *.webp filter=lfs diff=lfs merge=lfs -text
57
+ # Video files - compressed
58
+ *.mp4 filter=lfs diff=lfs merge=lfs -text
59
+ *.webm filter=lfs diff=lfs merge=lfs -text
60
+ CyberSec-Dataset_escaped.jsonl filter=lfs diff=lfs merge=lfs -text
data_folder/CyberSec-Dataset_escaped.jsonl ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:da1d7b888aa697bd44b6e24ea95e02277c34bcd40c9cf6e701cf72bcac0f9143
3
+ size 360844505
data_folder/Fenrir.png ADDED

Git LFS Details

  • SHA256: fb5056862ee67f4e13f1ebfe422146b03d994ae249c4794d5684113674699df1
  • Pointer size: 132 Bytes
  • Size of remote file: 1.1 MB
data_folder/README.md ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ task_categories:
4
+ - text-generation
5
+ language:
6
+ - en
7
+ tags:
8
+ - cybersecurity
9
+ - defensive-security
10
+ - instruction-tuning
11
+ size_categories:
12
+ - 10K<n<100K
13
+ dataset_info:
14
+ version: 1.1.0
15
+ ---
16
+ # Cybersecurity Defense Instruction-Tuning Dataset (v2.0)
17
+
18
+ <img src="https://huggingface.co/datasets/AlicanKiraz0/Cybersecurity-Dataset-Fenrir-v2.0/resolve/main/Fenrir.png" width="700" />
19
+
20
+ Created by Alican Kiraz
21
+
22
+
23
+
24
+ ## TL;DR
25
+ A ready-to-train dataset of **83,920** high-quality *system / user / assistant* triples for **defensive, alignment-safe cybersecurity SFT** training.
26
+ Apache-2.0 licensed and production-ready.
27
+ **Scope:** OWASP Top 10, MITRE ATT&CK, NIST CSF, CIS Controls, ASD Essential 8, modern authentication (OAuth 2 / OIDC / SAML), SSL / TLS, Cloud & DevSecOps, Cryptography, and AI Security.
28
+
29
+ ---
30
+
31
+ ## 1  What’s new in v2.0  (2025‑10‑06)
32
+
33
+ | Change | v1.1.0 | **v2.0.0** |
34
+ | ----------------- | ------------------------------------: | ----------------------------------------------------------------------------------------------: |
35
+ | **Rows** | 21 258 | **83 920** *(≈ 4×)* |
36
+ | **Coverage** | OWASP, NIST CSF, +MITRE ATT&CK, CIS | + Deeper Cloud (AWS/Azure/GCP), modern auth hardening, crypto hygiene, AI‑security interplay |
37
+ | **Quality gates** | Dedup, PII scrub, hallucination scans | + adversarial refusal tests (jailbreak/injection), static policy linting, content risk taxonomy |
38
+ | **Format** | Parquet (chat triples) | + stricter schema checks, stable row IDs |
39
+ | **License** | Apache‑2.0 | Apache‑2.0 |
40
+
41
+
42
+ **Highlights**
43
+
44
+ * **Big**: 83,920 chat samples with explanations at **senior security engineer** level.
45
+ * **Safe-by-design**: Built-in **rejection patterns** and *alignment* checks against malicious or exploit requests.
46
+ * **Framework-aware**: Content is **mapped to standards** (OWASP / ATT&CK / NIST / CIS).
47
+ * **Cloud & DevSecOps first**: IAM, secrets, CI/CD, container / k8s hardening, logging / SIEM, incident response.
48
+ * **Commercial-friendly**: **Apache-2.0** licensed.
49
+
50
+ ---
51
+
52
+ ## 2  Dataset Summary
53
+
54
+ | Property | Value |
55
+ | ------------ | ------------------------------------------------------ |
56
+ | **Language** | English |
57
+ | **License** | Apache‑2.0 |
58
+ | **Format** | Parquet (columnar) |
59
+ | **Rows** | **83 920** |
60
+ | **Columns** | `system`, `user`, `assistant` |
61
+ | **Split** | `train` (100 %) |
62
+
63
+ **Record schema (chat triple)**
64
+
65
+ ```json
66
+ {
67
+ "system": "You are a seasoned cyber‑defense AI that follows industry ethics...",
68
+ "user": "Compare mitigations for Reflected vs Stored XSS in a modern SPA.",
69
+ "assistant": "Reflected and Stored XSS share core mitigation pillars—output encoding..."
70
+ }
71
+ ```
72
+
73
+ ---
74
+
75
+ ## 3. Coverage & Design
76
+
77
+ ### 3.1 Domains & Frameworks
78
+
79
+ * **AppSec & Web**: OWASP Top 10, secure coding, input/output handling, SSRF, deserialization.
80
+ * **Cloud Security**: IAM guardrails, least privilege, key rotation, KMS/HSM, network segmentation, posture mgmt.
81
+ * **DevSecOps**: SAST/DAST, SBOM, supply‑chain, CI/CD signing, container & Kubernetes hardening.
82
+ * **Identity & Access**: OAuth2/OIDC/SAML, MFA/Phishing‑resistant auth, session mgmt.
83
+ * **Crypto Hygiene**: TLS configs, AEAD modes, key lifecycle, randomness, password hashing.
84
+ * **Detection & Response**: logging, SIEM correlation, threat hunting, IR playbooks.
85
+ * **AI‑Security Interplay**: prompt injection defense, data‑poisoning awareness, model‑misuse refusals.
86
+
87
+ ### 3.2 Instruction styles
88
+
89
+ * Compare/contrast, step‑by‑step mitigation, checklists, “why it fails” root‑cause analyses, policy rationale, trade‑offs, and “refuse with explanation” for dual‑use prompts.
90
+
91
+ ---
92
+
93
+ ## 4. Data Creation & Quality
94
+
95
+ 1. **Source harvesting**: 250 k+ public technical docs (standards, RFCs, white‑papers, vendor guidance).
96
+ 2. **Extraction**: boilerplate stripping, language detection, heuristic paragraph segmentation.
97
+ 3. **Topical filtering**: keyword+embedding retrieval towards defensive security only.
98
+ 4. **Instruction synthesis**: prompts → *system/user/assistant*; enforced ethics & refusal templates.
99
+ 5. **Quality gates** *(multi‑layer)*
100
+
101
+ * **Deduplication**: MinHash + LSH cluster pruning.
102
+ * **PII & profanity scrub**.
103
+ * **Hallucination/inconsistency scans** (LLM‑aided).
104
+ * **Refusal‑pattern tests**: jailbreak & prompt‑injection triggers; no exploit‑building steps.
105
+ * **Manual spot review** (~3 % sample).
106
+
107
+ ---
108
+
109
+ ## 5. Ethical Use & Safety
110
+
111
+ * **Dual‑use risk**: Dataset intentionally avoids exploit crafting; offensive requests receive **explanatory refusals**.
112
+ * **Bias**: Focus on widely used frameworks (OWASP/NIST/CIS).
113
+
114
+ * *Roadmap*: more regional standards (e.g., ISO/IEC, GDPR security controls).
115
+ * **Provenance**: Only public sources; licensing respected; outputs released under **Apache‑2.0**.
116
+
117
+ ---
118
+
119
+ ## 6. Limitations
120
+
121
+ * English‑only.
122
+ * Predominantly defensive stance; red‑team tactics only for mitigation context.
123
+ * Security evolves rapidly; periodic refresh planned.
124
+
125
+ ---
126
+
127
+ ## 7. Example Records
128
+
129
+ **Mitigation checklist:** hardening steps, rationales, pitfalls, references to standards.
130
+ **Refusal sample:** clearly declines malware/exploit construction with safe alternatives (logging, detection, patching).
131
+
132
+ > *All examples adhere to the `system/user/assistant` schema and are engineered to be alignment‑safe.*
133
+
134
+ ---
135
+
136
+ ## 8. Citation
137
+
138
+ ```bibtex
139
+ @dataset{alican_kiraz_2025_heimdall_v2_0,
140
+ author = {Alican Kiraz},
141
+ title = {Fenrir v2.0 — Cybersecurity Defense Instruction-Tuning Dataset},
142
+ year = {2025},
143
+ publisher = {Hugging Face},
144
+ url = {https://huggingface.co/datasets/AlicanKiraz0/Cybersecurity-Dataset-Heimdall-v2.0}
145
+ }
146
+ ```
147
+
148
+ ---
149
+
150
+ ## 9. Changelog
151
+
152
+ * **v2.0.0** (2025‑10‑06) — Expanded to **83 920** rows; deeper Cloud/DevSecOps/Identity coverage; stronger adversarial refusal tests; stricter schema checks.
153
+ * **v1.1.0** (2025‑06‑21) — 21 258 rows; broadened framework coverage; improved automatic quality gates.
154
+ * **v1.0.0** (2025‑06‑17) — Initial 2 500 rows.
155
+
156
+ ---
data_preprocessor.py ADDED
@@ -0,0 +1,63 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # data_preprocessor.py
2
+ import hashlib
3
+ import re
4
+ from collections import Counter
5
+
6
+ class DataPreprocessor:
7
+ def __init__(self):
8
+ self.seen_hashes = set()
9
+
10
+ def deduplicate(self, texts):
11
+ """Remove exact duplicate lines."""
12
+ unique_texts = []
13
+ for text in texts:
14
+ text_hash = hashlib.md5(text.encode()).hexdigest()
15
+ if text_hash not in self.seen_hashes:
16
+ self.seen_hashes.add(text_hash)
17
+ unique_texts.append(text)
18
+ print(f"Removed {len(texts) - len(unique_texts)} duplicates.")
19
+ return unique_texts
20
+
21
+ def filter_quality(self, texts, min_words=5, max_word_ratio=0.9):
22
+ """Filter short texts and those with too many repeated words."""
23
+ filtered = []
24
+ for text in texts:
25
+ words = text.split()
26
+ if len(words) < min_words:
27
+ continue
28
+ # Check if > 90% of words are the same (spam)
29
+ if len(words) > 0 and max(Counter(words).values()) / len(words) > max_word_ratio:
30
+ continue
31
+ filtered.append(text)
32
+ print(f"Removed {len(texts) - len(filtered)} low-quality texts.")
33
+ return filtered
34
+
35
+ def remove_toxic(self, texts, toxic_keywords=None):
36
+ """Basic toxicity filter."""
37
+ if toxic_keywords is None:
38
+ toxic_keywords = ["hate", "violence", "kill", "racist", "sexist", "nazi"]
39
+ filtered = []
40
+ for text in texts:
41
+ lower_text = text.lower()
42
+ if any(kw in lower_text for kw in toxic_keywords):
43
+ continue
44
+ filtered.append(text)
45
+ print(f"Removed {len(texts) - len(filtered)} potentially toxic texts.")
46
+ return filtered
47
+
48
+ def preprocess_file(self, input_path, output_path):
49
+ with open(input_path, 'r', encoding='utf-8') as f:
50
+ texts = [line.strip() for line in f if line.strip()]
51
+
52
+ texts = self.deduplicate(texts)
53
+ texts = self.filter_quality(texts)
54
+ texts = self.remove_toxic(texts)
55
+
56
+ with open(output_path, 'w', encoding='utf-8') as f:
57
+ for text in texts:
58
+ f.write(text + '\n')
59
+ print(f"Saved cleaned data to {output_path}")
60
+
61
+ # Usage:
62
+ # preprocessor = DataPreprocessor()
63
+ # preprocessor.preprocess_file("data/raw_train.txt", "data/train.txt")
dataset.py ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # dataset.py
2
+ import torch
3
+ from torch.utils.data import Dataset, IterableDataset
4
+ import os
5
+ import random
6
+ from datasets import load_dataset as hf_load_dataset
7
+
8
+
9
+ class TextDataset(Dataset):
10
+ """
11
+ Simple dataset for a single text file (used for validation).
12
+ """
13
+ def __init__(self, file_path, tokenizer, max_len):
14
+ self.tokenizer = tokenizer
15
+ self.max_len = max_len
16
+ with open(file_path, 'r', encoding='utf-8') as f:
17
+ self.texts = [line.strip() for line in f if line.strip()]
18
+
19
+ def __len__(self):
20
+ return len(self.texts)
21
+
22
+ def __getitem__(self, idx):
23
+ text = self.texts[idx]
24
+ if hasattr(self.tokenizer, 'encode'):
25
+ ids = self.tokenizer.encode(text)
26
+ else:
27
+ ids = self.tokenizer.encode(text, add_special_tokens=True)
28
+ if len(ids) > self.max_len:
29
+ ids = ids[:self.max_len]
30
+ else:
31
+ if hasattr(self.tokenizer, 'pad_id'):
32
+ pad_id = self.tokenizer.pad_id
33
+ elif hasattr(self.tokenizer, 'pad_token_id'):
34
+ pad_id = self.tokenizer.pad_token_id
35
+ else:
36
+ pad_id = 0
37
+ ids = ids + [pad_id] * (self.max_len - len(ids))
38
+ return torch.tensor(ids, dtype=torch.long)
39
+
40
+
41
+ class MultiSourceTextDataset(Dataset):
42
+ """
43
+ Flexible dataset that can load from:
44
+ - Local text files
45
+ - Hugging Face datasets
46
+ """
47
+ def __init__(self, tokenizer, max_len=512, local_files=None, hf_datasets=None, shuffle=True):
48
+ self.tokenizer = tokenizer
49
+ self.max_len = max_len
50
+ self.shuffle = shuffle
51
+ self.data = []
52
+
53
+ if local_files:
54
+ for file_path in local_files:
55
+ if os.path.exists(file_path):
56
+ with open(file_path, 'r', encoding='utf-8') as f:
57
+ lines = [line.strip() for line in f if line.strip()]
58
+ self.data.extend(lines)
59
+ print(f" Loaded {len(lines)} from {file_path}")
60
+
61
+ if hf_datasets:
62
+ for ds_name in hf_datasets:
63
+ try:
64
+ ds = hf_load_dataset(ds_name, split="train", trust_remote_code=True)
65
+ if "content" in ds.column_names:
66
+ texts = [item["content"] for item in ds]
67
+ elif "text" in ds.column_names:
68
+ texts = [item["text"] for item in ds]
69
+ else:
70
+ col = ds.column_names[0]
71
+ texts = [item[col] for item in ds]
72
+ self.data.extend(texts)
73
+ print(f" Loaded {len(texts)} from {ds_name}")
74
+ except Exception as e:
75
+ print(f" ⚠️ Error loading {ds_name}: {e}")
76
+
77
+ if not self.data:
78
+ print("⚠️ No data loaded!")
79
+
80
+ def __len__(self):
81
+ return len(self.data)
82
+
83
+ def __getitem__(self, idx):
84
+ text = self.data[idx]
85
+ if hasattr(self.tokenizer, 'encode'):
86
+ ids = self.tokenizer.encode(text)
87
+ else:
88
+ ids = self.tokenizer.encode(text, add_special_tokens=True)
89
+ if len(ids) > self.max_len:
90
+ ids = ids[:self.max_len]
91
+ else:
92
+ if hasattr(self.tokenizer, 'pad_id'):
93
+ pad_id = self.tokenizer.pad_id
94
+ elif hasattr(self.tokenizer, 'pad_token_id'):
95
+ pad_id = self.tokenizer.pad_token_id
96
+ else:
97
+ pad_id = 0
98
+ ids = ids + [pad_id] * (self.max_len - len(ids))
99
+ return torch.tensor(ids, dtype=torch.long)
100
+
101
+
102
+ class StreamingMultiSourceDataset(IterableDataset):
103
+ """
104
+ Memory-efficient streaming dataset for huge datasets.
105
+ """
106
+ def __init__(self, tokenizer, max_len=512, file_paths=None, shuffle=True, buffer_size=10000):
107
+ self.tokenizer = tokenizer
108
+ self.max_len = max_len
109
+ self.file_paths = file_paths or []
110
+ self.shuffle = shuffle
111
+ self.buffer_size = buffer_size
112
+
113
+ def __iter__(self):
114
+ for file_path in self.file_paths:
115
+ if not os.path.exists(file_path):
116
+ continue
117
+ with open(file_path, 'r', encoding='utf-8') as f:
118
+ buffer = []
119
+ for line in f:
120
+ line = line.strip()
121
+ if line:
122
+ if self.shuffle:
123
+ buffer.append(line)
124
+ if len(buffer) >= self.buffer_size:
125
+ random.shuffle(buffer)
126
+ for item in buffer:
127
+ yield self._tokenize(item)
128
+ buffer = []
129
+ else:
130
+ yield self._tokenize(line)
131
+ if buffer:
132
+ random.shuffle(buffer)
133
+ for item in buffer:
134
+ yield self._tokenize(item)
135
+
136
+ def _tokenize(self, text):
137
+ if hasattr(self.tokenizer, 'encode'):
138
+ ids = self.tokenizer.encode(text)
139
+ else:
140
+ ids = self.tokenizer.encode(text, add_special_tokens=True)
141
+ if len(ids) > self.max_len:
142
+ ids = ids[:self.max_len]
143
+ else:
144
+ if hasattr(self.tokenizer, 'pad_id'):
145
+ pad_id = self.tokenizer.pad_id
146
+ elif hasattr(self.tokenizer, 'pad_token_id'):
147
+ pad_id = self.tokenizer.pad_token_id
148
+ else:
149
+ pad_id = 0
150
+ ids = ids + [pad_id] * (self.max_len - len(ids))
151
+ return torch.tensor(ids, dtype=torch.long)
152
+
153
+
154
+ def create_mixed_dataset(tokenizer, max_len=512, config=None):
155
+ """
156
+ Factory function that creates a mixed dataset.
157
+ Uses streaming if config.use_streaming is True.
158
+ """
159
+ local_files = []
160
+ for f in ["data/code_train.txt", "data/cyber_train.txt", "data/general_train.txt", "data/train.txt"]:
161
+ if os.path.exists(f):
162
+ local_files.append(f)
163
+
164
+ use_streaming = config and hasattr(config, 'use_streaming') and config.use_streaming
165
+
166
+ if use_streaming:
167
+ return StreamingMultiSourceDataset(
168
+ tokenizer=tokenizer,
169
+ max_len=max_len,
170
+ file_paths=local_files,
171
+ shuffle=True,
172
+ buffer_size=10000
173
+ )
174
+ else:
175
+ return MultiSourceTextDataset(
176
+ tokenizer=tokenizer,
177
+ max_len=max_len,
178
+ local_files=local_files,
179
+ hf_datasets=[],
180
+ shuffle=True
181
+ )
dpo_trainer.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # dpo_trainer.py
2
+ import torch
3
+ import torch.nn as nn
4
+ import torch.nn.functional as F
5
+ from torch.utils.data import Dataset, DataLoader
6
+ from transformers import AutoTokenizer
7
+ import random
8
+
9
+ class PreferenceDataset(Dataset):
10
+ def __init__(self, data_path, tokenizer, max_len=512):
11
+ self.tokenizer = tokenizer
12
+ self.max_len = max_len
13
+ self.data = []
14
+ with open(data_path, 'r', encoding='utf-8') as f:
15
+ for line in f:
16
+ parts = line.strip().split('\t')
17
+ if len(parts) == 3:
18
+ prompt, chosen, rejected = parts
19
+ self.data.append((prompt, chosen, rejected))
20
+
21
+ def __len__(self):
22
+ return len(self.data)
23
+
24
+ def __getitem__(self, idx):
25
+ prompt, chosen, rejected = self.data[idx]
26
+ # Encode
27
+ prompt_ids = self.tokenizer.encode(prompt)
28
+ chosen_ids = prompt_ids + self.tokenizer.encode(chosen)[1:] # remove BOS
29
+ rejected_ids = prompt_ids + self.tokenizer.encode(rejected)[1:]
30
+ # Pad
31
+ if len(chosen_ids) > self.max_len:
32
+ chosen_ids = chosen_ids[:self.max_len]
33
+ else:
34
+ chosen_ids += [self.tokenizer.pad_id] * (self.max_len - len(chosen_ids))
35
+ if len(rejected_ids) > self.max_len:
36
+ rejected_ids = rejected_ids[:self.max_len]
37
+ else:
38
+ rejected_ids += [self.tokenizer.pad_id] * (self.max_len - len(rejected_ids))
39
+ return torch.tensor(chosen_ids, dtype=torch.long), torch.tensor(rejected_ids, dtype=torch.long)
40
+
41
+ class DPOTrainer:
42
+ def __init__(self, model, ref_model, train_loader, config, device, beta=0.1, lr=1e-5):
43
+ self.model = model
44
+ self.ref_model = ref_model
45
+ self.train_loader = train_loader
46
+ self.config = config
47
+ self.device = device
48
+ self.beta = beta
49
+ self.optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
50
+
51
+ def compute_dpo_loss(self, chosen_logps, rejected_logps):
52
+ # DPO loss: -log(sigmoid(beta * (chosen - rejected)))
53
+ diff = chosen_logps - rejected_logps
54
+ loss = -F.logsigmoid(self.beta * diff).mean()
55
+ return loss
56
+
57
+ def train_step(self, batch_chosen, batch_rejected):
58
+ batch_chosen = batch_chosen.to(self.device)
59
+ batch_rejected = batch_rejected.to(self.device)
60
+
61
+ # Forward pass on chosen
62
+ logits_chosen = self.model(batch_chosen)
63
+ logits_ref_chosen = self.ref_model(batch_chosen)
64
+
65
+ # Forward pass on rejected
66
+ logits_rejected = self.model(batch_rejected)
67
+ logits_ref_rejected = self.ref_model(batch_rejected)
68
+
69
+ # Compute log probabilities (ignoring padding)
70
+ logps_chosen = F.log_softmax(logits_chosen, dim=-1).gather(-1, batch_chosen.unsqueeze(-1)).squeeze(-1).sum(-1)
71
+ logps_ref_chosen = F.log_softmax(logits_ref_chosen, dim=-1).gather(-1, batch_chosen.unsqueeze(-1)).squeeze(-1).sum(-1)
72
+
73
+ logps_rejected = F.log_softmax(logits_rejected, dim=-1).gather(-1, batch_rejected.unsqueeze(-1)).squeeze(-1).sum(-1)
74
+ logps_ref_rejected = F.log_softmax(logits_ref_rejected, dim=-1).gather(-1, batch_rejected.unsqueeze(-1)).squeeze(-1).sum(-1)
75
+
76
+ # DPO loss
77
+ loss = self.compute_dpo_loss(logps_chosen - logps_ref_chosen, logps_rejected - logps_ref_rejected)
78
+
79
+ self.optimizer.zero_grad()
80
+ loss.backward()
81
+ torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
82
+ self.optimizer.step()
83
+ return loss.item()
84
+
85
+ def train(self, epochs=1):
86
+ self.model.train()
87
+ for epoch in range(epochs):
88
+ total_loss = 0
89
+ for batch_chosen, batch_rejected in self.train_loader:
90
+ loss = self.train_step(batch_chosen, batch_rejected)
91
+ total_loss += loss
92
+ print(f"DPO Epoch {epoch+1}, Loss: {total_loss/len(self.train_loader):.4f}")
93
+
94
+ # Usage:
95
+ # dpo_dataset = PreferenceDataset("data/preference.txt", tokenizer)
96
+ # dpo_loader = DataLoader(dpo_dataset, batch_size=4, shuffle=True)
97
+ # ref_model = KbiduForCausalLM(config) # copy of base model (frozen)
98
+ # dpo_trainer = DPOTrainer(model, ref_model, dpo_loader, config, device)
99
+ # dpo_trainer.train(epochs=3)
embedding.py ADDED
@@ -0,0 +1,11 @@
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # embedding.py
2
+ import torch.nn as nn
3
+
4
+ class Embedding(nn.Module):
5
+ def __init__(self, vocab_size, d_model):
6
+ super().__init__()
7
+ self.d_model = d_model
8
+ self.embed = nn.Embedding(vocab_size, d_model)
9
+
10
+ def forward(self, x):
11
+ return self.embed(x) * (self.d_model ** 0.5)
eval_math.py ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # eval_math.py
2
+ import re
3
+ from datasets import load_dataset
4
+ from inference import Inference
5
+ from tqdm import tqdm
6
+
7
+ def extract_answer(text):
8
+ """Extract final numeric answer from text."""
9
+ # Try to find the last number in the text
10
+ numbers = re.findall(r'[-+]?\d*\.?\d+', text)
11
+ if numbers:
12
+ return numbers[-1]
13
+ return None
14
+
15
+ def evaluate_gsm8k(model, tokenizer, device, num_samples=100):
16
+ """Evaluate on GSM8K dataset."""
17
+ dataset = load_dataset("gsm8k", "main", split="test")
18
+ total = 0
19
+ correct = 0
20
+ inference_engine = Inference(model, tokenizer, device, max_new_tokens=200)
21
+
22
+ for i, example in enumerate(tqdm(dataset.select(range(num_samples)))):
23
+ question = example["question"]
24
+ true_answer = example["answer"].split("####")[-1].strip()
25
+
26
+ # Generate with CoT
27
+ output = inference_engine.generate(question, use_cot=True)
28
+ pred_answer = extract_answer(output)
29
+
30
+ if pred_answer and pred_answer in true_answer:
31
+ correct += 1
32
+ total += 1
33
+
34
+ accuracy = correct / total
35
+ print(f"GSM8K Accuracy ({num_samples} samples): {accuracy:.2%}")
36
+ return accuracy
37
+
38
+ def evaluate_math(model, tokenizer, device, num_samples=100):
39
+ """Evaluate on MATH dataset (college math)."""
40
+ # Similar logic for MATH dataset
41
+ dataset = load_dataset("math_dataset", split="test")
42
+ # ... implementation similar to above
43
+ pass
44
+
45
+ # Usage:
46
+ # from model import KbiduForCausalLM
47
+ # model = KbiduForCausalLM(config).to(device)
48
+ # evaluate_gsm8k(model, tokenizer, device, num_samples=50)
faiss_index.faiss ADDED
Binary file (1.58 kB). View file
 
faiss_index_chunks.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ This is sample training text.
feedforward.py ADDED
@@ -0,0 +1,14 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # feedforward.py
2
+ import torch.nn as nn
3
+ import torch.nn.functional as F
4
+
5
+ class SwiGLU(nn.Module):
6
+ def __init__(self, d_model, d_ff, dropout=0.1):
7
+ super().__init__()
8
+ self.w1 = nn.Linear(d_model, d_ff, bias=False)
9
+ self.w2 = nn.Linear(d_ff, d_model, bias=False)
10
+ self.w3 = nn.Linear(d_model, d_ff, bias=False)
11
+ self.dropout = nn.Dropout(dropout)
12
+
13
+ def forward(self, x):
14
+ return self.w2(self.dropout(F.silu(self.w1(x)) * self.w3(x)))
inference.py ADDED
@@ -0,0 +1,107 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # inference.py
2
+ import torch
3
+ import torch.nn.functional as F
4
+ from collections import Counter
5
+ import re
6
+
7
+ class Inference:
8
+ def __init__(self, model, tokenizer, device, retriever=None, max_new_tokens=300, memory_size=5):
9
+ self.model = model
10
+ self.tokenizer = tokenizer
11
+ self.device = device
12
+ self.retriever = retriever
13
+ self.max_new_tokens = max_new_tokens
14
+ self.memory = []
15
+ self.memory_size = memory_size
16
+
17
+ def _build_prompt(self, query, use_cot=True, tone="empathetic", think_before_answer=False):
18
+ tone_map = {
19
+ "empathetic": "You are Kbidu AI, a warm, kind, and emotionally intelligent assistant.",
20
+ "professional": "You are Kbidu AI, a precise, clear, and objective assistant.",
21
+ "humorous": "You are Kbidu AI, a witty and fun assistant.",
22
+ "supportive": "You are Kbidu AI, a supportive and encouraging mentor."
23
+ }
24
+ system_prompt = tone_map.get(tone, tone_map["empathetic"])
25
+
26
+ # RAG context
27
+ context = ""
28
+ if self.retriever:
29
+ results = self.retriever.retrieve(query, top_k=3)
30
+ if results:
31
+ context = "Relevant information:\n" + "\n".join([r["chunk"] for r in results]) + "\n\n"
32
+
33
+ # Conversation memory
34
+ memory_text = ""
35
+ if self.memory:
36
+ memory_text = "Conversation history:\n"
37
+ for user, assistant in self.memory[-self.memory_size:]:
38
+ memory_text += f"User: {user}\nAssistant: {assistant}\n"
39
+ memory_text += "\n"
40
+
41
+ # Think-before-answer instruction
42
+ think_instruction = ""
43
+ if think_before_answer:
44
+ think_instruction = (
45
+ "Before giving your final answer, think step by step inside your head.\n"
46
+ "Then, provide your final answer after the line 'Final Answer:'.\n"
47
+ "Do not include your reasoning in the final output unless asked.\n"
48
+ )
49
+
50
+ cot_instruction = "Let's think step by step.\n" if use_cot else ""
51
+
52
+ prompt = f"{system_prompt}\n\n{context}{memory_text}{think_instruction}{cot_instruction}User: {query}\nAssistant:"
53
+ return prompt
54
+
55
+ def generate(self, query, use_cot=True, temperature=0.7, tone="empathetic", think_before_answer=False, show_reasoning=False):
56
+ prompt = self._build_prompt(query, use_cot, tone, think_before_answer)
57
+ input_ids = torch.tensor([self.tokenizer.encode(prompt)]).to(self.device)
58
+
59
+ for _ in range(self.max_new_tokens):
60
+ with torch.no_grad():
61
+ logits = self.model(input_ids)
62
+ next_logits = logits[0, -1, :] / temperature
63
+ probs = F.softmax(next_logits, dim=-1)
64
+ next_token = torch.multinomial(probs, num_samples=1).unsqueeze(0)
65
+ input_ids = torch.cat([input_ids, next_token], dim=-1)
66
+ if next_token.item() == self.tokenizer.eos_id:
67
+ break
68
+
69
+ full_response = self.tokenizer.decode(input_ids[0].tolist())
70
+
71
+ # Extract assistant part
72
+ if "Assistant:" in full_response:
73
+ response = full_response.split("Assistant:", 1)[-1].strip()
74
+ else:
75
+ response = full_response[len(prompt):].strip()
76
+
77
+ # If think_before_answer is True, try to extract final answer after "Final Answer:"
78
+ if think_before_answer:
79
+ if "Final Answer:" in response:
80
+ final_answer = response.split("Final Answer:", 1)[-1].strip()
81
+ if show_reasoning:
82
+ # Return the full response (reasoning + final answer)
83
+ self.memory.append((query, response))
84
+ return response
85
+ else:
86
+ # Return only the final answer
87
+ self.memory.append((query, final_answer))
88
+ return final_answer
89
+ else:
90
+ # If no "Final Answer:" found, return the full response (fallback)
91
+ self.memory.append((query, response))
92
+ return response
93
+ else:
94
+ self.memory.append((query, response))
95
+ return response
96
+
97
+ def generate_with_self_consistency(self, query, n=5, temperature=0.7, tone="empathetic", think_before_answer=False):
98
+ answers = []
99
+ for _ in range(n):
100
+ ans = self.generate(query, use_cot=True, temperature=temperature, tone=tone, think_before_answer=think_before_answer, show_reasoning=False)
101
+ if "Answer:" in ans:
102
+ ans = ans.split("Answer:")[-1].strip()
103
+ answers.append(ans)
104
+ if answers:
105
+ counter = Counter(answers)
106
+ return counter.most_common(1)[0][0]
107
+ return answers[0] if answers else ""
kbidu_env/.gitignore ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ # Created by venv; see https://docs.python.org/3/library/venv.html
2
+ *
kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab64835b63093c31975df2692756c8008e7ec190e4aa86f15a713bcc62e1d432
3
+ size 216576
kbidu_env/Lib/site-packages/__pycache__/isympy.cpython-314.pyc ADDED
Binary file (11.3 kB). View file
 
kbidu_env/Lib/site-packages/__pycache__/six.cpython-314.pyc ADDED
Binary file (43 kB). View file
 
kbidu_env/Lib/site-packages/__pycache__/threadpoolctl.cpython-314.pyc ADDED
Binary file (59.5 kB). View file
 
kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:42c2b14d3bd1c9fc087ac29ced4cef30dc3b08c3a322502df232532b1cb6aa1f
3
+ size 175506
kbidu_env/Lib/site-packages/_distutils_hack/__init__.py ADDED
@@ -0,0 +1,220 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # don't import any costly modules
2
+ import sys
3
+ import os
4
+
5
+
6
+ def warn_distutils_present():
7
+ if 'distutils' not in sys.modules:
8
+ return
9
+ import warnings
10
+
11
+ warnings.warn(
12
+ "Distutils was imported before Setuptools, but importing Setuptools "
13
+ "also replaces the `distutils` module in `sys.modules`. This may lead "
14
+ "to undesirable behaviors or errors. To avoid these issues, avoid "
15
+ "using distutils directly, ensure that setuptools is installed in the "
16
+ "traditional way (e.g. not an editable install), and/or make sure "
17
+ "that setuptools is always imported before distutils."
18
+ )
19
+
20
+
21
+ def clear_distutils():
22
+ if 'distutils' not in sys.modules:
23
+ return
24
+ import warnings
25
+
26
+ warnings.warn("Setuptools is replacing distutils.")
27
+ mods = [
28
+ name
29
+ for name in sys.modules
30
+ if name == "distutils" or name.startswith("distutils.")
31
+ ]
32
+ for name in mods:
33
+ del sys.modules[name]
34
+
35
+
36
+ def enabled():
37
+ """
38
+ Allow selection of distutils by environment variable.
39
+ """
40
+ which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local')
41
+ return which == 'local'
42
+
43
+
44
+ def ensure_local_distutils():
45
+ import importlib
46
+
47
+ clear_distutils()
48
+
49
+ # With the DistutilsMetaFinder in place,
50
+ # perform an import to cause distutils to be
51
+ # loaded from setuptools._distutils. Ref #2906.
52
+ with shim():
53
+ importlib.import_module('distutils')
54
+
55
+ # check that submodules load as expected
56
+ core = importlib.import_module('distutils.core')
57
+ assert '_distutils' in core.__file__, core.__file__
58
+ assert 'setuptools._distutils.log' not in sys.modules
59
+
60
+
61
+ def do_override():
62
+ """
63
+ Ensure that the local copy of distutils is preferred over stdlib.
64
+
65
+ See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
66
+ for more motivation.
67
+ """
68
+ if enabled():
69
+ warn_distutils_present()
70
+ ensure_local_distutils()
71
+
72
+
73
+ class _TrivialRe:
74
+ def __init__(self, *patterns):
75
+ self._patterns = patterns
76
+
77
+ def match(self, string):
78
+ return all(pat in string for pat in self._patterns)
79
+
80
+
81
+ class DistutilsMetaFinder:
82
+ def find_spec(self, fullname, path, target=None):
83
+ # optimization: only consider top level modules and those
84
+ # found in the CPython test suite.
85
+ if path is not None and not fullname.startswith('test.'):
86
+ return None
87
+
88
+ method_name = 'spec_for_{fullname}'.format(**locals())
89
+ method = getattr(self, method_name, lambda: None)
90
+ return method()
91
+
92
+ def spec_for_distutils(self):
93
+ if self.is_cpython():
94
+ return None
95
+
96
+ import importlib
97
+ import importlib.abc
98
+ import importlib.util
99
+
100
+ try:
101
+ mod = importlib.import_module('setuptools._distutils')
102
+ except Exception:
103
+ # There are a couple of cases where setuptools._distutils
104
+ # may not be present:
105
+ # - An older Setuptools without a local distutils is
106
+ # taking precedence. Ref #2957.
107
+ # - Path manipulation during sitecustomize removes
108
+ # setuptools from the path but only after the hook
109
+ # has been loaded. Ref #2980.
110
+ # In either case, fall back to stdlib behavior.
111
+ return None
112
+
113
+ class DistutilsLoader(importlib.abc.Loader):
114
+ def create_module(self, spec):
115
+ mod.__name__ = 'distutils'
116
+ return mod
117
+
118
+ def exec_module(self, module):
119
+ pass
120
+
121
+ return importlib.util.spec_from_loader(
122
+ 'distutils', DistutilsLoader(), origin=mod.__file__
123
+ )
124
+
125
+ @staticmethod
126
+ def is_cpython():
127
+ """
128
+ Suppress supplying distutils for CPython (build and tests).
129
+ Ref #2965 and #3007.
130
+ """
131
+ return os.path.isfile('pybuilddir.txt')
132
+
133
+ def spec_for_pip(self):
134
+ """
135
+ Ensure stdlib distutils when running under pip.
136
+ See pypa/pip#8761 for rationale.
137
+ """
138
+ if sys.version_info >= (3, 12) or self.pip_imported_during_build():
139
+ return
140
+ clear_distutils()
141
+ self.spec_for_distutils = lambda: None
142
+
143
+ @classmethod
144
+ def pip_imported_during_build(cls):
145
+ """
146
+ Detect if pip is being imported in a build script. Ref #2355.
147
+ """
148
+ import traceback
149
+
150
+ return any(
151
+ cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None)
152
+ )
153
+
154
+ @staticmethod
155
+ def frame_file_is_setup(frame):
156
+ """
157
+ Return True if the indicated frame suggests a setup.py file.
158
+ """
159
+ # some frames may not have __file__ (#2940)
160
+ return frame.f_globals.get('__file__', '').endswith('setup.py')
161
+
162
+ def spec_for_sensitive_tests(self):
163
+ """
164
+ Ensure stdlib distutils when running select tests under CPython.
165
+
166
+ python/cpython#91169
167
+ """
168
+ clear_distutils()
169
+ self.spec_for_distutils = lambda: None
170
+
171
+ sensitive_tests = (
172
+ [
173
+ 'test.test_distutils',
174
+ 'test.test_peg_generator',
175
+ 'test.test_importlib',
176
+ ]
177
+ if sys.version_info < (3, 10)
178
+ else [
179
+ 'test.test_distutils',
180
+ ]
181
+ )
182
+
183
+
184
+ for name in DistutilsMetaFinder.sensitive_tests:
185
+ setattr(
186
+ DistutilsMetaFinder,
187
+ f'spec_for_{name}',
188
+ DistutilsMetaFinder.spec_for_sensitive_tests,
189
+ )
190
+
191
+
192
+ DISTUTILS_FINDER = DistutilsMetaFinder()
193
+
194
+
195
+ def add_shim():
196
+ DISTUTILS_FINDER in sys.meta_path or insert_shim()
197
+
198
+
199
+ class shim:
200
+ def __enter__(self):
201
+ insert_shim()
202
+
203
+ def __exit__(self, exc, value, tb):
204
+ _remove_shim()
205
+
206
+
207
+ def insert_shim():
208
+ sys.meta_path.insert(0, DISTUTILS_FINDER)
209
+
210
+
211
+ def _remove_shim():
212
+ try:
213
+ sys.meta_path.remove(DISTUTILS_FINDER)
214
+ except ValueError:
215
+ pass
216
+
217
+
218
+ if sys.version_info < (3, 12):
219
+ # DistutilsMetaFinder can only be disabled in Python < 3.12 (PEP 632)
220
+ remove_shim = _remove_shim
kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-314.pyc ADDED
Binary file (10.5 kB). View file