Upload folder using huggingface_hub
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +7 -0
- Dockerfile +18 -0
- __pycache__/attention.cpython-314.pyc +0 -0
- __pycache__/config.cpython-314.pyc +0 -0
- __pycache__/dataset.cpython-314.pyc +0 -0
- __pycache__/embedding.cpython-314.pyc +0 -0
- __pycache__/feedforward.cpython-314.pyc +0 -0
- __pycache__/inference.cpython-314.pyc +0 -0
- __pycache__/layernorm.cpython-314.pyc +0 -0
- __pycache__/loss.cpython-314.pyc +0 -0
- __pycache__/model.cpython-314.pyc +0 -0
- __pycache__/positional_encoding.cpython-314.pyc +0 -0
- __pycache__/retriever.cpython-314.pyc +0 -0
- __pycache__/tokenizer.cpython-314.pyc +0 -0
- __pycache__/trainer.cpython-314.pyc +0 -0
- __pycache__/transformer_block.cpython-314.pyc +0 -0
- __pycache__/utils.cpython-314.pyc +0 -0
- attention.py +48 -0
- chat_template.py +36 -0
- config.py +56 -0
- data/code_train.txt +276 -0
- data/code_valid.txt +73 -0
- data/cyber_train.txt +83 -0
- data/cyber_valid.txt +19 -0
- data/general_train.txt +65 -0
- data/general_valid.txt +35 -0
- data/tokenizer/tokenizer.json +169 -0
- data/train.txt +56 -0
- data/valid.txt +29 -0
- data_folder/.gitattributes +60 -0
- data_folder/CyberSec-Dataset_escaped.jsonl +3 -0
- data_folder/Fenrir.png +3 -0
- data_folder/README.md +156 -0
- data_preprocessor.py +63 -0
- dataset.py +181 -0
- dpo_trainer.py +99 -0
- embedding.py +11 -0
- eval_math.py +48 -0
- faiss_index.faiss +0 -0
- faiss_index_chunks.txt +1 -0
- feedforward.py +14 -0
- inference.py +107 -0
- kbidu_env/.gitignore +2 -0
- kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd +3 -0
- kbidu_env/Lib/site-packages/__pycache__/isympy.cpython-314.pyc +0 -0
- kbidu_env/Lib/site-packages/__pycache__/six.cpython-314.pyc +0 -0
- kbidu_env/Lib/site-packages/__pycache__/threadpoolctl.cpython-314.pyc +0 -0
- kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc +3 -0
- kbidu_env/Lib/site-packages/_distutils_hack/__init__.py +220 -0
- kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-314.pyc +0 -0
.gitattributes
CHANGED
|
@@ -33,3 +33,10 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
data_folder/CyberSec-Dataset_escaped.jsonl filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
data_folder/Fenrir.png filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
kbidu_env/Lib/site-packages/accelerate/__pycache__/accelerator.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/dataclasses.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
kbidu_env/Lib/site-packages/accelerate/utils/__pycache__/modeling.cpython-314.pyc filter=lfs diff=lfs merge=lfs -text
|
Dockerfile
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
FROM nvidia/cuda:12.4.0-runtime-ubuntu22.04
|
| 2 |
+
|
| 3 |
+
ENV DEBIAN_FRONTEND=noninteractive
|
| 4 |
+
ENV PYTHONUNBUFFERED=1
|
| 5 |
+
|
| 6 |
+
RUN apt-get update && apt-get install -y \
|
| 7 |
+
python3.10 python3-pip git wget \
|
| 8 |
+
&& ln -s /usr/bin/python3.10 /usr/bin/python \
|
| 9 |
+
&& rm -rf /var/lib/apt/lists/*
|
| 10 |
+
|
| 11 |
+
WORKDIR /workspace
|
| 12 |
+
COPY requirements.txt .
|
| 13 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 14 |
+
|
| 15 |
+
COPY . ./KbiduAI/
|
| 16 |
+
WORKDIR /workspace/KbiduAI
|
| 17 |
+
|
| 18 |
+
ENTRYPOINT ["torchrun", "--nproc_per_node=1", "main.py"]
|
__pycache__/attention.cpython-314.pyc
ADDED
|
Binary file (3.55 kB). View file
|
|
|
__pycache__/config.cpython-314.pyc
ADDED
|
Binary file (1.29 kB). View file
|
|
|
__pycache__/dataset.cpython-314.pyc
ADDED
|
Binary file (10.1 kB). View file
|
|
|
__pycache__/embedding.cpython-314.pyc
ADDED
|
Binary file (1.07 kB). View file
|
|
|
__pycache__/feedforward.cpython-314.pyc
ADDED
|
Binary file (1.53 kB). View file
|
|
|
__pycache__/inference.cpython-314.pyc
ADDED
|
Binary file (6.02 kB). View file
|
|
|
__pycache__/layernorm.cpython-314.pyc
ADDED
|
Binary file (1.36 kB). View file
|
|
|
__pycache__/loss.cpython-314.pyc
ADDED
|
Binary file (1.36 kB). View file
|
|
|
__pycache__/model.cpython-314.pyc
ADDED
|
Binary file (3.6 kB). View file
|
|
|
__pycache__/positional_encoding.cpython-314.pyc
ADDED
|
Binary file (2.21 kB). View file
|
|
|
__pycache__/retriever.cpython-314.pyc
ADDED
|
Binary file (5.71 kB). View file
|
|
|
__pycache__/tokenizer.cpython-314.pyc
ADDED
|
Binary file (1.82 kB). View file
|
|
|
__pycache__/trainer.cpython-314.pyc
ADDED
|
Binary file (6.11 kB). View file
|
|
|
__pycache__/transformer_block.cpython-314.pyc
ADDED
|
Binary file (1.83 kB). View file
|
|
|
__pycache__/utils.cpython-314.pyc
ADDED
|
Binary file (1.42 kB). View file
|
|
|
attention.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# attention.py
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
|
| 6 |
+
class MultiHeadAttention(nn.Module):
|
| 7 |
+
def __init__(self, d_model, num_heads, num_kv_heads=None, dropout=0.1):
|
| 8 |
+
super().__init__()
|
| 9 |
+
assert d_model % num_heads == 0
|
| 10 |
+
self.d_model = d_model
|
| 11 |
+
self.num_heads = num_heads
|
| 12 |
+
self.num_kv_heads = num_kv_heads or num_heads
|
| 13 |
+
self.head_dim = d_model // num_heads
|
| 14 |
+
|
| 15 |
+
self.W_q = nn.Linear(d_model, num_heads * self.head_dim, bias=False)
|
| 16 |
+
self.W_k = nn.Linear(d_model, self.num_kv_heads * self.head_dim, bias=False)
|
| 17 |
+
self.W_v = nn.Linear(d_model, self.num_kv_heads * self.head_dim, bias=False)
|
| 18 |
+
self.W_o = nn.Linear(num_heads * self.head_dim, d_model, bias=False)
|
| 19 |
+
self.dropout = nn.Dropout(dropout)
|
| 20 |
+
|
| 21 |
+
def forward(self, x, mask=None, rotary=None):
|
| 22 |
+
batch, seq_len, _ = x.size()
|
| 23 |
+
|
| 24 |
+
q = self.W_q(x).view(batch, seq_len, self.num_heads, self.head_dim)
|
| 25 |
+
k = self.W_k(x).view(batch, seq_len, self.num_kv_heads, self.head_dim)
|
| 26 |
+
v = self.W_v(x).view(batch, seq_len, self.num_kv_heads, self.head_dim)
|
| 27 |
+
|
| 28 |
+
if rotary is not None:
|
| 29 |
+
q = rotary(q, seq_len)
|
| 30 |
+
k = rotary(k, seq_len)
|
| 31 |
+
|
| 32 |
+
if self.num_kv_heads != self.num_heads:
|
| 33 |
+
k = k.repeat_interleave(self.num_heads // self.num_kv_heads, dim=2)
|
| 34 |
+
v = v.repeat_interleave(self.num_heads // self.num_kv_heads, dim=2)
|
| 35 |
+
|
| 36 |
+
# Only use is_causal if mask is None, otherwise use explicit mask
|
| 37 |
+
use_causal = mask is None
|
| 38 |
+
|
| 39 |
+
attn_output = F.scaled_dot_product_attention(
|
| 40 |
+
q.transpose(1, 2), k.transpose(1, 2), v.transpose(1, 2),
|
| 41 |
+
attn_mask=mask,
|
| 42 |
+
dropout_p=self.dropout.p if self.training else 0.0,
|
| 43 |
+
is_causal=use_causal
|
| 44 |
+
)
|
| 45 |
+
attn_output = attn_output.transpose(1, 2).contiguous().view(batch, seq_len, -1)
|
| 46 |
+
return self.W_o(attn_output)
|
| 47 |
+
|
| 48 |
+
|
chat_template.py
ADDED
|
@@ -0,0 +1,36 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# chat_template.py
|
| 2 |
+
|
| 3 |
+
class ChatTemplate:
|
| 4 |
+
def __init__(self, system_prompt=None):
|
| 5 |
+
self.system_prompt = system_prompt or "You are Kbidu AI, a helpful, harmless, and honest assistant specialized in mathematics and cybersecurity."
|
| 6 |
+
self.history = []
|
| 7 |
+
|
| 8 |
+
def format_messages(self, messages):
|
| 9 |
+
"""
|
| 10 |
+
messages: list of dicts [{"role": "system/user/assistant", "content": "..."}]
|
| 11 |
+
Returns: formatted prompt string
|
| 12 |
+
"""
|
| 13 |
+
formatted = []
|
| 14 |
+
if self.system_prompt:
|
| 15 |
+
formatted.append(f"System: {self.system_prompt}")
|
| 16 |
+
|
| 17 |
+
for msg in messages:
|
| 18 |
+
role = msg["role"].capitalize()
|
| 19 |
+
content = msg["content"]
|
| 20 |
+
formatted.append(f"{role}: {content}")
|
| 21 |
+
|
| 22 |
+
# Add final assistant turn
|
| 23 |
+
formatted.append("Assistant:")
|
| 24 |
+
return "\n".join(formatted)
|
| 25 |
+
|
| 26 |
+
def build_prompt(self, query, history=None):
|
| 27 |
+
"""Build prompt from conversation history and new query."""
|
| 28 |
+
messages = []
|
| 29 |
+
if history:
|
| 30 |
+
messages.extend(history)
|
| 31 |
+
messages.append({"role": "user", "content": query})
|
| 32 |
+
return self.format_messages(messages)
|
| 33 |
+
|
| 34 |
+
# Usage:
|
| 35 |
+
# template = ChatTemplate()
|
| 36 |
+
# prompt = template.build_prompt("What is RSA encryption?", history=[...])
|
config.py
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# config.py
|
| 2 |
+
|
| 3 |
+
class KbiduTestConfig:
|
| 4 |
+
"""Quick test mode – trains in ~2 minutes on a single GPU."""
|
| 5 |
+
vocab_size = 1000
|
| 6 |
+
d_model = 128
|
| 7 |
+
num_layers = 4
|
| 8 |
+
num_heads = 4
|
| 9 |
+
num_kv_heads = 2
|
| 10 |
+
d_ff = 256
|
| 11 |
+
max_seq_len = 64
|
| 12 |
+
dropout = 0.1
|
| 13 |
+
layer_norm_eps = 1e-6
|
| 14 |
+
use_gradient_checkpointing = True
|
| 15 |
+
|
| 16 |
+
batch_size = 4
|
| 17 |
+
learning_rate = 3e-4
|
| 18 |
+
weight_decay = 0.1
|
| 19 |
+
warmup_steps = 10
|
| 20 |
+
total_steps = 50
|
| 21 |
+
gradient_accumulation_steps = 2
|
| 22 |
+
|
| 23 |
+
use_streaming = True
|
| 24 |
+
|
| 25 |
+
# --- Vision settings (for multimodal training) ---
|
| 26 |
+
vision_model = "openai/clip-vit-base-patch32"
|
| 27 |
+
freeze_vision = True
|
| 28 |
+
num_image_tokens = 8
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
class Kbidu30BConfig:
|
| 32 |
+
"""32B parameter model – production configuration."""
|
| 33 |
+
vocab_size = 50000
|
| 34 |
+
d_model = 7168
|
| 35 |
+
num_layers = 60
|
| 36 |
+
num_heads = 56
|
| 37 |
+
num_kv_heads = 8
|
| 38 |
+
d_ff = 19200
|
| 39 |
+
max_seq_len = 2048
|
| 40 |
+
dropout = 0.1
|
| 41 |
+
layer_norm_eps = 1e-6
|
| 42 |
+
use_gradient_checkpointing = True
|
| 43 |
+
|
| 44 |
+
batch_size = 2
|
| 45 |
+
learning_rate = 3e-4
|
| 46 |
+
weight_decay = 0.1
|
| 47 |
+
warmup_steps = 2000
|
| 48 |
+
total_steps = 100000
|
| 49 |
+
gradient_accumulation_steps = 8
|
| 50 |
+
|
| 51 |
+
use_streaming = True
|
| 52 |
+
|
| 53 |
+
# --- Vision settings (for multimodal training) ---
|
| 54 |
+
vision_model = "openai/clip-vit-base-patch32"
|
| 55 |
+
freeze_vision = True
|
| 56 |
+
num_image_tokens = 64
|
data/code_train.txt
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def fibonacci(n):
|
| 2 |
+
if n <= 1:
|
| 3 |
+
return n
|
| 4 |
+
return fibonacci(n-1) + fibonacci(n-2)
|
| 5 |
+
|
| 6 |
+
class Person:
|
| 7 |
+
def __init__(self, name, age):
|
| 8 |
+
self.name = name
|
| 9 |
+
self.age = age
|
| 10 |
+
|
| 11 |
+
def greet(self):
|
| 12 |
+
return f"Hello, my name is {self.name}"
|
| 13 |
+
|
| 14 |
+
def is_prime(n):
|
| 15 |
+
if n < 2:
|
| 16 |
+
return False
|
| 17 |
+
for i in range(2, int(n**0.5) + 1):
|
| 18 |
+
if n % i == 0:
|
| 19 |
+
return False
|
| 20 |
+
return True
|
| 21 |
+
|
| 22 |
+
import pandas as pd
|
| 23 |
+
df = pd.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]})
|
| 24 |
+
print(df.mean())
|
| 25 |
+
|
| 26 |
+
def quick_sort(arr):
|
| 27 |
+
if len(arr) <= 1:
|
| 28 |
+
return arr
|
| 29 |
+
pivot = arr[len(arr) // 2]
|
| 30 |
+
left = [x for x in arr if x < pivot]
|
| 31 |
+
middle = [x for x in arr if x == pivot]
|
| 32 |
+
right = [x for x in arr if x > pivot]
|
| 33 |
+
return quick_sort(left) + middle + quick_sort(right)
|
| 34 |
+
|
| 35 |
+
class Stack:
|
| 36 |
+
def __init__(self):
|
| 37 |
+
self.items = []
|
| 38 |
+
def push(self, item):
|
| 39 |
+
self.items.append(item)
|
| 40 |
+
def pop(self):
|
| 41 |
+
return self.items.pop() if self.items else None
|
| 42 |
+
def peek(self):
|
| 43 |
+
return self.items[-1] if self.items else None
|
| 44 |
+
|
| 45 |
+
import requests
|
| 46 |
+
response = requests.get("https://api.github.com")
|
| 47 |
+
data = response.json()
|
| 48 |
+
|
| 49 |
+
def read_file(filepath):
|
| 50 |
+
with open(filepath, 'r') as f:
|
| 51 |
+
return f.read()
|
| 52 |
+
|
| 53 |
+
def write_file(filepath, content):
|
| 54 |
+
with open(filepath, 'w') as f:
|
| 55 |
+
f.write(content)
|
| 56 |
+
|
| 57 |
+
class LinkedList:
|
| 58 |
+
def __init__(self):
|
| 59 |
+
self.head = None
|
| 60 |
+
def append(self, data):
|
| 61 |
+
new_node = Node(data)
|
| 62 |
+
if not self.head:
|
| 63 |
+
self.head = new_node
|
| 64 |
+
return
|
| 65 |
+
current = self.head
|
| 66 |
+
while current.next:
|
| 67 |
+
current = current.next
|
| 68 |
+
current.next = new_node
|
| 69 |
+
|
| 70 |
+
import numpy as np
|
| 71 |
+
arr = np.array([1, 2, 3, 4, 5])
|
| 72 |
+
print(arr.mean())
|
| 73 |
+
|
| 74 |
+
def binary_search(arr, target):
|
| 75 |
+
low, high = 0, len(arr) - 1
|
| 76 |
+
while low <= high:
|
| 77 |
+
mid = (low + high) // 2
|
| 78 |
+
if arr[mid] == target:
|
| 79 |
+
return mid
|
| 80 |
+
elif arr[mid] < target:
|
| 81 |
+
low = mid + 1
|
| 82 |
+
else:
|
| 83 |
+
high = mid - 1
|
| 84 |
+
return -1
|
| 85 |
+
|
| 86 |
+
def merge_sort(arr):
|
| 87 |
+
if len(arr) <= 1:
|
| 88 |
+
return arr
|
| 89 |
+
mid = len(arr) // 2
|
| 90 |
+
left = merge_sort(arr[:mid])
|
| 91 |
+
right = merge_sort(arr[mid:])
|
| 92 |
+
return merge(left, right)
|
| 93 |
+
|
| 94 |
+
def merge(left, right):
|
| 95 |
+
result = []
|
| 96 |
+
i = j = 0
|
| 97 |
+
while i < len(left) and j < len(right):
|
| 98 |
+
if left[i] <= right[j]:
|
| 99 |
+
result.append(left[i])
|
| 100 |
+
i += 1
|
| 101 |
+
else:
|
| 102 |
+
result.append(right[j])
|
| 103 |
+
j += 1
|
| 104 |
+
result.extend(left[i:])
|
| 105 |
+
result.extend(right[j:])
|
| 106 |
+
return result
|
| 107 |
+
|
| 108 |
+
class Node:
|
| 109 |
+
def __init__(self, data):
|
| 110 |
+
self.data = data
|
| 111 |
+
self.next = None
|
| 112 |
+
|
| 113 |
+
import re
|
| 114 |
+
pattern = re.compile(r'\d+')
|
| 115 |
+
matches = pattern.findall("There are 123 apples and 456 oranges")
|
| 116 |
+
|
| 117 |
+
def factorial(n):
|
| 118 |
+
if n <= 1:
|
| 119 |
+
return 1
|
| 120 |
+
return n * factorial(n - 1)
|
| 121 |
+
|
| 122 |
+
def decorator(func):
|
| 123 |
+
def wrapper(*args, **kwargs):
|
| 124 |
+
print("Before function call")
|
| 125 |
+
result = func(*args, **kwargs)
|
| 126 |
+
print("After function call")
|
| 127 |
+
return result
|
| 128 |
+
return wrapper
|
| 129 |
+
|
| 130 |
+
@decorator
|
| 131 |
+
def hello():
|
| 132 |
+
print("Hello, World!")
|
| 133 |
+
|
| 134 |
+
import json
|
| 135 |
+
data = {"name": "Alice", "age": 30}
|
| 136 |
+
json_str = json.dumps(data)
|
| 137 |
+
parsed = json.loads(json_str)
|
| 138 |
+
|
| 139 |
+
def count_words(text):
|
| 140 |
+
words = text.split()
|
| 141 |
+
return len(words)
|
| 142 |
+
|
| 143 |
+
class Queue:
|
| 144 |
+
def __init__(self):
|
| 145 |
+
self.items = []
|
| 146 |
+
def enqueue(self, item):
|
| 147 |
+
self.items.append(item)
|
| 148 |
+
def dequeue(self):
|
| 149 |
+
return self.items.pop(0) if self.items else None
|
| 150 |
+
|
| 151 |
+
def reverse_string(s):
|
| 152 |
+
return s[::-1]
|
| 153 |
+
|
| 154 |
+
def is_palindrome(s):
|
| 155 |
+
return s == s[::-1]
|
| 156 |
+
|
| 157 |
+
import sys
|
| 158 |
+
print(sys.version)
|
| 159 |
+
|
| 160 |
+
def map_function(func, arr):
|
| 161 |
+
return [func(x) for x in arr]
|
| 162 |
+
|
| 163 |
+
def filter_function(func, arr):
|
| 164 |
+
return [x for x in arr if func(x)]
|
| 165 |
+
|
| 166 |
+
def reduce_function(func, arr):
|
| 167 |
+
result = arr[0]
|
| 168 |
+
for x in arr[1:]:
|
| 169 |
+
result = func(result, x)
|
| 170 |
+
return result
|
| 171 |
+
|
| 172 |
+
import os
|
| 173 |
+
files = os.listdir(".")
|
| 174 |
+
|
| 175 |
+
def power(base, exp):
|
| 176 |
+
return base ** exp
|
| 177 |
+
|
| 178 |
+
def gcd(a, b):
|
| 179 |
+
while b:
|
| 180 |
+
a, b = b, a % b
|
| 181 |
+
return a
|
| 182 |
+
|
| 183 |
+
def lcm(a, b):
|
| 184 |
+
return a * b // gcd(a, b)
|
| 185 |
+
|
| 186 |
+
def is_even(n):
|
| 187 |
+
return n % 2 == 0
|
| 188 |
+
|
| 189 |
+
def is_odd(n):
|
| 190 |
+
return n % 2 != 0
|
| 191 |
+
|
| 192 |
+
def sum_of_squares(arr):
|
| 193 |
+
return sum(x**2 for x in arr)
|
| 194 |
+
|
| 195 |
+
def average(arr):
|
| 196 |
+
return sum(arr) / len(arr) if arr else 0
|
| 197 |
+
|
| 198 |
+
def median(arr):
|
| 199 |
+
sorted_arr = sorted(arr)
|
| 200 |
+
n = len(sorted_arr)
|
| 201 |
+
if n % 2 == 0:
|
| 202 |
+
return (sorted_arr[n//2 - 1] + sorted_arr[n//2]) / 2
|
| 203 |
+
return sorted_arr[n//2]
|
| 204 |
+
|
| 205 |
+
def mode(arr):
|
| 206 |
+
from collections import Counter
|
| 207 |
+
counter = Counter(arr)
|
| 208 |
+
return max(counter, key=counter.get)
|
| 209 |
+
|
| 210 |
+
def std_dev(arr):
|
| 211 |
+
mean = average(arr)
|
| 212 |
+
variance = sum((x - mean) ** 2 for x in arr) / len(arr)
|
| 213 |
+
return variance ** 0.5
|
| 214 |
+
|
| 215 |
+
import math
|
| 216 |
+
print(math.pi)
|
| 217 |
+
print(math.e)
|
| 218 |
+
|
| 219 |
+
def sqrt(n):
|
| 220 |
+
return n ** 0.5
|
| 221 |
+
|
| 222 |
+
def log(n):
|
| 223 |
+
import math
|
| 224 |
+
return math.log(n)
|
| 225 |
+
|
| 226 |
+
def exp(n):
|
| 227 |
+
import math
|
| 228 |
+
return math.exp(n)
|
| 229 |
+
|
| 230 |
+
import datetime
|
| 231 |
+
now = datetime.datetime.now()
|
| 232 |
+
print(now.strftime("%Y-%m-%d %H:%M:%S"))
|
| 233 |
+
|
| 234 |
+
def get_date():
|
| 235 |
+
import datetime
|
| 236 |
+
return datetime.date.today()
|
| 237 |
+
|
| 238 |
+
def get_time():
|
| 239 |
+
import datetime
|
| 240 |
+
return datetime.datetime.now().time()
|
| 241 |
+
|
| 242 |
+
class BankAccount:
|
| 243 |
+
def __init__(self, balance=0):
|
| 244 |
+
self.balance = balance
|
| 245 |
+
def deposit(self, amount):
|
| 246 |
+
self.balance += amount
|
| 247 |
+
def withdraw(self, amount):
|
| 248 |
+
if amount > self.balance:
|
| 249 |
+
raise ValueError("Insufficient funds")
|
| 250 |
+
self.balance -= amount
|
| 251 |
+
def get_balance(self):
|
| 252 |
+
return self.balance
|
| 253 |
+
|
| 254 |
+
class SavingsAccount(BankAccount):
|
| 255 |
+
def __init__(self, balance=0, interest_rate=0.01):
|
| 256 |
+
super().__init__(balance)
|
| 257 |
+
self.interest_rate = interest_rate
|
| 258 |
+
def add_interest(self):
|
| 259 |
+
self.balance *= (1 + self.interest_rate)
|
| 260 |
+
|
| 261 |
+
import random
|
| 262 |
+
print(random.randint(1, 100))
|
| 263 |
+
print(random.choice(["apple", "banana", "cherry"]))
|
| 264 |
+
|
| 265 |
+
def shuffle_list(arr):
|
| 266 |
+
import random
|
| 267 |
+
random.shuffle(arr)
|
| 268 |
+
return arr
|
| 269 |
+
|
| 270 |
+
def sample_list(arr, k):
|
| 271 |
+
import random
|
| 272 |
+
return random.sample(arr, k)
|
| 273 |
+
|
| 274 |
+
def random_choice(arr):
|
| 275 |
+
import random
|
| 276 |
+
return random.choice(arr)
|
data/code_valid.txt
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def quicksort(arr):
|
| 2 |
+
if len(arr) <= 1:
|
| 3 |
+
return arr
|
| 4 |
+
pivot = arr[len(arr)//2]
|
| 5 |
+
left = [x for x in arr if x < pivot]
|
| 6 |
+
middle = [x for x in arr if x == pivot]
|
| 7 |
+
right = [x for x in arr if x > pivot]
|
| 8 |
+
return quicksort(left) + middle + quicksort(right)
|
| 9 |
+
|
| 10 |
+
class Animal:
|
| 11 |
+
def __init__(self, name):
|
| 12 |
+
self.name = name
|
| 13 |
+
def speak(self):
|
| 14 |
+
return f"{self.name} makes a sound"
|
| 15 |
+
|
| 16 |
+
class Dog(Animal):
|
| 17 |
+
def speak(self):
|
| 18 |
+
return f"{self.name} barks"
|
| 19 |
+
|
| 20 |
+
class Cat(Animal):
|
| 21 |
+
def speak(self):
|
| 22 |
+
return f"{self.name} meows"
|
| 23 |
+
|
| 24 |
+
def parse_json(json_str):
|
| 25 |
+
import json
|
| 26 |
+
return json.loads(json_str)
|
| 27 |
+
|
| 28 |
+
def to_json(obj):
|
| 29 |
+
import json
|
| 30 |
+
return json.dumps(obj)
|
| 31 |
+
|
| 32 |
+
def read_csv(filepath):
|
| 33 |
+
import csv
|
| 34 |
+
with open(filepath, 'r') as f:
|
| 35 |
+
reader = csv.reader(f)
|
| 36 |
+
return list(reader)
|
| 37 |
+
|
| 38 |
+
def write_csv(filepath, data):
|
| 39 |
+
import csv
|
| 40 |
+
with open(filepath, 'w', newline='') as f:
|
| 41 |
+
writer = csv.writer(f)
|
| 42 |
+
writer.writerows(data)
|
| 43 |
+
|
| 44 |
+
class ListNode:
|
| 45 |
+
def __init__(self, val=0, next=None):
|
| 46 |
+
self.val = val
|
| 47 |
+
self.next = next
|
| 48 |
+
|
| 49 |
+
def reverse_linked_list(head):
|
| 50 |
+
prev = None
|
| 51 |
+
current = head
|
| 52 |
+
while current:
|
| 53 |
+
next_node = current.next
|
| 54 |
+
current.next = prev
|
| 55 |
+
prev = current
|
| 56 |
+
current = next_node
|
| 57 |
+
return prev
|
| 58 |
+
|
| 59 |
+
def detect_cycle(head):
|
| 60 |
+
slow = fast = head
|
| 61 |
+
while fast and fast.next:
|
| 62 |
+
slow = slow.next
|
| 63 |
+
fast = fast.next.next
|
| 64 |
+
if slow == fast:
|
| 65 |
+
return True
|
| 66 |
+
return False
|
| 67 |
+
|
| 68 |
+
def find_middle(head):
|
| 69 |
+
slow = fast = head
|
| 70 |
+
while fast and fast.next:
|
| 71 |
+
slow = slow.next
|
| 72 |
+
fast = fast.next.next
|
| 73 |
+
return slow
|
data/cyber_train.txt
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
SQL injection is a code injection technique that attackers use to exploit vulnerabilities in web applications. It occurs when user input is not properly sanitized and is directly embedded into SQL queries.
|
| 2 |
+
|
| 3 |
+
Buffer overflow is a vulnerability that occurs when a program writes more data to a buffer than it can hold, causing the excess data to overflow into adjacent memory locations.
|
| 4 |
+
|
| 5 |
+
Cross-Site Scripting (XSS) is a security vulnerability that allows attackers to inject malicious scripts into web pages viewed by other users.
|
| 6 |
+
|
| 7 |
+
Ransomware is a type of malware that encrypts a victim's files and demands a ransom payment in exchange for the decryption key.
|
| 8 |
+
|
| 9 |
+
A zero-day vulnerability is a previously unknown security flaw that has not yet been patched by the software vendor. Attackers can exploit these vulnerabilities before the vendor releases a fix.
|
| 10 |
+
|
| 11 |
+
Phishing is a social engineering attack that attempts to trick victims into revealing sensitive information by masquerading as a trustworthy entity.
|
| 12 |
+
|
| 13 |
+
The CIA triad (Confidentiality, Integrity, Availability) is the foundation of information security.
|
| 14 |
+
|
| 15 |
+
Public-key cryptography uses a pair of keys: a public key for encryption and a private key for decryption. The public key can be shared openly, while the private key must be kept secret.
|
| 16 |
+
|
| 17 |
+
Symmetric encryption uses the same key for both encryption and decryption. Both the sender and receiver must share the same secret key.
|
| 18 |
+
|
| 19 |
+
A firewall is a network security device that monitors and filters incoming and outgoing network traffic based on predetermined security rules.
|
| 20 |
+
|
| 21 |
+
Multi-factor authentication (MFA) requires users to provide two or more verification factors to gain access to a resource, significantly improving security.
|
| 22 |
+
|
| 23 |
+
A Denial-of-Service (DoS) attack aims to make a machine or network resource unavailable by overwhelming it with a flood of illegitimate requests.
|
| 24 |
+
|
| 25 |
+
The MITRE ATT&CK framework is a globally accessible knowledge base of adversary tactics and techniques based on real-world observations.
|
| 26 |
+
|
| 27 |
+
Vulnerability management is the cyclical practice of identifying, classifying, remediating, and mitigating vulnerabilities.
|
| 28 |
+
|
| 29 |
+
Penetration testing is an authorized simulated cyberattack on a computer system performed to evaluate the security of the system.
|
| 30 |
+
|
| 31 |
+
Security Information and Event Management (SIEM) provides real-time analysis of security alerts generated by applications and network hardware.
|
| 32 |
+
|
| 33 |
+
The OWASP Top 10 is a standard awareness document for developers and web application security, representing a broad consensus about the most critical security risks to web applications.
|
| 34 |
+
|
| 35 |
+
Cryptography is the practice and study of techniques for secure communication in the presence of adversarial behavior.
|
| 36 |
+
|
| 37 |
+
A vulnerability scanner is a tool that automatically identifies security weaknesses in systems and applications.
|
| 38 |
+
|
| 39 |
+
Social engineering is the psychological manipulation of people to perform actions or divulge confidential information.
|
| 40 |
+
|
| 41 |
+
Digital forensics is the process of uncovering and interpreting electronic data for use in a court of law.
|
| 42 |
+
|
| 43 |
+
Endpoint Detection and Response (EDR) is a cybersecurity technology that continuously monitors and responds to advanced threats.
|
| 44 |
+
|
| 45 |
+
Zero Trust security model assumes that no user, device, or network traffic should be trusted by default, even if it originates from inside the organization's network perimeter.
|
| 46 |
+
|
| 47 |
+
Access control is the selective restriction of access to a resource, ensuring that only authorized users can access sensitive data.
|
| 48 |
+
|
| 49 |
+
Data encryption transforms readable data into an unreadable format using an algorithm and a key, ensuring that only authorized parties can decrypt and access the data.
|
| 50 |
+
|
| 51 |
+
Intrusion Detection System (IDS) monitors network traffic for suspicious activity and alerts administrators when potential threats are detected.
|
| 52 |
+
|
| 53 |
+
Intrusion Prevention System (IPS) actively blocks detected threats in real-time, preventing them from reaching the target.
|
| 54 |
+
|
| 55 |
+
Role-Based Access Control (RBAC) restricts system access to authorized users based on their roles within an organization.
|
| 56 |
+
|
| 57 |
+
A man-in-the-middle attack occurs when an attacker secretly intercepts and relays messages between two parties who believe they are communicating directly with each other.
|
| 58 |
+
|
| 59 |
+
Vulnerability exploitation is the process of taking advantage of a vulnerability to compromise a system.
|
| 60 |
+
|
| 61 |
+
Patch management is the process of distributing and applying updates to software to fix security vulnerabilities and other bugs.
|
| 62 |
+
|
| 63 |
+
Security audits evaluate an organization's security posture by assessing its policies, procedures, and technical controls against established standards.
|
| 64 |
+
|
| 65 |
+
The Principle of Least Privilege states that users should be given only the minimum levels of access necessary to perform their job functions.
|
| 66 |
+
|
| 67 |
+
Security awareness training educates employees about security risks and best practices to reduce human error and improve overall security posture.
|
| 68 |
+
|
| 69 |
+
Ransomware attacks often use phishing emails to deliver the malware, tricking users into opening malicious attachments or clicking on malicious links.
|
| 70 |
+
|
| 71 |
+
Two-factor authentication (2FA) is a security process that requires users to provide two different authentication factors to verify their identity.
|
| 72 |
+
|
| 73 |
+
Security Operations Center (SOC) is a centralized unit that deals with security issues on an organizational and technical level.
|
| 74 |
+
|
| 75 |
+
Threat intelligence is evidence-based knowledge about existing or emerging threats that can be used to inform security decisions.
|
| 76 |
+
|
| 77 |
+
A security policy is a document that outlines an organization's security requirements, controls, and procedures.
|
| 78 |
+
|
| 79 |
+
Breach and Attack Simulation (BAS) is a security testing methodology that continuously simulates attacks on an organization's infrastructure to identify vulnerabilities.
|
| 80 |
+
|
| 81 |
+
Container security focuses on securing containerized applications and the infrastructure that supports them, including vulnerabilities in container images and runtime environments.
|
| 82 |
+
|
| 83 |
+
Security orchestration, automation, and response (SOAR) is a suite of software solutions that allow organizations to collect data about security threats and respond to events without human assistance.
|
data/cyber_valid.txt
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
The OWASP Top 10 includes Injection, Broken Authentication, Sensitive Data Exposure, and other critical security risks.
|
| 2 |
+
|
| 3 |
+
Defense in depth is a security strategy that layers multiple defensive mechanisms to protect systems and data, so if one layer fails, others still provide protection.
|
| 4 |
+
|
| 5 |
+
Public Key Infrastructure (PKI) is a system that creates, manages, distributes, and revokes digital certificates.
|
| 6 |
+
|
| 7 |
+
A certificate authority (CA) is a trusted entity that issues digital certificates used to verify the identity of individuals or organizations.
|
| 8 |
+
|
| 9 |
+
Digital signatures provide authentication, integrity, and non-repudiation for digital documents and transactions.
|
| 10 |
+
|
| 11 |
+
Hashing algorithms like SHA-256 are used to verify data integrity by producing a unique fixed-size hash for any given input.
|
| 12 |
+
|
| 13 |
+
Secure Sockets Layer (SSL) and Transport Layer Security (TLS) are cryptographic protocols that provide secure communication over a computer network.
|
| 14 |
+
|
| 15 |
+
The principle of least privilege ensures that users only have the access rights necessary to perform their job functions.
|
| 16 |
+
|
| 17 |
+
Identity and Access Management (IAM) is the framework of policies and technologies that ensure the right individuals access the right resources at the right times for the right reasons.
|
| 18 |
+
|
| 19 |
+
Privileged Access Management (PAM) focuses on securing, controlling, and monitoring access to privileged accounts.
|
data/general_train.txt
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Artificial intelligence is the simulation of human intelligence processes by machines, especially computer systems. These processes include learning, reasoning, and self-correction.
|
| 2 |
+
|
| 3 |
+
Machine learning is a subset of artificial intelligence that enables systems to learn and improve from experience without being explicitly programmed.
|
| 4 |
+
|
| 5 |
+
Deep learning is a type of machine learning that uses neural networks with many layers to analyze various factors of data.
|
| 6 |
+
|
| 7 |
+
Natural language processing (NLP) is a branch of artificial intelligence that gives computers the ability to understand text and spoken words in much the same way human beings can.
|
| 8 |
+
|
| 9 |
+
Computer vision is a field of artificial intelligence that trains computers to interpret and understand the visual world.
|
| 10 |
+
|
| 11 |
+
The Internet of Things (IoT) refers to the network of physical objects embedded with sensors, software, and other technologies for the purpose of connecting and exchanging data with other devices and systems over the internet.
|
| 12 |
+
|
| 13 |
+
Blockchain technology is a decentralized, distributed ledger that records the provenance of a digital asset.
|
| 14 |
+
|
| 15 |
+
Cloud computing is the on-demand availability of computer system resources, especially data storage and computing power, without direct active management by the user.
|
| 16 |
+
|
| 17 |
+
Quantum computing is a type of computation that harnesses the collective properties of quantum states, such as superposition, interference, and entanglement, to perform calculations.
|
| 18 |
+
|
| 19 |
+
Big data refers to extremely large data sets that may be analyzed computationally to reveal patterns, trends, and associations.
|
| 20 |
+
|
| 21 |
+
Data science is an interdisciplinary field that uses scientific methods, processes, algorithms, and systems to extract knowledge and insights from structured and unstructured data.
|
| 22 |
+
|
| 23 |
+
Reinforcement learning is an area of machine learning concerned with how software agents ought to take actions in an environment to maximize the cumulative reward.
|
| 24 |
+
|
| 25 |
+
Generative AI refers to artificial intelligence systems that can generate new content such as text, images, audio, and video.
|
| 26 |
+
|
| 27 |
+
Neural networks are computing systems inspired by the biological neural networks that constitute animal brains.
|
| 28 |
+
|
| 29 |
+
Large Language Models (LLMs) are a type of artificial intelligence model that uses deep learning techniques and massive data sets to understand, summarize, generate, and predict new content.
|
| 30 |
+
|
| 31 |
+
Transformer architecture is a neural network architecture that uses self-attention mechanisms to process sequential data, and is the foundation of modern LLMs.
|
| 32 |
+
|
| 33 |
+
The attention mechanism in deep learning allows models to focus on relevant parts of the input when generating output, significantly improving performance on tasks like translation and summarization.
|
| 34 |
+
|
| 35 |
+
Transfer learning is a machine learning technique where a model trained on one task is reused as the starting point for a model on a second task.
|
| 36 |
+
|
| 37 |
+
Fine-tuning is the process of taking a pre-trained model and training it further on a new dataset to adapt it to a specific task.
|
| 38 |
+
|
| 39 |
+
Overfitting occurs when a machine learning model learns the training data too well, including its noise and outliers, resulting in poor performance on new data.
|
| 40 |
+
|
| 41 |
+
Regularization is a technique used to prevent overfitting by adding a penalty term to the loss function to discourage complex models.
|
| 42 |
+
|
| 43 |
+
Gradient descent is an optimization algorithm used to minimize the loss function by iteratively moving in the direction of steepest descent.
|
| 44 |
+
|
| 45 |
+
Backpropagation is an algorithm used to train neural networks by calculating gradients of the loss function with respect to the weights.
|
| 46 |
+
|
| 47 |
+
The learning rate is a hyperparameter that controls how much to change the model weights in response to the estimated error each time the model weights are updated.
|
| 48 |
+
|
| 49 |
+
Batch size is the number of training examples utilized in one iteration of model training.
|
| 50 |
+
|
| 51 |
+
Epoch refers to one complete pass through the entire training dataset during model training.
|
| 52 |
+
|
| 53 |
+
Activation functions such as ReLU, sigmoid, and tanh introduce non-linearity into neural networks, enabling them to learn complex patterns.
|
| 54 |
+
|
| 55 |
+
Dropout is a regularization technique where randomly selected neurons are ignored during training to prevent overfitting.
|
| 56 |
+
|
| 57 |
+
Convolutional Neural Networks (CNNs) are a class of deep neural networks commonly applied to analyzing visual imagery.
|
| 58 |
+
|
| 59 |
+
Recurrent Neural Networks (RNNs) are a class of neural networks that are designed to handle sequential data, such as time series or natural language.
|
| 60 |
+
|
| 61 |
+
Attention mechanisms in neural networks allow the model to weigh the importance of different input elements, improving performance on tasks like machine translation and text summarization.
|
| 62 |
+
|
| 63 |
+
Word embeddings are dense vector representations of words that capture semantic meaning, allowing models to understand word relationships.
|
| 64 |
+
|
| 65 |
+
Tokenization is the process of breaking down text into smaller units called tokens, which are the fundamental building blocks for language models.
|
data/general_valid.txt
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Supervised learning is a type of machine learning where models are trained on labeled data, learning to map inputs to outputs.
|
| 2 |
+
|
| 3 |
+
Unsupervised learning is a type of machine learning that uses unlabeled data to discover hidden patterns or data groupings.
|
| 4 |
+
|
| 5 |
+
Semi-supervised learning uses both labeled and unlabeled data to train machine learning models, combining the strengths of supervised and unsupervised learning.
|
| 6 |
+
|
| 7 |
+
Reinforcement learning involves training agents to make sequences of decisions by rewarding desirable behaviors and punishing undesirable ones.
|
| 8 |
+
|
| 9 |
+
Natural language understanding (NLU) is a subset of NLP that focuses on machine comprehension of human language, enabling systems to grasp meaning, intent, and context.
|
| 10 |
+
|
| 11 |
+
Natural language generation (NLG) is a subset of NLP that focuses on generating natural language text or speech from structured data.
|
| 12 |
+
|
| 13 |
+
Sentiment analysis uses NLP to identify and extract subjective information from source materials, determining the emotional tone behind words.
|
| 14 |
+
|
| 15 |
+
Machine translation is the task of automatically translating text from one language to another using computational methods.
|
| 16 |
+
|
| 17 |
+
Text summarization is the process of creating a concise and coherent summary of a longer text document.
|
| 18 |
+
|
| 19 |
+
Information retrieval is the process of obtaining relevant information from a collection of resources based on a user's query.
|
| 20 |
+
|
| 21 |
+
Question answering systems automatically answer questions posed by humans in natural language.
|
| 22 |
+
|
| 23 |
+
Named Entity Recognition (NER) is a subtask of NLP that locates and classifies named entities in text into predefined categories such as person names, organizations, locations, and more.
|
| 24 |
+
|
| 25 |
+
Coreference resolution is the task of finding all expressions that refer to the same entity in a text.
|
| 26 |
+
|
| 27 |
+
Dependency parsing is the process of analyzing the grammatical structure of a sentence to establish relationships between words.
|
| 28 |
+
|
| 29 |
+
Part-of-speech tagging is the process of assigning a part-of-speech label (noun, verb, adjective, etc.) to each word in a text.
|
| 30 |
+
|
| 31 |
+
Semantic similarity measures how similar two pieces of text are in terms of their meaning, which is crucial for many NLP applications.
|
| 32 |
+
|
| 33 |
+
Text classification is the task of assigning predefined categories to text documents.
|
| 34 |
+
|
| 35 |
+
Topic modeling is a technique for discovering abstract topics in a collection of documents.
|
data/tokenizer/tokenizer.json
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"version": "1.0",
|
| 3 |
+
"truncation": null,
|
| 4 |
+
"padding": null,
|
| 5 |
+
"added_tokens": [
|
| 6 |
+
{
|
| 7 |
+
"id": 0,
|
| 8 |
+
"content": "<PAD>",
|
| 9 |
+
"single_word": false,
|
| 10 |
+
"lstrip": false,
|
| 11 |
+
"rstrip": false,
|
| 12 |
+
"normalized": false,
|
| 13 |
+
"special": true
|
| 14 |
+
},
|
| 15 |
+
{
|
| 16 |
+
"id": 1,
|
| 17 |
+
"content": "<UNK>",
|
| 18 |
+
"single_word": false,
|
| 19 |
+
"lstrip": false,
|
| 20 |
+
"rstrip": false,
|
| 21 |
+
"normalized": false,
|
| 22 |
+
"special": true
|
| 23 |
+
},
|
| 24 |
+
{
|
| 25 |
+
"id": 2,
|
| 26 |
+
"content": "<BOS>",
|
| 27 |
+
"single_word": false,
|
| 28 |
+
"lstrip": false,
|
| 29 |
+
"rstrip": false,
|
| 30 |
+
"normalized": false,
|
| 31 |
+
"special": true
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"id": 3,
|
| 35 |
+
"content": "<EOS>",
|
| 36 |
+
"single_word": false,
|
| 37 |
+
"lstrip": false,
|
| 38 |
+
"rstrip": false,
|
| 39 |
+
"normalized": false,
|
| 40 |
+
"special": true
|
| 41 |
+
}
|
| 42 |
+
],
|
| 43 |
+
"normalizer": null,
|
| 44 |
+
"pre_tokenizer": {
|
| 45 |
+
"type": "ByteLevel",
|
| 46 |
+
"add_prefix_space": true,
|
| 47 |
+
"trim_offsets": true,
|
| 48 |
+
"use_regex": true
|
| 49 |
+
},
|
| 50 |
+
"post_processor": {
|
| 51 |
+
"type": "ByteLevel",
|
| 52 |
+
"add_prefix_space": true,
|
| 53 |
+
"trim_offsets": true,
|
| 54 |
+
"use_regex": true
|
| 55 |
+
},
|
| 56 |
+
"decoder": {
|
| 57 |
+
"type": "ByteLevel",
|
| 58 |
+
"add_prefix_space": true,
|
| 59 |
+
"trim_offsets": true,
|
| 60 |
+
"use_regex": true
|
| 61 |
+
},
|
| 62 |
+
"model": {
|
| 63 |
+
"type": "BPE",
|
| 64 |
+
"dropout": null,
|
| 65 |
+
"unk_token": null,
|
| 66 |
+
"continuing_subword_prefix": null,
|
| 67 |
+
"end_of_word_suffix": null,
|
| 68 |
+
"fuse_unk": false,
|
| 69 |
+
"byte_fallback": false,
|
| 70 |
+
"ignore_merges": false,
|
| 71 |
+
"vocab": {
|
| 72 |
+
"<PAD>": 0,
|
| 73 |
+
"<UNK>": 1,
|
| 74 |
+
"<BOS>": 2,
|
| 75 |
+
"<EOS>": 3,
|
| 76 |
+
".": 4,
|
| 77 |
+
"T": 5,
|
| 78 |
+
"a": 6,
|
| 79 |
+
"d": 7,
|
| 80 |
+
"e": 8,
|
| 81 |
+
"g": 9,
|
| 82 |
+
"h": 10,
|
| 83 |
+
"i": 11,
|
| 84 |
+
"l": 12,
|
| 85 |
+
"m": 13,
|
| 86 |
+
"n": 14,
|
| 87 |
+
"o": 15,
|
| 88 |
+
"p": 16,
|
| 89 |
+
"r": 17,
|
| 90 |
+
"s": 18,
|
| 91 |
+
"t": 19,
|
| 92 |
+
"v": 20,
|
| 93 |
+
"x": 21,
|
| 94 |
+
"»": 22,
|
| 95 |
+
"¿": 23,
|
| 96 |
+
"ï": 24,
|
| 97 |
+
"Ċ": 25,
|
| 98 |
+
"č": 26,
|
| 99 |
+
"Ġ": 27,
|
| 100 |
+
"is": 28,
|
| 101 |
+
"Ġt": 29,
|
| 102 |
+
"Th": 30,
|
| 103 |
+
"ex": 31,
|
| 104 |
+
"in": 32,
|
| 105 |
+
"»¿": 33,
|
| 106 |
+
"": 34,
|
| 107 |
+
"čĊ": 35,
|
| 108 |
+
"Ġis": 36,
|
| 109 |
+
"Ġ": 37,
|
| 110 |
+
"Ġtex": 38,
|
| 111 |
+
"This": 39,
|
| 112 |
+
"Ġtext": 40
|
| 113 |
+
},
|
| 114 |
+
"merges": [
|
| 115 |
+
[
|
| 116 |
+
"i",
|
| 117 |
+
"s"
|
| 118 |
+
],
|
| 119 |
+
[
|
| 120 |
+
"Ġ",
|
| 121 |
+
"t"
|
| 122 |
+
],
|
| 123 |
+
[
|
| 124 |
+
"T",
|
| 125 |
+
"h"
|
| 126 |
+
],
|
| 127 |
+
[
|
| 128 |
+
"e",
|
| 129 |
+
"x"
|
| 130 |
+
],
|
| 131 |
+
[
|
| 132 |
+
"i",
|
| 133 |
+
"n"
|
| 134 |
+
],
|
| 135 |
+
[
|
| 136 |
+
"»",
|
| 137 |
+
"¿"
|
| 138 |
+
],
|
| 139 |
+
[
|
| 140 |
+
"ï",
|
| 141 |
+
"»¿"
|
| 142 |
+
],
|
| 143 |
+
[
|
| 144 |
+
"č",
|
| 145 |
+
"Ċ"
|
| 146 |
+
],
|
| 147 |
+
[
|
| 148 |
+
"Ġ",
|
| 149 |
+
"is"
|
| 150 |
+
],
|
| 151 |
+
[
|
| 152 |
+
"Ġ",
|
| 153 |
+
""
|
| 154 |
+
],
|
| 155 |
+
[
|
| 156 |
+
"Ġt",
|
| 157 |
+
"ex"
|
| 158 |
+
],
|
| 159 |
+
[
|
| 160 |
+
"Th",
|
| 161 |
+
"is"
|
| 162 |
+
],
|
| 163 |
+
[
|
| 164 |
+
"Ġtex",
|
| 165 |
+
"t"
|
| 166 |
+
]
|
| 167 |
+
]
|
| 168 |
+
}
|
| 169 |
+
}
|
data/train.txt
ADDED
|
@@ -0,0 +1,56 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
def hello_world():
|
| 2 |
+
print("Hello, World!")
|
| 3 |
+
|
| 4 |
+
SQL injection is a code injection technique used to attack data-driven applications.
|
| 5 |
+
|
| 6 |
+
Explain the concept of a buffer overflow vulnerability.
|
| 7 |
+
|
| 8 |
+
class Calculator:
|
| 9 |
+
def add(self, a, b):
|
| 10 |
+
return a + b
|
| 11 |
+
|
| 12 |
+
CVE-2024-1234 is a critical vulnerability in the Linux kernel.
|
| 13 |
+
|
| 14 |
+
Write a Python function to check if a number is prime.
|
| 15 |
+
|
| 16 |
+
Cybersecurity is the practice of protecting systems, networks, and programs from digital attacks.
|
| 17 |
+
|
| 18 |
+
What is a zero-day vulnerability?
|
| 19 |
+
|
| 20 |
+
def factorial(n):
|
| 21 |
+
if n <= 1:
|
| 22 |
+
return 1
|
| 23 |
+
return n * factorial(n - 1)
|
| 24 |
+
|
| 25 |
+
Ransomware is a type of malicious software that threatens to publish the victim's data.
|
| 26 |
+
|
| 27 |
+
How does public-key cryptography work?
|
| 28 |
+
|
| 29 |
+
import requests
|
| 30 |
+
response = requests.get("https://api.example.com/data")
|
| 31 |
+
|
| 32 |
+
A firewall is a network security device that monitors and filters incoming and outgoing network traffic.
|
| 33 |
+
|
| 34 |
+
Explain the difference between symmetric and asymmetric encryption.
|
| 35 |
+
|
| 36 |
+
def binary_search(arr, target):
|
| 37 |
+
left, right = 0, len(arr) - 1
|
| 38 |
+
while left <= right:
|
| 39 |
+
mid = (left + right) // 2
|
| 40 |
+
if arr[mid] == target:
|
| 41 |
+
return mid
|
| 42 |
+
elif arr[mid] < target:
|
| 43 |
+
left = mid + 1
|
| 44 |
+
else:
|
| 45 |
+
right = mid - 1
|
| 46 |
+
return -1
|
| 47 |
+
|
| 48 |
+
What is phishing?
|
| 49 |
+
|
| 50 |
+
Malware analysis is the process of understanding the behavior and purpose of a suspicious file or URL.
|
| 51 |
+
|
| 52 |
+
Use the os module to list all files in a directory.
|
| 53 |
+
|
| 54 |
+
Cross-site scripting (XSS) is a security vulnerability that allows attackers to inject malicious scripts into web pages.
|
| 55 |
+
|
| 56 |
+
Explain the concept of defense in depth.
|
data/valid.txt
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
What is a DDoS attack?
|
| 2 |
+
|
| 3 |
+
A DDoS attack is a malicious attempt to disrupt normal traffic of a targeted server.
|
| 4 |
+
|
| 5 |
+
Write a Python decorator that measures execution time.
|
| 6 |
+
|
| 7 |
+
Encryption is the process of converting plaintext into ciphertext.
|
| 8 |
+
|
| 9 |
+
What is the OWASP Top 10?
|
| 10 |
+
|
| 11 |
+
def merge_sort(arr):
|
| 12 |
+
if len(arr) <= 1:
|
| 13 |
+
return arr
|
| 14 |
+
mid = len(arr) // 2
|
| 15 |
+
left = merge_sort(arr[:mid])
|
| 16 |
+
right = merge_sort(arr[mid:])
|
| 17 |
+
return merge(left, right)
|
| 18 |
+
|
| 19 |
+
What is a man-in-the-middle attack?
|
| 20 |
+
|
| 21 |
+
Secure coding practices help prevent vulnerabilities in software.
|
| 22 |
+
|
| 23 |
+
Explain the concept of least privilege.
|
| 24 |
+
|
| 25 |
+
Hash functions are used to verify data integrity.
|
| 26 |
+
|
| 27 |
+
What is a vulnerability scanner?
|
| 28 |
+
|
| 29 |
+
Firewalls can be hardware-based or software-based.
|
data_folder/.gitattributes
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
*.7z filter=lfs diff=lfs merge=lfs -text
|
| 2 |
+
*.arrow filter=lfs diff=lfs merge=lfs -text
|
| 3 |
+
*.bin filter=lfs diff=lfs merge=lfs -text
|
| 4 |
+
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
| 5 |
+
*.ckpt filter=lfs diff=lfs merge=lfs -text
|
| 6 |
+
*.ftz filter=lfs diff=lfs merge=lfs -text
|
| 7 |
+
*.gz filter=lfs diff=lfs merge=lfs -text
|
| 8 |
+
*.h5 filter=lfs diff=lfs merge=lfs -text
|
| 9 |
+
*.joblib filter=lfs diff=lfs merge=lfs -text
|
| 10 |
+
*.lfs.* filter=lfs diff=lfs merge=lfs -text
|
| 11 |
+
*.lz4 filter=lfs diff=lfs merge=lfs -text
|
| 12 |
+
*.mds filter=lfs diff=lfs merge=lfs -text
|
| 13 |
+
*.mlmodel filter=lfs diff=lfs merge=lfs -text
|
| 14 |
+
*.model filter=lfs diff=lfs merge=lfs -text
|
| 15 |
+
*.msgpack filter=lfs diff=lfs merge=lfs -text
|
| 16 |
+
*.npy filter=lfs diff=lfs merge=lfs -text
|
| 17 |
+
*.npz filter=lfs diff=lfs merge=lfs -text
|
| 18 |
+
*.onnx filter=lfs diff=lfs merge=lfs -text
|
| 19 |
+
*.ot filter=lfs diff=lfs merge=lfs -text
|
| 20 |
+
*.parquet filter=lfs diff=lfs merge=lfs -text
|
| 21 |
+
*.pb filter=lfs diff=lfs merge=lfs -text
|
| 22 |
+
*.pickle filter=lfs diff=lfs merge=lfs -text
|
| 23 |
+
*.pkl filter=lfs diff=lfs merge=lfs -text
|
| 24 |
+
*.pt filter=lfs diff=lfs merge=lfs -text
|
| 25 |
+
*.pth filter=lfs diff=lfs merge=lfs -text
|
| 26 |
+
*.rar filter=lfs diff=lfs merge=lfs -text
|
| 27 |
+
*.safetensors filter=lfs diff=lfs merge=lfs -text
|
| 28 |
+
saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
| 29 |
+
*.tar.* filter=lfs diff=lfs merge=lfs -text
|
| 30 |
+
*.tar filter=lfs diff=lfs merge=lfs -text
|
| 31 |
+
*.tflite filter=lfs diff=lfs merge=lfs -text
|
| 32 |
+
*.tgz filter=lfs diff=lfs merge=lfs -text
|
| 33 |
+
*.wasm filter=lfs diff=lfs merge=lfs -text
|
| 34 |
+
*.xz filter=lfs diff=lfs merge=lfs -text
|
| 35 |
+
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 36 |
+
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 37 |
+
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 38 |
+
# Audio files - uncompressed
|
| 39 |
+
*.pcm filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
*.sam filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
*.raw filter=lfs diff=lfs merge=lfs -text
|
| 42 |
+
# Audio files - compressed
|
| 43 |
+
*.aac filter=lfs diff=lfs merge=lfs -text
|
| 44 |
+
*.flac filter=lfs diff=lfs merge=lfs -text
|
| 45 |
+
*.mp3 filter=lfs diff=lfs merge=lfs -text
|
| 46 |
+
*.ogg filter=lfs diff=lfs merge=lfs -text
|
| 47 |
+
*.wav filter=lfs diff=lfs merge=lfs -text
|
| 48 |
+
# Image files - uncompressed
|
| 49 |
+
*.bmp filter=lfs diff=lfs merge=lfs -text
|
| 50 |
+
*.gif filter=lfs diff=lfs merge=lfs -text
|
| 51 |
+
*.png filter=lfs diff=lfs merge=lfs -text
|
| 52 |
+
*.tiff filter=lfs diff=lfs merge=lfs -text
|
| 53 |
+
# Image files - compressed
|
| 54 |
+
*.jpg filter=lfs diff=lfs merge=lfs -text
|
| 55 |
+
*.jpeg filter=lfs diff=lfs merge=lfs -text
|
| 56 |
+
*.webp filter=lfs diff=lfs merge=lfs -text
|
| 57 |
+
# Video files - compressed
|
| 58 |
+
*.mp4 filter=lfs diff=lfs merge=lfs -text
|
| 59 |
+
*.webm filter=lfs diff=lfs merge=lfs -text
|
| 60 |
+
CyberSec-Dataset_escaped.jsonl filter=lfs diff=lfs merge=lfs -text
|
data_folder/CyberSec-Dataset_escaped.jsonl
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:da1d7b888aa697bd44b6e24ea95e02277c34bcd40c9cf6e701cf72bcac0f9143
|
| 3 |
+
size 360844505
|
data_folder/Fenrir.png
ADDED
|
Git LFS Details
|
data_folder/README.md
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: apache-2.0
|
| 3 |
+
task_categories:
|
| 4 |
+
- text-generation
|
| 5 |
+
language:
|
| 6 |
+
- en
|
| 7 |
+
tags:
|
| 8 |
+
- cybersecurity
|
| 9 |
+
- defensive-security
|
| 10 |
+
- instruction-tuning
|
| 11 |
+
size_categories:
|
| 12 |
+
- 10K<n<100K
|
| 13 |
+
dataset_info:
|
| 14 |
+
version: 1.1.0
|
| 15 |
+
---
|
| 16 |
+
# Cybersecurity Defense Instruction-Tuning Dataset (v2.0)
|
| 17 |
+
|
| 18 |
+
<img src="https://huggingface.co/datasets/AlicanKiraz0/Cybersecurity-Dataset-Fenrir-v2.0/resolve/main/Fenrir.png" width="700" />
|
| 19 |
+
|
| 20 |
+
Created by Alican Kiraz
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
## TL;DR
|
| 25 |
+
A ready-to-train dataset of **83,920** high-quality *system / user / assistant* triples for **defensive, alignment-safe cybersecurity SFT** training.
|
| 26 |
+
Apache-2.0 licensed and production-ready.
|
| 27 |
+
**Scope:** OWASP Top 10, MITRE ATT&CK, NIST CSF, CIS Controls, ASD Essential 8, modern authentication (OAuth 2 / OIDC / SAML), SSL / TLS, Cloud & DevSecOps, Cryptography, and AI Security.
|
| 28 |
+
|
| 29 |
+
---
|
| 30 |
+
|
| 31 |
+
## 1 What’s new in v2.0 (2025‑10‑06)
|
| 32 |
+
|
| 33 |
+
| Change | v1.1.0 | **v2.0.0** |
|
| 34 |
+
| ----------------- | ------------------------------------: | ----------------------------------------------------------------------------------------------: |
|
| 35 |
+
| **Rows** | 21 258 | **83 920** *(≈ 4×)* |
|
| 36 |
+
| **Coverage** | OWASP, NIST CSF, +MITRE ATT&CK, CIS | + Deeper Cloud (AWS/Azure/GCP), modern auth hardening, crypto hygiene, AI‑security interplay |
|
| 37 |
+
| **Quality gates** | Dedup, PII scrub, hallucination scans | + adversarial refusal tests (jailbreak/injection), static policy linting, content risk taxonomy |
|
| 38 |
+
| **Format** | Parquet (chat triples) | + stricter schema checks, stable row IDs |
|
| 39 |
+
| **License** | Apache‑2.0 | Apache‑2.0 |
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
**Highlights**
|
| 43 |
+
|
| 44 |
+
* **Big**: 83,920 chat samples with explanations at **senior security engineer** level.
|
| 45 |
+
* **Safe-by-design**: Built-in **rejection patterns** and *alignment* checks against malicious or exploit requests.
|
| 46 |
+
* **Framework-aware**: Content is **mapped to standards** (OWASP / ATT&CK / NIST / CIS).
|
| 47 |
+
* **Cloud & DevSecOps first**: IAM, secrets, CI/CD, container / k8s hardening, logging / SIEM, incident response.
|
| 48 |
+
* **Commercial-friendly**: **Apache-2.0** licensed.
|
| 49 |
+
|
| 50 |
+
---
|
| 51 |
+
|
| 52 |
+
## 2 Dataset Summary
|
| 53 |
+
|
| 54 |
+
| Property | Value |
|
| 55 |
+
| ------------ | ------------------------------------------------------ |
|
| 56 |
+
| **Language** | English |
|
| 57 |
+
| **License** | Apache‑2.0 |
|
| 58 |
+
| **Format** | Parquet (columnar) |
|
| 59 |
+
| **Rows** | **83 920** |
|
| 60 |
+
| **Columns** | `system`, `user`, `assistant` |
|
| 61 |
+
| **Split** | `train` (100 %) |
|
| 62 |
+
|
| 63 |
+
**Record schema (chat triple)**
|
| 64 |
+
|
| 65 |
+
```json
|
| 66 |
+
{
|
| 67 |
+
"system": "You are a seasoned cyber‑defense AI that follows industry ethics...",
|
| 68 |
+
"user": "Compare mitigations for Reflected vs Stored XSS in a modern SPA.",
|
| 69 |
+
"assistant": "Reflected and Stored XSS share core mitigation pillars—output encoding..."
|
| 70 |
+
}
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
---
|
| 74 |
+
|
| 75 |
+
## 3. Coverage & Design
|
| 76 |
+
|
| 77 |
+
### 3.1 Domains & Frameworks
|
| 78 |
+
|
| 79 |
+
* **AppSec & Web**: OWASP Top 10, secure coding, input/output handling, SSRF, deserialization.
|
| 80 |
+
* **Cloud Security**: IAM guardrails, least privilege, key rotation, KMS/HSM, network segmentation, posture mgmt.
|
| 81 |
+
* **DevSecOps**: SAST/DAST, SBOM, supply‑chain, CI/CD signing, container & Kubernetes hardening.
|
| 82 |
+
* **Identity & Access**: OAuth2/OIDC/SAML, MFA/Phishing‑resistant auth, session mgmt.
|
| 83 |
+
* **Crypto Hygiene**: TLS configs, AEAD modes, key lifecycle, randomness, password hashing.
|
| 84 |
+
* **Detection & Response**: logging, SIEM correlation, threat hunting, IR playbooks.
|
| 85 |
+
* **AI‑Security Interplay**: prompt injection defense, data‑poisoning awareness, model‑misuse refusals.
|
| 86 |
+
|
| 87 |
+
### 3.2 Instruction styles
|
| 88 |
+
|
| 89 |
+
* Compare/contrast, step‑by‑step mitigation, checklists, “why it fails” root‑cause analyses, policy rationale, trade‑offs, and “refuse with explanation” for dual‑use prompts.
|
| 90 |
+
|
| 91 |
+
---
|
| 92 |
+
|
| 93 |
+
## 4. Data Creation & Quality
|
| 94 |
+
|
| 95 |
+
1. **Source harvesting**: 250 k+ public technical docs (standards, RFCs, white‑papers, vendor guidance).
|
| 96 |
+
2. **Extraction**: boilerplate stripping, language detection, heuristic paragraph segmentation.
|
| 97 |
+
3. **Topical filtering**: keyword+embedding retrieval towards defensive security only.
|
| 98 |
+
4. **Instruction synthesis**: prompts → *system/user/assistant*; enforced ethics & refusal templates.
|
| 99 |
+
5. **Quality gates** *(multi‑layer)*
|
| 100 |
+
|
| 101 |
+
* **Deduplication**: MinHash + LSH cluster pruning.
|
| 102 |
+
* **PII & profanity scrub**.
|
| 103 |
+
* **Hallucination/inconsistency scans** (LLM‑aided).
|
| 104 |
+
* **Refusal‑pattern tests**: jailbreak & prompt‑injection triggers; no exploit‑building steps.
|
| 105 |
+
* **Manual spot review** (~3 % sample).
|
| 106 |
+
|
| 107 |
+
---
|
| 108 |
+
|
| 109 |
+
## 5. Ethical Use & Safety
|
| 110 |
+
|
| 111 |
+
* **Dual‑use risk**: Dataset intentionally avoids exploit crafting; offensive requests receive **explanatory refusals**.
|
| 112 |
+
* **Bias**: Focus on widely used frameworks (OWASP/NIST/CIS).
|
| 113 |
+
|
| 114 |
+
* *Roadmap*: more regional standards (e.g., ISO/IEC, GDPR security controls).
|
| 115 |
+
* **Provenance**: Only public sources; licensing respected; outputs released under **Apache‑2.0**.
|
| 116 |
+
|
| 117 |
+
---
|
| 118 |
+
|
| 119 |
+
## 6. Limitations
|
| 120 |
+
|
| 121 |
+
* English‑only.
|
| 122 |
+
* Predominantly defensive stance; red‑team tactics only for mitigation context.
|
| 123 |
+
* Security evolves rapidly; periodic refresh planned.
|
| 124 |
+
|
| 125 |
+
---
|
| 126 |
+
|
| 127 |
+
## 7. Example Records
|
| 128 |
+
|
| 129 |
+
**Mitigation checklist:** hardening steps, rationales, pitfalls, references to standards.
|
| 130 |
+
**Refusal sample:** clearly declines malware/exploit construction with safe alternatives (logging, detection, patching).
|
| 131 |
+
|
| 132 |
+
> *All examples adhere to the `system/user/assistant` schema and are engineered to be alignment‑safe.*
|
| 133 |
+
|
| 134 |
+
---
|
| 135 |
+
|
| 136 |
+
## 8. Citation
|
| 137 |
+
|
| 138 |
+
```bibtex
|
| 139 |
+
@dataset{alican_kiraz_2025_heimdall_v2_0,
|
| 140 |
+
author = {Alican Kiraz},
|
| 141 |
+
title = {Fenrir v2.0 — Cybersecurity Defense Instruction-Tuning Dataset},
|
| 142 |
+
year = {2025},
|
| 143 |
+
publisher = {Hugging Face},
|
| 144 |
+
url = {https://huggingface.co/datasets/AlicanKiraz0/Cybersecurity-Dataset-Heimdall-v2.0}
|
| 145 |
+
}
|
| 146 |
+
```
|
| 147 |
+
|
| 148 |
+
---
|
| 149 |
+
|
| 150 |
+
## 9. Changelog
|
| 151 |
+
|
| 152 |
+
* **v2.0.0** (2025‑10‑06) — Expanded to **83 920** rows; deeper Cloud/DevSecOps/Identity coverage; stronger adversarial refusal tests; stricter schema checks.
|
| 153 |
+
* **v1.1.0** (2025‑06‑21) — 21 258 rows; broadened framework coverage; improved automatic quality gates.
|
| 154 |
+
* **v1.0.0** (2025‑06‑17) — Initial 2 500 rows.
|
| 155 |
+
|
| 156 |
+
---
|
data_preprocessor.py
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# data_preprocessor.py
|
| 2 |
+
import hashlib
|
| 3 |
+
import re
|
| 4 |
+
from collections import Counter
|
| 5 |
+
|
| 6 |
+
class DataPreprocessor:
|
| 7 |
+
def __init__(self):
|
| 8 |
+
self.seen_hashes = set()
|
| 9 |
+
|
| 10 |
+
def deduplicate(self, texts):
|
| 11 |
+
"""Remove exact duplicate lines."""
|
| 12 |
+
unique_texts = []
|
| 13 |
+
for text in texts:
|
| 14 |
+
text_hash = hashlib.md5(text.encode()).hexdigest()
|
| 15 |
+
if text_hash not in self.seen_hashes:
|
| 16 |
+
self.seen_hashes.add(text_hash)
|
| 17 |
+
unique_texts.append(text)
|
| 18 |
+
print(f"Removed {len(texts) - len(unique_texts)} duplicates.")
|
| 19 |
+
return unique_texts
|
| 20 |
+
|
| 21 |
+
def filter_quality(self, texts, min_words=5, max_word_ratio=0.9):
|
| 22 |
+
"""Filter short texts and those with too many repeated words."""
|
| 23 |
+
filtered = []
|
| 24 |
+
for text in texts:
|
| 25 |
+
words = text.split()
|
| 26 |
+
if len(words) < min_words:
|
| 27 |
+
continue
|
| 28 |
+
# Check if > 90% of words are the same (spam)
|
| 29 |
+
if len(words) > 0 and max(Counter(words).values()) / len(words) > max_word_ratio:
|
| 30 |
+
continue
|
| 31 |
+
filtered.append(text)
|
| 32 |
+
print(f"Removed {len(texts) - len(filtered)} low-quality texts.")
|
| 33 |
+
return filtered
|
| 34 |
+
|
| 35 |
+
def remove_toxic(self, texts, toxic_keywords=None):
|
| 36 |
+
"""Basic toxicity filter."""
|
| 37 |
+
if toxic_keywords is None:
|
| 38 |
+
toxic_keywords = ["hate", "violence", "kill", "racist", "sexist", "nazi"]
|
| 39 |
+
filtered = []
|
| 40 |
+
for text in texts:
|
| 41 |
+
lower_text = text.lower()
|
| 42 |
+
if any(kw in lower_text for kw in toxic_keywords):
|
| 43 |
+
continue
|
| 44 |
+
filtered.append(text)
|
| 45 |
+
print(f"Removed {len(texts) - len(filtered)} potentially toxic texts.")
|
| 46 |
+
return filtered
|
| 47 |
+
|
| 48 |
+
def preprocess_file(self, input_path, output_path):
|
| 49 |
+
with open(input_path, 'r', encoding='utf-8') as f:
|
| 50 |
+
texts = [line.strip() for line in f if line.strip()]
|
| 51 |
+
|
| 52 |
+
texts = self.deduplicate(texts)
|
| 53 |
+
texts = self.filter_quality(texts)
|
| 54 |
+
texts = self.remove_toxic(texts)
|
| 55 |
+
|
| 56 |
+
with open(output_path, 'w', encoding='utf-8') as f:
|
| 57 |
+
for text in texts:
|
| 58 |
+
f.write(text + '\n')
|
| 59 |
+
print(f"Saved cleaned data to {output_path}")
|
| 60 |
+
|
| 61 |
+
# Usage:
|
| 62 |
+
# preprocessor = DataPreprocessor()
|
| 63 |
+
# preprocessor.preprocess_file("data/raw_train.txt", "data/train.txt")
|
dataset.py
ADDED
|
@@ -0,0 +1,181 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# dataset.py
|
| 2 |
+
import torch
|
| 3 |
+
from torch.utils.data import Dataset, IterableDataset
|
| 4 |
+
import os
|
| 5 |
+
import random
|
| 6 |
+
from datasets import load_dataset as hf_load_dataset
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class TextDataset(Dataset):
|
| 10 |
+
"""
|
| 11 |
+
Simple dataset for a single text file (used for validation).
|
| 12 |
+
"""
|
| 13 |
+
def __init__(self, file_path, tokenizer, max_len):
|
| 14 |
+
self.tokenizer = tokenizer
|
| 15 |
+
self.max_len = max_len
|
| 16 |
+
with open(file_path, 'r', encoding='utf-8') as f:
|
| 17 |
+
self.texts = [line.strip() for line in f if line.strip()]
|
| 18 |
+
|
| 19 |
+
def __len__(self):
|
| 20 |
+
return len(self.texts)
|
| 21 |
+
|
| 22 |
+
def __getitem__(self, idx):
|
| 23 |
+
text = self.texts[idx]
|
| 24 |
+
if hasattr(self.tokenizer, 'encode'):
|
| 25 |
+
ids = self.tokenizer.encode(text)
|
| 26 |
+
else:
|
| 27 |
+
ids = self.tokenizer.encode(text, add_special_tokens=True)
|
| 28 |
+
if len(ids) > self.max_len:
|
| 29 |
+
ids = ids[:self.max_len]
|
| 30 |
+
else:
|
| 31 |
+
if hasattr(self.tokenizer, 'pad_id'):
|
| 32 |
+
pad_id = self.tokenizer.pad_id
|
| 33 |
+
elif hasattr(self.tokenizer, 'pad_token_id'):
|
| 34 |
+
pad_id = self.tokenizer.pad_token_id
|
| 35 |
+
else:
|
| 36 |
+
pad_id = 0
|
| 37 |
+
ids = ids + [pad_id] * (self.max_len - len(ids))
|
| 38 |
+
return torch.tensor(ids, dtype=torch.long)
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
class MultiSourceTextDataset(Dataset):
|
| 42 |
+
"""
|
| 43 |
+
Flexible dataset that can load from:
|
| 44 |
+
- Local text files
|
| 45 |
+
- Hugging Face datasets
|
| 46 |
+
"""
|
| 47 |
+
def __init__(self, tokenizer, max_len=512, local_files=None, hf_datasets=None, shuffle=True):
|
| 48 |
+
self.tokenizer = tokenizer
|
| 49 |
+
self.max_len = max_len
|
| 50 |
+
self.shuffle = shuffle
|
| 51 |
+
self.data = []
|
| 52 |
+
|
| 53 |
+
if local_files:
|
| 54 |
+
for file_path in local_files:
|
| 55 |
+
if os.path.exists(file_path):
|
| 56 |
+
with open(file_path, 'r', encoding='utf-8') as f:
|
| 57 |
+
lines = [line.strip() for line in f if line.strip()]
|
| 58 |
+
self.data.extend(lines)
|
| 59 |
+
print(f" Loaded {len(lines)} from {file_path}")
|
| 60 |
+
|
| 61 |
+
if hf_datasets:
|
| 62 |
+
for ds_name in hf_datasets:
|
| 63 |
+
try:
|
| 64 |
+
ds = hf_load_dataset(ds_name, split="train", trust_remote_code=True)
|
| 65 |
+
if "content" in ds.column_names:
|
| 66 |
+
texts = [item["content"] for item in ds]
|
| 67 |
+
elif "text" in ds.column_names:
|
| 68 |
+
texts = [item["text"] for item in ds]
|
| 69 |
+
else:
|
| 70 |
+
col = ds.column_names[0]
|
| 71 |
+
texts = [item[col] for item in ds]
|
| 72 |
+
self.data.extend(texts)
|
| 73 |
+
print(f" Loaded {len(texts)} from {ds_name}")
|
| 74 |
+
except Exception as e:
|
| 75 |
+
print(f" ⚠️ Error loading {ds_name}: {e}")
|
| 76 |
+
|
| 77 |
+
if not self.data:
|
| 78 |
+
print("⚠️ No data loaded!")
|
| 79 |
+
|
| 80 |
+
def __len__(self):
|
| 81 |
+
return len(self.data)
|
| 82 |
+
|
| 83 |
+
def __getitem__(self, idx):
|
| 84 |
+
text = self.data[idx]
|
| 85 |
+
if hasattr(self.tokenizer, 'encode'):
|
| 86 |
+
ids = self.tokenizer.encode(text)
|
| 87 |
+
else:
|
| 88 |
+
ids = self.tokenizer.encode(text, add_special_tokens=True)
|
| 89 |
+
if len(ids) > self.max_len:
|
| 90 |
+
ids = ids[:self.max_len]
|
| 91 |
+
else:
|
| 92 |
+
if hasattr(self.tokenizer, 'pad_id'):
|
| 93 |
+
pad_id = self.tokenizer.pad_id
|
| 94 |
+
elif hasattr(self.tokenizer, 'pad_token_id'):
|
| 95 |
+
pad_id = self.tokenizer.pad_token_id
|
| 96 |
+
else:
|
| 97 |
+
pad_id = 0
|
| 98 |
+
ids = ids + [pad_id] * (self.max_len - len(ids))
|
| 99 |
+
return torch.tensor(ids, dtype=torch.long)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
class StreamingMultiSourceDataset(IterableDataset):
|
| 103 |
+
"""
|
| 104 |
+
Memory-efficient streaming dataset for huge datasets.
|
| 105 |
+
"""
|
| 106 |
+
def __init__(self, tokenizer, max_len=512, file_paths=None, shuffle=True, buffer_size=10000):
|
| 107 |
+
self.tokenizer = tokenizer
|
| 108 |
+
self.max_len = max_len
|
| 109 |
+
self.file_paths = file_paths or []
|
| 110 |
+
self.shuffle = shuffle
|
| 111 |
+
self.buffer_size = buffer_size
|
| 112 |
+
|
| 113 |
+
def __iter__(self):
|
| 114 |
+
for file_path in self.file_paths:
|
| 115 |
+
if not os.path.exists(file_path):
|
| 116 |
+
continue
|
| 117 |
+
with open(file_path, 'r', encoding='utf-8') as f:
|
| 118 |
+
buffer = []
|
| 119 |
+
for line in f:
|
| 120 |
+
line = line.strip()
|
| 121 |
+
if line:
|
| 122 |
+
if self.shuffle:
|
| 123 |
+
buffer.append(line)
|
| 124 |
+
if len(buffer) >= self.buffer_size:
|
| 125 |
+
random.shuffle(buffer)
|
| 126 |
+
for item in buffer:
|
| 127 |
+
yield self._tokenize(item)
|
| 128 |
+
buffer = []
|
| 129 |
+
else:
|
| 130 |
+
yield self._tokenize(line)
|
| 131 |
+
if buffer:
|
| 132 |
+
random.shuffle(buffer)
|
| 133 |
+
for item in buffer:
|
| 134 |
+
yield self._tokenize(item)
|
| 135 |
+
|
| 136 |
+
def _tokenize(self, text):
|
| 137 |
+
if hasattr(self.tokenizer, 'encode'):
|
| 138 |
+
ids = self.tokenizer.encode(text)
|
| 139 |
+
else:
|
| 140 |
+
ids = self.tokenizer.encode(text, add_special_tokens=True)
|
| 141 |
+
if len(ids) > self.max_len:
|
| 142 |
+
ids = ids[:self.max_len]
|
| 143 |
+
else:
|
| 144 |
+
if hasattr(self.tokenizer, 'pad_id'):
|
| 145 |
+
pad_id = self.tokenizer.pad_id
|
| 146 |
+
elif hasattr(self.tokenizer, 'pad_token_id'):
|
| 147 |
+
pad_id = self.tokenizer.pad_token_id
|
| 148 |
+
else:
|
| 149 |
+
pad_id = 0
|
| 150 |
+
ids = ids + [pad_id] * (self.max_len - len(ids))
|
| 151 |
+
return torch.tensor(ids, dtype=torch.long)
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def create_mixed_dataset(tokenizer, max_len=512, config=None):
|
| 155 |
+
"""
|
| 156 |
+
Factory function that creates a mixed dataset.
|
| 157 |
+
Uses streaming if config.use_streaming is True.
|
| 158 |
+
"""
|
| 159 |
+
local_files = []
|
| 160 |
+
for f in ["data/code_train.txt", "data/cyber_train.txt", "data/general_train.txt", "data/train.txt"]:
|
| 161 |
+
if os.path.exists(f):
|
| 162 |
+
local_files.append(f)
|
| 163 |
+
|
| 164 |
+
use_streaming = config and hasattr(config, 'use_streaming') and config.use_streaming
|
| 165 |
+
|
| 166 |
+
if use_streaming:
|
| 167 |
+
return StreamingMultiSourceDataset(
|
| 168 |
+
tokenizer=tokenizer,
|
| 169 |
+
max_len=max_len,
|
| 170 |
+
file_paths=local_files,
|
| 171 |
+
shuffle=True,
|
| 172 |
+
buffer_size=10000
|
| 173 |
+
)
|
| 174 |
+
else:
|
| 175 |
+
return MultiSourceTextDataset(
|
| 176 |
+
tokenizer=tokenizer,
|
| 177 |
+
max_len=max_len,
|
| 178 |
+
local_files=local_files,
|
| 179 |
+
hf_datasets=[],
|
| 180 |
+
shuffle=True
|
| 181 |
+
)
|
dpo_trainer.py
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# dpo_trainer.py
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn as nn
|
| 4 |
+
import torch.nn.functional as F
|
| 5 |
+
from torch.utils.data import Dataset, DataLoader
|
| 6 |
+
from transformers import AutoTokenizer
|
| 7 |
+
import random
|
| 8 |
+
|
| 9 |
+
class PreferenceDataset(Dataset):
|
| 10 |
+
def __init__(self, data_path, tokenizer, max_len=512):
|
| 11 |
+
self.tokenizer = tokenizer
|
| 12 |
+
self.max_len = max_len
|
| 13 |
+
self.data = []
|
| 14 |
+
with open(data_path, 'r', encoding='utf-8') as f:
|
| 15 |
+
for line in f:
|
| 16 |
+
parts = line.strip().split('\t')
|
| 17 |
+
if len(parts) == 3:
|
| 18 |
+
prompt, chosen, rejected = parts
|
| 19 |
+
self.data.append((prompt, chosen, rejected))
|
| 20 |
+
|
| 21 |
+
def __len__(self):
|
| 22 |
+
return len(self.data)
|
| 23 |
+
|
| 24 |
+
def __getitem__(self, idx):
|
| 25 |
+
prompt, chosen, rejected = self.data[idx]
|
| 26 |
+
# Encode
|
| 27 |
+
prompt_ids = self.tokenizer.encode(prompt)
|
| 28 |
+
chosen_ids = prompt_ids + self.tokenizer.encode(chosen)[1:] # remove BOS
|
| 29 |
+
rejected_ids = prompt_ids + self.tokenizer.encode(rejected)[1:]
|
| 30 |
+
# Pad
|
| 31 |
+
if len(chosen_ids) > self.max_len:
|
| 32 |
+
chosen_ids = chosen_ids[:self.max_len]
|
| 33 |
+
else:
|
| 34 |
+
chosen_ids += [self.tokenizer.pad_id] * (self.max_len - len(chosen_ids))
|
| 35 |
+
if len(rejected_ids) > self.max_len:
|
| 36 |
+
rejected_ids = rejected_ids[:self.max_len]
|
| 37 |
+
else:
|
| 38 |
+
rejected_ids += [self.tokenizer.pad_id] * (self.max_len - len(rejected_ids))
|
| 39 |
+
return torch.tensor(chosen_ids, dtype=torch.long), torch.tensor(rejected_ids, dtype=torch.long)
|
| 40 |
+
|
| 41 |
+
class DPOTrainer:
|
| 42 |
+
def __init__(self, model, ref_model, train_loader, config, device, beta=0.1, lr=1e-5):
|
| 43 |
+
self.model = model
|
| 44 |
+
self.ref_model = ref_model
|
| 45 |
+
self.train_loader = train_loader
|
| 46 |
+
self.config = config
|
| 47 |
+
self.device = device
|
| 48 |
+
self.beta = beta
|
| 49 |
+
self.optimizer = torch.optim.AdamW(model.parameters(), lr=lr)
|
| 50 |
+
|
| 51 |
+
def compute_dpo_loss(self, chosen_logps, rejected_logps):
|
| 52 |
+
# DPO loss: -log(sigmoid(beta * (chosen - rejected)))
|
| 53 |
+
diff = chosen_logps - rejected_logps
|
| 54 |
+
loss = -F.logsigmoid(self.beta * diff).mean()
|
| 55 |
+
return loss
|
| 56 |
+
|
| 57 |
+
def train_step(self, batch_chosen, batch_rejected):
|
| 58 |
+
batch_chosen = batch_chosen.to(self.device)
|
| 59 |
+
batch_rejected = batch_rejected.to(self.device)
|
| 60 |
+
|
| 61 |
+
# Forward pass on chosen
|
| 62 |
+
logits_chosen = self.model(batch_chosen)
|
| 63 |
+
logits_ref_chosen = self.ref_model(batch_chosen)
|
| 64 |
+
|
| 65 |
+
# Forward pass on rejected
|
| 66 |
+
logits_rejected = self.model(batch_rejected)
|
| 67 |
+
logits_ref_rejected = self.ref_model(batch_rejected)
|
| 68 |
+
|
| 69 |
+
# Compute log probabilities (ignoring padding)
|
| 70 |
+
logps_chosen = F.log_softmax(logits_chosen, dim=-1).gather(-1, batch_chosen.unsqueeze(-1)).squeeze(-1).sum(-1)
|
| 71 |
+
logps_ref_chosen = F.log_softmax(logits_ref_chosen, dim=-1).gather(-1, batch_chosen.unsqueeze(-1)).squeeze(-1).sum(-1)
|
| 72 |
+
|
| 73 |
+
logps_rejected = F.log_softmax(logits_rejected, dim=-1).gather(-1, batch_rejected.unsqueeze(-1)).squeeze(-1).sum(-1)
|
| 74 |
+
logps_ref_rejected = F.log_softmax(logits_ref_rejected, dim=-1).gather(-1, batch_rejected.unsqueeze(-1)).squeeze(-1).sum(-1)
|
| 75 |
+
|
| 76 |
+
# DPO loss
|
| 77 |
+
loss = self.compute_dpo_loss(logps_chosen - logps_ref_chosen, logps_rejected - logps_ref_rejected)
|
| 78 |
+
|
| 79 |
+
self.optimizer.zero_grad()
|
| 80 |
+
loss.backward()
|
| 81 |
+
torch.nn.utils.clip_grad_norm_(self.model.parameters(), 1.0)
|
| 82 |
+
self.optimizer.step()
|
| 83 |
+
return loss.item()
|
| 84 |
+
|
| 85 |
+
def train(self, epochs=1):
|
| 86 |
+
self.model.train()
|
| 87 |
+
for epoch in range(epochs):
|
| 88 |
+
total_loss = 0
|
| 89 |
+
for batch_chosen, batch_rejected in self.train_loader:
|
| 90 |
+
loss = self.train_step(batch_chosen, batch_rejected)
|
| 91 |
+
total_loss += loss
|
| 92 |
+
print(f"DPO Epoch {epoch+1}, Loss: {total_loss/len(self.train_loader):.4f}")
|
| 93 |
+
|
| 94 |
+
# Usage:
|
| 95 |
+
# dpo_dataset = PreferenceDataset("data/preference.txt", tokenizer)
|
| 96 |
+
# dpo_loader = DataLoader(dpo_dataset, batch_size=4, shuffle=True)
|
| 97 |
+
# ref_model = KbiduForCausalLM(config) # copy of base model (frozen)
|
| 98 |
+
# dpo_trainer = DPOTrainer(model, ref_model, dpo_loader, config, device)
|
| 99 |
+
# dpo_trainer.train(epochs=3)
|
embedding.py
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# embedding.py
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
|
| 4 |
+
class Embedding(nn.Module):
|
| 5 |
+
def __init__(self, vocab_size, d_model):
|
| 6 |
+
super().__init__()
|
| 7 |
+
self.d_model = d_model
|
| 8 |
+
self.embed = nn.Embedding(vocab_size, d_model)
|
| 9 |
+
|
| 10 |
+
def forward(self, x):
|
| 11 |
+
return self.embed(x) * (self.d_model ** 0.5)
|
eval_math.py
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# eval_math.py
|
| 2 |
+
import re
|
| 3 |
+
from datasets import load_dataset
|
| 4 |
+
from inference import Inference
|
| 5 |
+
from tqdm import tqdm
|
| 6 |
+
|
| 7 |
+
def extract_answer(text):
|
| 8 |
+
"""Extract final numeric answer from text."""
|
| 9 |
+
# Try to find the last number in the text
|
| 10 |
+
numbers = re.findall(r'[-+]?\d*\.?\d+', text)
|
| 11 |
+
if numbers:
|
| 12 |
+
return numbers[-1]
|
| 13 |
+
return None
|
| 14 |
+
|
| 15 |
+
def evaluate_gsm8k(model, tokenizer, device, num_samples=100):
|
| 16 |
+
"""Evaluate on GSM8K dataset."""
|
| 17 |
+
dataset = load_dataset("gsm8k", "main", split="test")
|
| 18 |
+
total = 0
|
| 19 |
+
correct = 0
|
| 20 |
+
inference_engine = Inference(model, tokenizer, device, max_new_tokens=200)
|
| 21 |
+
|
| 22 |
+
for i, example in enumerate(tqdm(dataset.select(range(num_samples)))):
|
| 23 |
+
question = example["question"]
|
| 24 |
+
true_answer = example["answer"].split("####")[-1].strip()
|
| 25 |
+
|
| 26 |
+
# Generate with CoT
|
| 27 |
+
output = inference_engine.generate(question, use_cot=True)
|
| 28 |
+
pred_answer = extract_answer(output)
|
| 29 |
+
|
| 30 |
+
if pred_answer and pred_answer in true_answer:
|
| 31 |
+
correct += 1
|
| 32 |
+
total += 1
|
| 33 |
+
|
| 34 |
+
accuracy = correct / total
|
| 35 |
+
print(f"GSM8K Accuracy ({num_samples} samples): {accuracy:.2%}")
|
| 36 |
+
return accuracy
|
| 37 |
+
|
| 38 |
+
def evaluate_math(model, tokenizer, device, num_samples=100):
|
| 39 |
+
"""Evaluate on MATH dataset (college math)."""
|
| 40 |
+
# Similar logic for MATH dataset
|
| 41 |
+
dataset = load_dataset("math_dataset", split="test")
|
| 42 |
+
# ... implementation similar to above
|
| 43 |
+
pass
|
| 44 |
+
|
| 45 |
+
# Usage:
|
| 46 |
+
# from model import KbiduForCausalLM
|
| 47 |
+
# model = KbiduForCausalLM(config).to(device)
|
| 48 |
+
# evaluate_gsm8k(model, tokenizer, device, num_samples=50)
|
faiss_index.faiss
ADDED
|
Binary file (1.58 kB). View file
|
|
|
faiss_index_chunks.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
This is sample training text.
|
feedforward.py
ADDED
|
@@ -0,0 +1,14 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# feedforward.py
|
| 2 |
+
import torch.nn as nn
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
|
| 5 |
+
class SwiGLU(nn.Module):
|
| 6 |
+
def __init__(self, d_model, d_ff, dropout=0.1):
|
| 7 |
+
super().__init__()
|
| 8 |
+
self.w1 = nn.Linear(d_model, d_ff, bias=False)
|
| 9 |
+
self.w2 = nn.Linear(d_ff, d_model, bias=False)
|
| 10 |
+
self.w3 = nn.Linear(d_model, d_ff, bias=False)
|
| 11 |
+
self.dropout = nn.Dropout(dropout)
|
| 12 |
+
|
| 13 |
+
def forward(self, x):
|
| 14 |
+
return self.w2(self.dropout(F.silu(self.w1(x)) * self.w3(x)))
|
inference.py
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# inference.py
|
| 2 |
+
import torch
|
| 3 |
+
import torch.nn.functional as F
|
| 4 |
+
from collections import Counter
|
| 5 |
+
import re
|
| 6 |
+
|
| 7 |
+
class Inference:
|
| 8 |
+
def __init__(self, model, tokenizer, device, retriever=None, max_new_tokens=300, memory_size=5):
|
| 9 |
+
self.model = model
|
| 10 |
+
self.tokenizer = tokenizer
|
| 11 |
+
self.device = device
|
| 12 |
+
self.retriever = retriever
|
| 13 |
+
self.max_new_tokens = max_new_tokens
|
| 14 |
+
self.memory = []
|
| 15 |
+
self.memory_size = memory_size
|
| 16 |
+
|
| 17 |
+
def _build_prompt(self, query, use_cot=True, tone="empathetic", think_before_answer=False):
|
| 18 |
+
tone_map = {
|
| 19 |
+
"empathetic": "You are Kbidu AI, a warm, kind, and emotionally intelligent assistant.",
|
| 20 |
+
"professional": "You are Kbidu AI, a precise, clear, and objective assistant.",
|
| 21 |
+
"humorous": "You are Kbidu AI, a witty and fun assistant.",
|
| 22 |
+
"supportive": "You are Kbidu AI, a supportive and encouraging mentor."
|
| 23 |
+
}
|
| 24 |
+
system_prompt = tone_map.get(tone, tone_map["empathetic"])
|
| 25 |
+
|
| 26 |
+
# RAG context
|
| 27 |
+
context = ""
|
| 28 |
+
if self.retriever:
|
| 29 |
+
results = self.retriever.retrieve(query, top_k=3)
|
| 30 |
+
if results:
|
| 31 |
+
context = "Relevant information:\n" + "\n".join([r["chunk"] for r in results]) + "\n\n"
|
| 32 |
+
|
| 33 |
+
# Conversation memory
|
| 34 |
+
memory_text = ""
|
| 35 |
+
if self.memory:
|
| 36 |
+
memory_text = "Conversation history:\n"
|
| 37 |
+
for user, assistant in self.memory[-self.memory_size:]:
|
| 38 |
+
memory_text += f"User: {user}\nAssistant: {assistant}\n"
|
| 39 |
+
memory_text += "\n"
|
| 40 |
+
|
| 41 |
+
# Think-before-answer instruction
|
| 42 |
+
think_instruction = ""
|
| 43 |
+
if think_before_answer:
|
| 44 |
+
think_instruction = (
|
| 45 |
+
"Before giving your final answer, think step by step inside your head.\n"
|
| 46 |
+
"Then, provide your final answer after the line 'Final Answer:'.\n"
|
| 47 |
+
"Do not include your reasoning in the final output unless asked.\n"
|
| 48 |
+
)
|
| 49 |
+
|
| 50 |
+
cot_instruction = "Let's think step by step.\n" if use_cot else ""
|
| 51 |
+
|
| 52 |
+
prompt = f"{system_prompt}\n\n{context}{memory_text}{think_instruction}{cot_instruction}User: {query}\nAssistant:"
|
| 53 |
+
return prompt
|
| 54 |
+
|
| 55 |
+
def generate(self, query, use_cot=True, temperature=0.7, tone="empathetic", think_before_answer=False, show_reasoning=False):
|
| 56 |
+
prompt = self._build_prompt(query, use_cot, tone, think_before_answer)
|
| 57 |
+
input_ids = torch.tensor([self.tokenizer.encode(prompt)]).to(self.device)
|
| 58 |
+
|
| 59 |
+
for _ in range(self.max_new_tokens):
|
| 60 |
+
with torch.no_grad():
|
| 61 |
+
logits = self.model(input_ids)
|
| 62 |
+
next_logits = logits[0, -1, :] / temperature
|
| 63 |
+
probs = F.softmax(next_logits, dim=-1)
|
| 64 |
+
next_token = torch.multinomial(probs, num_samples=1).unsqueeze(0)
|
| 65 |
+
input_ids = torch.cat([input_ids, next_token], dim=-1)
|
| 66 |
+
if next_token.item() == self.tokenizer.eos_id:
|
| 67 |
+
break
|
| 68 |
+
|
| 69 |
+
full_response = self.tokenizer.decode(input_ids[0].tolist())
|
| 70 |
+
|
| 71 |
+
# Extract assistant part
|
| 72 |
+
if "Assistant:" in full_response:
|
| 73 |
+
response = full_response.split("Assistant:", 1)[-1].strip()
|
| 74 |
+
else:
|
| 75 |
+
response = full_response[len(prompt):].strip()
|
| 76 |
+
|
| 77 |
+
# If think_before_answer is True, try to extract final answer after "Final Answer:"
|
| 78 |
+
if think_before_answer:
|
| 79 |
+
if "Final Answer:" in response:
|
| 80 |
+
final_answer = response.split("Final Answer:", 1)[-1].strip()
|
| 81 |
+
if show_reasoning:
|
| 82 |
+
# Return the full response (reasoning + final answer)
|
| 83 |
+
self.memory.append((query, response))
|
| 84 |
+
return response
|
| 85 |
+
else:
|
| 86 |
+
# Return only the final answer
|
| 87 |
+
self.memory.append((query, final_answer))
|
| 88 |
+
return final_answer
|
| 89 |
+
else:
|
| 90 |
+
# If no "Final Answer:" found, return the full response (fallback)
|
| 91 |
+
self.memory.append((query, response))
|
| 92 |
+
return response
|
| 93 |
+
else:
|
| 94 |
+
self.memory.append((query, response))
|
| 95 |
+
return response
|
| 96 |
+
|
| 97 |
+
def generate_with_self_consistency(self, query, n=5, temperature=0.7, tone="empathetic", think_before_answer=False):
|
| 98 |
+
answers = []
|
| 99 |
+
for _ in range(n):
|
| 100 |
+
ans = self.generate(query, use_cot=True, temperature=temperature, tone=tone, think_before_answer=think_before_answer, show_reasoning=False)
|
| 101 |
+
if "Answer:" in ans:
|
| 102 |
+
ans = ans.split("Answer:")[-1].strip()
|
| 103 |
+
answers.append(ans)
|
| 104 |
+
if answers:
|
| 105 |
+
counter = Counter(answers)
|
| 106 |
+
return counter.most_common(1)[0][0]
|
| 107 |
+
return answers[0] if answers else ""
|
kbidu_env/.gitignore
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Created by venv; see https://docs.python.org/3/library/venv.html
|
| 2 |
+
*
|
kbidu_env/Lib/site-packages/81d243bd2c585b0f4821__mypyc.cp314-win_amd64.pyd
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:ab64835b63093c31975df2692756c8008e7ec190e4aa86f15a713bcc62e1d432
|
| 3 |
+
size 216576
|
kbidu_env/Lib/site-packages/__pycache__/isympy.cpython-314.pyc
ADDED
|
Binary file (11.3 kB). View file
|
|
|
kbidu_env/Lib/site-packages/__pycache__/six.cpython-314.pyc
ADDED
|
Binary file (43 kB). View file
|
|
|
kbidu_env/Lib/site-packages/__pycache__/threadpoolctl.cpython-314.pyc
ADDED
|
Binary file (59.5 kB). View file
|
|
|
kbidu_env/Lib/site-packages/__pycache__/typing_extensions.cpython-314.pyc
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:42c2b14d3bd1c9fc087ac29ced4cef30dc3b08c3a322502df232532b1cb6aa1f
|
| 3 |
+
size 175506
|
kbidu_env/Lib/site-packages/_distutils_hack/__init__.py
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# don't import any costly modules
|
| 2 |
+
import sys
|
| 3 |
+
import os
|
| 4 |
+
|
| 5 |
+
|
| 6 |
+
def warn_distutils_present():
|
| 7 |
+
if 'distutils' not in sys.modules:
|
| 8 |
+
return
|
| 9 |
+
import warnings
|
| 10 |
+
|
| 11 |
+
warnings.warn(
|
| 12 |
+
"Distutils was imported before Setuptools, but importing Setuptools "
|
| 13 |
+
"also replaces the `distutils` module in `sys.modules`. This may lead "
|
| 14 |
+
"to undesirable behaviors or errors. To avoid these issues, avoid "
|
| 15 |
+
"using distutils directly, ensure that setuptools is installed in the "
|
| 16 |
+
"traditional way (e.g. not an editable install), and/or make sure "
|
| 17 |
+
"that setuptools is always imported before distutils."
|
| 18 |
+
)
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def clear_distutils():
|
| 22 |
+
if 'distutils' not in sys.modules:
|
| 23 |
+
return
|
| 24 |
+
import warnings
|
| 25 |
+
|
| 26 |
+
warnings.warn("Setuptools is replacing distutils.")
|
| 27 |
+
mods = [
|
| 28 |
+
name
|
| 29 |
+
for name in sys.modules
|
| 30 |
+
if name == "distutils" or name.startswith("distutils.")
|
| 31 |
+
]
|
| 32 |
+
for name in mods:
|
| 33 |
+
del sys.modules[name]
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def enabled():
|
| 37 |
+
"""
|
| 38 |
+
Allow selection of distutils by environment variable.
|
| 39 |
+
"""
|
| 40 |
+
which = os.environ.get('SETUPTOOLS_USE_DISTUTILS', 'local')
|
| 41 |
+
return which == 'local'
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def ensure_local_distutils():
|
| 45 |
+
import importlib
|
| 46 |
+
|
| 47 |
+
clear_distutils()
|
| 48 |
+
|
| 49 |
+
# With the DistutilsMetaFinder in place,
|
| 50 |
+
# perform an import to cause distutils to be
|
| 51 |
+
# loaded from setuptools._distutils. Ref #2906.
|
| 52 |
+
with shim():
|
| 53 |
+
importlib.import_module('distutils')
|
| 54 |
+
|
| 55 |
+
# check that submodules load as expected
|
| 56 |
+
core = importlib.import_module('distutils.core')
|
| 57 |
+
assert '_distutils' in core.__file__, core.__file__
|
| 58 |
+
assert 'setuptools._distutils.log' not in sys.modules
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def do_override():
|
| 62 |
+
"""
|
| 63 |
+
Ensure that the local copy of distutils is preferred over stdlib.
|
| 64 |
+
|
| 65 |
+
See https://github.com/pypa/setuptools/issues/417#issuecomment-392298401
|
| 66 |
+
for more motivation.
|
| 67 |
+
"""
|
| 68 |
+
if enabled():
|
| 69 |
+
warn_distutils_present()
|
| 70 |
+
ensure_local_distutils()
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
class _TrivialRe:
|
| 74 |
+
def __init__(self, *patterns):
|
| 75 |
+
self._patterns = patterns
|
| 76 |
+
|
| 77 |
+
def match(self, string):
|
| 78 |
+
return all(pat in string for pat in self._patterns)
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
class DistutilsMetaFinder:
|
| 82 |
+
def find_spec(self, fullname, path, target=None):
|
| 83 |
+
# optimization: only consider top level modules and those
|
| 84 |
+
# found in the CPython test suite.
|
| 85 |
+
if path is not None and not fullname.startswith('test.'):
|
| 86 |
+
return None
|
| 87 |
+
|
| 88 |
+
method_name = 'spec_for_{fullname}'.format(**locals())
|
| 89 |
+
method = getattr(self, method_name, lambda: None)
|
| 90 |
+
return method()
|
| 91 |
+
|
| 92 |
+
def spec_for_distutils(self):
|
| 93 |
+
if self.is_cpython():
|
| 94 |
+
return None
|
| 95 |
+
|
| 96 |
+
import importlib
|
| 97 |
+
import importlib.abc
|
| 98 |
+
import importlib.util
|
| 99 |
+
|
| 100 |
+
try:
|
| 101 |
+
mod = importlib.import_module('setuptools._distutils')
|
| 102 |
+
except Exception:
|
| 103 |
+
# There are a couple of cases where setuptools._distutils
|
| 104 |
+
# may not be present:
|
| 105 |
+
# - An older Setuptools without a local distutils is
|
| 106 |
+
# taking precedence. Ref #2957.
|
| 107 |
+
# - Path manipulation during sitecustomize removes
|
| 108 |
+
# setuptools from the path but only after the hook
|
| 109 |
+
# has been loaded. Ref #2980.
|
| 110 |
+
# In either case, fall back to stdlib behavior.
|
| 111 |
+
return None
|
| 112 |
+
|
| 113 |
+
class DistutilsLoader(importlib.abc.Loader):
|
| 114 |
+
def create_module(self, spec):
|
| 115 |
+
mod.__name__ = 'distutils'
|
| 116 |
+
return mod
|
| 117 |
+
|
| 118 |
+
def exec_module(self, module):
|
| 119 |
+
pass
|
| 120 |
+
|
| 121 |
+
return importlib.util.spec_from_loader(
|
| 122 |
+
'distutils', DistutilsLoader(), origin=mod.__file__
|
| 123 |
+
)
|
| 124 |
+
|
| 125 |
+
@staticmethod
|
| 126 |
+
def is_cpython():
|
| 127 |
+
"""
|
| 128 |
+
Suppress supplying distutils for CPython (build and tests).
|
| 129 |
+
Ref #2965 and #3007.
|
| 130 |
+
"""
|
| 131 |
+
return os.path.isfile('pybuilddir.txt')
|
| 132 |
+
|
| 133 |
+
def spec_for_pip(self):
|
| 134 |
+
"""
|
| 135 |
+
Ensure stdlib distutils when running under pip.
|
| 136 |
+
See pypa/pip#8761 for rationale.
|
| 137 |
+
"""
|
| 138 |
+
if sys.version_info >= (3, 12) or self.pip_imported_during_build():
|
| 139 |
+
return
|
| 140 |
+
clear_distutils()
|
| 141 |
+
self.spec_for_distutils = lambda: None
|
| 142 |
+
|
| 143 |
+
@classmethod
|
| 144 |
+
def pip_imported_during_build(cls):
|
| 145 |
+
"""
|
| 146 |
+
Detect if pip is being imported in a build script. Ref #2355.
|
| 147 |
+
"""
|
| 148 |
+
import traceback
|
| 149 |
+
|
| 150 |
+
return any(
|
| 151 |
+
cls.frame_file_is_setup(frame) for frame, line in traceback.walk_stack(None)
|
| 152 |
+
)
|
| 153 |
+
|
| 154 |
+
@staticmethod
|
| 155 |
+
def frame_file_is_setup(frame):
|
| 156 |
+
"""
|
| 157 |
+
Return True if the indicated frame suggests a setup.py file.
|
| 158 |
+
"""
|
| 159 |
+
# some frames may not have __file__ (#2940)
|
| 160 |
+
return frame.f_globals.get('__file__', '').endswith('setup.py')
|
| 161 |
+
|
| 162 |
+
def spec_for_sensitive_tests(self):
|
| 163 |
+
"""
|
| 164 |
+
Ensure stdlib distutils when running select tests under CPython.
|
| 165 |
+
|
| 166 |
+
python/cpython#91169
|
| 167 |
+
"""
|
| 168 |
+
clear_distutils()
|
| 169 |
+
self.spec_for_distutils = lambda: None
|
| 170 |
+
|
| 171 |
+
sensitive_tests = (
|
| 172 |
+
[
|
| 173 |
+
'test.test_distutils',
|
| 174 |
+
'test.test_peg_generator',
|
| 175 |
+
'test.test_importlib',
|
| 176 |
+
]
|
| 177 |
+
if sys.version_info < (3, 10)
|
| 178 |
+
else [
|
| 179 |
+
'test.test_distutils',
|
| 180 |
+
]
|
| 181 |
+
)
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
for name in DistutilsMetaFinder.sensitive_tests:
|
| 185 |
+
setattr(
|
| 186 |
+
DistutilsMetaFinder,
|
| 187 |
+
f'spec_for_{name}',
|
| 188 |
+
DistutilsMetaFinder.spec_for_sensitive_tests,
|
| 189 |
+
)
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
DISTUTILS_FINDER = DistutilsMetaFinder()
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def add_shim():
|
| 196 |
+
DISTUTILS_FINDER in sys.meta_path or insert_shim()
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
class shim:
|
| 200 |
+
def __enter__(self):
|
| 201 |
+
insert_shim()
|
| 202 |
+
|
| 203 |
+
def __exit__(self, exc, value, tb):
|
| 204 |
+
_remove_shim()
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def insert_shim():
|
| 208 |
+
sys.meta_path.insert(0, DISTUTILS_FINDER)
|
| 209 |
+
|
| 210 |
+
|
| 211 |
+
def _remove_shim():
|
| 212 |
+
try:
|
| 213 |
+
sys.meta_path.remove(DISTUTILS_FINDER)
|
| 214 |
+
except ValueError:
|
| 215 |
+
pass
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
if sys.version_info < (3, 12):
|
| 219 |
+
# DistutilsMetaFinder can only be disabled in Python < 3.12 (PEP 632)
|
| 220 |
+
remove_shim = _remove_shim
|
kbidu_env/Lib/site-packages/_distutils_hack/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (10.5 kB). View file
|
|
|