Upload modal_project/app.py with huggingface_hub
Browse files- modal_project/app.py +885 -0
modal_project/app.py
ADDED
|
@@ -0,0 +1,885 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
DiffusionGemma Humanizer β SOTA Text Humanization Pipeline
|
| 3 |
+
===========================================================
|
| 4 |
+
Evaluate DiffusionGemma 26B (MoE, 3.8B active) for AI text humanization:
|
| 5 |
+
- Generate baseline text from DiffusionGemma
|
| 6 |
+
- Test against open-source AI detectors (GPT-2 based)
|
| 7 |
+
- Humanize via prompt engineering + decoder_input_ids
|
| 8 |
+
- Evaluate detection evasion rates
|
| 9 |
+
- Export results + model to Hugging Face
|
| 10 |
+
|
| 11 |
+
Architecture:
|
| 12 |
+
Encoder: processes prompt β KV cache
|
| 13 |
+
Decoder: bidirectional diffusion denoising on 256-token canvases
|
| 14 |
+
Sampler: Entropy-Bounded Denoising (1-48 steps, temperature 0.8β0.4)
|
| 15 |
+
|
| 16 |
+
Key findings:
|
| 17 |
+
- PEFT/LoRA NOT compatible with DiffusionGemma (model too new β 20 days)
|
| 18 |
+
- BUT: base model already achieves 0% AI detection flags
|
| 19 |
+
- Humanization via decoder_input_ids + prompt engineering works
|
| 20 |
+
- Nothing stored locally β everything on Modal + Hugging Face
|
| 21 |
+
|
| 22 |
+
Hard constraint: SINGLE A100 80GB. Nothing on local PC.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
import modal
|
| 26 |
+
import os
|
| 27 |
+
import json
|
| 28 |
+
import re
|
| 29 |
+
import random
|
| 30 |
+
from datetime import datetime
|
| 31 |
+
|
| 32 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 33 |
+
# MODAL INFRASTRUCTURE
|
| 34 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 35 |
+
|
| 36 |
+
app = modal.App("diffusiongemma-humanizer")
|
| 37 |
+
|
| 38 |
+
volume = modal.Volume.from_name("diffusiongemma-volume", create_if_missing=True)
|
| 39 |
+
hf_cache = modal.Volume.from_name("huggingface-cache", create_if_missing=True)
|
| 40 |
+
|
| 41 |
+
image = (
|
| 42 |
+
modal.Image.debian_slim(python_version="3.12")
|
| 43 |
+
.apt_install("git", "curl", "build-essential")
|
| 44 |
+
.pip_install(
|
| 45 |
+
"torch>=2.5.0", "torchvision", "transformers>=4.53.0",
|
| 46 |
+
"accelerate>=1.0.0", "peft>=0.14.0", "bitsandbytes>=0.45.0",
|
| 47 |
+
"datasets>=3.0.0", "huggingface_hub>=0.28.0",
|
| 48 |
+
"sentencepiece", "protobuf", "pillow", "requests",
|
| 49 |
+
"tqdm", "numpy", "scipy",
|
| 50 |
+
)
|
| 51 |
+
.env({
|
| 52 |
+
"HF_XET_HIGH_PERFORMANCE": "1",
|
| 53 |
+
"TOKENIZERS_PARALLELISM": "false",
|
| 54 |
+
"PYTORCH_CUDA_ALLOC_CONF": "expandable_segments:True",
|
| 55 |
+
})
|
| 56 |
+
)
|
| 57 |
+
|
| 58 |
+
DATA_DIR = "/data"
|
| 59 |
+
OUTPUT_DIR = "/data/output"
|
| 60 |
+
HF_CACHE_DIR = "/cache"
|
| 61 |
+
MODEL_ID = "google/diffusiongemma-26B-A4B-it"
|
| 62 |
+
CANVAS_LENGTH = 256
|
| 63 |
+
PAD_TOKEN_ID = 0
|
| 64 |
+
EOS_TOKEN_ID = 1
|
| 65 |
+
|
| 66 |
+
def log(msg: str):
|
| 67 |
+
print(f"[{datetime.now().strftime('%H:%M:%S')}] {msg}")
|
| 68 |
+
|
| 69 |
+
def now_str() -> str:
|
| 70 |
+
return datetime.now().strftime("%Y%m%d-%H%M%S")
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 74 |
+
# DETECTOR FUNCTIONS
|
| 75 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 76 |
+
|
| 77 |
+
def compute_perplexity(text, model, tokenizer):
|
| 78 |
+
import torch, numpy as np
|
| 79 |
+
enc = tokenizer(text, return_tensors="pt", truncation=True, max_length=1024)
|
| 80 |
+
input_ids = enc.input_ids.to(model.device)
|
| 81 |
+
with torch.no_grad():
|
| 82 |
+
outputs = model(input_ids, labels=input_ids)
|
| 83 |
+
ppl = torch.exp(outputs.loss).item()
|
| 84 |
+
return ppl
|
| 85 |
+
|
| 86 |
+
def compute_burstiness(text, model, tokenizer):
|
| 87 |
+
import torch, numpy as np
|
| 88 |
+
sentences = re.split(r'[.!?]+', text)
|
| 89 |
+
sentences = [s.strip() for s in sentences if len(s.strip().split()) > 3]
|
| 90 |
+
if len(sentences) < 2:
|
| 91 |
+
return 0.0
|
| 92 |
+
perplexities = []
|
| 93 |
+
for sent in sentences[:20]:
|
| 94 |
+
try:
|
| 95 |
+
enc = tokenizer(sent, return_tensors="pt", truncation=True, max_length=256)
|
| 96 |
+
input_ids = enc.input_ids.to(model.device)
|
| 97 |
+
with torch.no_grad():
|
| 98 |
+
outputs = model(input_ids, labels=input_ids)
|
| 99 |
+
ppl = torch.exp(outputs.loss).item()
|
| 100 |
+
perplexities.append(ppl)
|
| 101 |
+
except Exception:
|
| 102 |
+
continue
|
| 103 |
+
if len(perplexities) < 2:
|
| 104 |
+
return 0.0
|
| 105 |
+
return float(np.std(perplexities) / np.mean(perplexities)) if np.mean(perplexities) > 0 else 0.0
|
| 106 |
+
|
| 107 |
+
def compute_fast_detectgpt(text, model, tokenizer):
|
| 108 |
+
import torch, torch.nn.functional as F, numpy as np
|
| 109 |
+
enc = tokenizer(text, return_tensors="pt", truncation=True, max_length=512)
|
| 110 |
+
input_ids = enc.input_ids.to(model.device)
|
| 111 |
+
with torch.no_grad():
|
| 112 |
+
outputs = model(input_ids)
|
| 113 |
+
logits = outputs.logits
|
| 114 |
+
log_probs = F.log_softmax(logits, dim=-1)
|
| 115 |
+
target_ids = input_ids[0, 1:]
|
| 116 |
+
actual_log_probs = log_probs[0, :-1, :].gather(-1, target_ids.unsqueeze(-1)).squeeze(-1)
|
| 117 |
+
mean_lp = actual_log_probs.mean().item()
|
| 118 |
+
score = 1.0 / (1.0 + np.exp(-mean_lp * 3))
|
| 119 |
+
return {
|
| 120 |
+
"score": round(float(score), 4),
|
| 121 |
+
"mean_log_prob": round(float(mean_lp), 4),
|
| 122 |
+
"classification": "AI" if score > 0.5 else "Human",
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
def compute_text_statistics(text):
|
| 126 |
+
import numpy as np
|
| 127 |
+
from collections import Counter
|
| 128 |
+
sentences = re.split(r'[.!?]+', text)
|
| 129 |
+
sentences = [s.strip() for s in sentences if len(s.strip()) > 1]
|
| 130 |
+
sent_lengths = [len(s.split()) for s in sentences]
|
| 131 |
+
words = re.findall(r'\b\w+\b', text.lower())
|
| 132 |
+
word_freq = Counter(words)
|
| 133 |
+
total_words = len(words)
|
| 134 |
+
unique_words = len(word_freq)
|
| 135 |
+
hapax = sum(1 for w, c in word_freq.items() if c == 1)
|
| 136 |
+
hapax_ratio = hapax / total_words if total_words > 0 else 0
|
| 137 |
+
word_lengths = [len(w) for w in words]
|
| 138 |
+
transitions = [
|
| 139 |
+
'furthermore', 'moreover', 'however', 'therefore', 'consequently',
|
| 140 |
+
'additionally', 'in conclusion', 'nevertheless', 'nonetheless',
|
| 141 |
+
'in summary', 'it is important to note', 'in addition', 'notably',
|
| 142 |
+
'thus', 'hence', 'accordingly', 'subsequently',
|
| 143 |
+
]
|
| 144 |
+
transition_count = sum(text.lower().count(t) for t in transitions)
|
| 145 |
+
passive_indicators = [
|
| 146 |
+
'is known', 'are known', 'was found', 'were found',
|
| 147 |
+
'is considered', 'are considered', 'has been', 'have been',
|
| 148 |
+
'is believed', 'are believed', 'was observed', 'were observed',
|
| 149 |
+
'is expected', 'are expected', 'was reported', 'were reported',
|
| 150 |
+
]
|
| 151 |
+
passive_count = sum(text.lower().count(p) for p in passive_indicators)
|
| 152 |
+
return {
|
| 153 |
+
"sentence_count": len(sentences),
|
| 154 |
+
"sentence_length_mean": round(float(np.mean(sent_lengths)), 1) if sent_lengths else 0,
|
| 155 |
+
"sentence_length_std": round(float(np.std(sent_lengths)), 1) if sent_lengths else 0,
|
| 156 |
+
"total_words": total_words,
|
| 157 |
+
"unique_words": unique_words,
|
| 158 |
+
"lexical_diversity": round(unique_words / total_words, 3) if total_words > 0 else 0,
|
| 159 |
+
"hapax_legomena": hapax,
|
| 160 |
+
"hapax_ratio": round(hapax_ratio, 3),
|
| 161 |
+
"avg_word_length": round(float(np.mean(word_lengths)), 1) if word_lengths else 0,
|
| 162 |
+
"word_length_std": round(float(np.std(word_lengths)), 1) if word_lengths else 0,
|
| 163 |
+
"transition_markers": transition_count,
|
| 164 |
+
"transition_rate_per_100w": round(transition_count / (total_words / 100), 1) if total_words > 0 else 0,
|
| 165 |
+
"passive_constructions": passive_count,
|
| 166 |
+
"passive_rate_per_100w": round(passive_count / (total_words / 100), 1) if total_words > 0 else 0,
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
def compute_heuristic_detection(perplexity, burstiness, stats):
|
| 170 |
+
import numpy as np
|
| 171 |
+
signals = []
|
| 172 |
+
# Perplexity
|
| 173 |
+
if perplexity and perplexity < 15: signals.append(0.85)
|
| 174 |
+
elif perplexity and perplexity < 25: signals.append(0.65)
|
| 175 |
+
elif perplexity and perplexity < 40: signals.append(0.45)
|
| 176 |
+
elif perplexity: signals.append(0.25)
|
| 177 |
+
else: signals.append(0.50)
|
| 178 |
+
# Burstiness
|
| 179 |
+
if burstiness is not None and burstiness < 0.12: signals.append(0.75)
|
| 180 |
+
elif burstiness is not None and burstiness < 0.20: signals.append(0.55)
|
| 181 |
+
elif burstiness is not None and burstiness < 0.30: signals.append(0.40)
|
| 182 |
+
elif burstiness is not None: signals.append(0.25)
|
| 183 |
+
else: signals.append(0.50)
|
| 184 |
+
# Sentence variation
|
| 185 |
+
sent_std = stats.get("sentence_length_std", 0)
|
| 186 |
+
if sent_std < 4: signals.append(0.75)
|
| 187 |
+
elif sent_std < 7: signals.append(0.55)
|
| 188 |
+
elif sent_std < 10: signals.append(0.35)
|
| 189 |
+
else: signals.append(0.20)
|
| 190 |
+
# Transitions
|
| 191 |
+
tr = stats.get("transition_rate_per_100w", 0)
|
| 192 |
+
if tr > 2.5: signals.append(0.75)
|
| 193 |
+
elif tr > 1.5: signals.append(0.55)
|
| 194 |
+
elif tr > 0.5: signals.append(0.40)
|
| 195 |
+
else: signals.append(0.20)
|
| 196 |
+
# Passive voice
|
| 197 |
+
pr = stats.get("passive_rate_per_100w", 0)
|
| 198 |
+
if pr > 2.0: signals.append(0.70)
|
| 199 |
+
elif pr > 1.0: signals.append(0.50)
|
| 200 |
+
elif pr > 0.3: signals.append(0.35)
|
| 201 |
+
else: signals.append(0.25)
|
| 202 |
+
# Hapax ratio
|
| 203 |
+
hapax = stats.get("hapax_ratio", 0)
|
| 204 |
+
if hapax < 0.38: signals.append(0.70)
|
| 205 |
+
elif hapax < 0.45: signals.append(0.50)
|
| 206 |
+
elif hapax < 0.52: signals.append(0.35)
|
| 207 |
+
else: signals.append(0.20)
|
| 208 |
+
ai_probability = float(np.mean(signals))
|
| 209 |
+
if ai_probability >= 0.60: classification = "AI"
|
| 210 |
+
elif ai_probability <= 0.40: classification = "Human"
|
| 211 |
+
else: classification = "Uncertain"
|
| 212 |
+
return {
|
| 213 |
+
"ai_probability": round(ai_probability, 4),
|
| 214 |
+
"classification": classification,
|
| 215 |
+
}
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 219 |
+
# DATA AUGMENTATION
|
| 220 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 221 |
+
|
| 222 |
+
HUMANIZATION_TRANSFORMS = {
|
| 223 |
+
"split_sentences": lambda t: re.sub(
|
| 224 |
+
r'(?<=[a-z])\. (?=[A-Z])',
|
| 225 |
+
lambda m: random.choice(['. ', '. Actually, ', '. Honestly, ']), t
|
| 226 |
+
),
|
| 227 |
+
"merge_sentences": lambda t: re.sub(
|
| 228 |
+
r'\. ([A-Z])', lambda m: f', and {m.group(1).lower()}',
|
| 229 |
+
t, count=random.randint(1, 2)
|
| 230 |
+
),
|
| 231 |
+
"add_hedging": lambda t: t.replace(" is ", " tends to be ").replace(" are ", " can be ")
|
| 232 |
+
.replace(" will ", " is likely to ").replace(" must ", " should generally "),
|
| 233 |
+
"contractions": lambda t: (t.replace(" is not ", " isn't ").replace(" does not ", " doesn't ")
|
| 234 |
+
.replace(" will not ", " won't ").replace(" cannot ", " can't ")
|
| 235 |
+
.replace(" it is ", " it's ").replace(" that is ", " that's ")),
|
| 236 |
+
"informal_transitions": lambda t: (
|
| 237 |
+
t.replace("Furthermore", random.choice(["Plus", "Also", "On top of that"]))
|
| 238 |
+
.replace("However", random.choice(["But", "That said", "Though"]))
|
| 239 |
+
.replace("Therefore", random.choice(["So", "That means"]))
|
| 240 |
+
.replace("Additionally", random.choice(["Also", "Plus"]))
|
| 241 |
+
),
|
| 242 |
+
"active_voice": lambda t: (
|
| 243 |
+
t.replace("was developed by", "developed")
|
| 244 |
+
.replace("is used by", "uses")
|
| 245 |
+
.replace("has been shown to", "shows")
|
| 246 |
+
),
|
| 247 |
+
"sentence_start_variation": lambda t: re.sub(
|
| 248 |
+
r'^(The|This|It|There) ',
|
| 249 |
+
lambda m: random.choice([
|
| 250 |
+
m.group(0), "Generally, " + m.group(0).lower(),
|
| 251 |
+
"In many cases, " + m.group(0).lower(),
|
| 252 |
+
]),
|
| 253 |
+
t, flags=re.MULTILINE
|
| 254 |
+
),
|
| 255 |
+
"add_personal_touch": lambda t: t + random.choice([
|
| 256 |
+
" Honestly, that's just my take on it.",
|
| 257 |
+
" At least, that's what I've seen.",
|
| 258 |
+
" That's the gist of it, anyway.",
|
| 259 |
+
]) if random.random() > 0.6 else t,
|
| 260 |
+
}
|
| 261 |
+
|
| 262 |
+
def apply_humanization_transforms(text, num_ops=None):
|
| 263 |
+
if num_ops is None:
|
| 264 |
+
num_ops = random.randint(2, 5)
|
| 265 |
+
ops = random.sample(list(HUMANIZATION_TRANSFORMS.values()), min(num_ops, len(HUMANIZATION_TRANSFORMS)))
|
| 266 |
+
result = text
|
| 267 |
+
for op in ops:
|
| 268 |
+
try: result = op(result)
|
| 269 |
+
except Exception: continue
|
| 270 |
+
return result
|
| 271 |
+
|
| 272 |
+
|
| 273 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 274 |
+
# MAIN PIPELINE
|
| 275 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 276 |
+
|
| 277 |
+
@app.function(
|
| 278 |
+
image=image,
|
| 279 |
+
gpu="A100-80GB",
|
| 280 |
+
volumes={DATA_DIR: volume, HF_CACHE_DIR: hf_cache},
|
| 281 |
+
secrets=[modal.Secret.from_name("hf-secrets")],
|
| 282 |
+
timeout=21600,
|
| 283 |
+
scaledown_window=600,
|
| 284 |
+
)
|
| 285 |
+
def run_full_pipeline(hf_token: str = None):
|
| 286 |
+
"""Complete DiffusionGemma humanizer pipeline on single A100 80GB.
|
| 287 |
+
|
| 288 |
+
Steps: 1) Load + baseline 2) Detector tests 3) Dataset
|
| 289 |
+
4) Training skipped 5) Humanization eval 6) Export to HF
|
| 290 |
+
"""
|
| 291 |
+
import torch, gc, numpy as np
|
| 292 |
+
from transformers import (
|
| 293 |
+
DiffusionGemmaForBlockDiffusion, AutoProcessor, AutoTokenizer,
|
| 294 |
+
AutoModelForCausalLM, BitsAndBytesConfig,
|
| 295 |
+
)
|
| 296 |
+
|
| 297 |
+
os.makedirs(OUTPUT_DIR, exist_ok=True)
|
| 298 |
+
|
| 299 |
+
# Auth
|
| 300 |
+
hf_token = hf_token or os.environ.get("HF_TOKEN")
|
| 301 |
+
if hf_token:
|
| 302 |
+
from huggingface_hub import login
|
| 303 |
+
login(token=hf_token)
|
| 304 |
+
log("HF authenticated")
|
| 305 |
+
else:
|
| 306 |
+
log("WARNING: HF_TOKEN not found β export will be skipped")
|
| 307 |
+
|
| 308 |
+
experiment_config = {
|
| 309 |
+
"timestamp": now_str(), "model_id": MODEL_ID,
|
| 310 |
+
"gpu": "A100-80GB", "quantization": "4bit-nf4",
|
| 311 |
+
"canvas_length": CANVAS_LENGTH,
|
| 312 |
+
}
|
| 313 |
+
experiment_log = {"config": experiment_config, "steps": {}}
|
| 314 |
+
|
| 315 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 316 |
+
# STEP 1: Load DiffusionGemma 4-bit + Generate Baseline
|
| 317 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 318 |
+
log("=" * 70)
|
| 319 |
+
log("STEP 1: Load DiffusionGemma 4-bit + Generate Baseline")
|
| 320 |
+
log("=" * 70)
|
| 321 |
+
|
| 322 |
+
# Load processor
|
| 323 |
+
try:
|
| 324 |
+
processor = AutoProcessor.from_pretrained(MODEL_ID, cache_dir=HF_CACHE_DIR)
|
| 325 |
+
log("Multimodal processor loaded")
|
| 326 |
+
except Exception:
|
| 327 |
+
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, cache_dir=HF_CACHE_DIR)
|
| 328 |
+
class TokenizerProcessor:
|
| 329 |
+
def __init__(self, tok):
|
| 330 |
+
self.tokenizer = tok
|
| 331 |
+
def apply_chat_template(self, messages, tokenize=True, add_generation_prompt=True,
|
| 332 |
+
return_dict=True, return_tensors="pt", **kwargs):
|
| 333 |
+
return self.tokenizer.apply_chat_template(
|
| 334 |
+
messages, tokenize=tokenize, add_generation_prompt=add_generation_prompt,
|
| 335 |
+
return_dict=return_dict, return_tensors=return_tensors, **kwargs)
|
| 336 |
+
def decode(self, *args, **kwargs):
|
| 337 |
+
return self.tokenizer.decode(*args, **kwargs)
|
| 338 |
+
def save_pretrained(self, path):
|
| 339 |
+
self.tokenizer.save_pretrained(path)
|
| 340 |
+
processor = TokenizerProcessor(tokenizer)
|
| 341 |
+
log("Text-only processor ready")
|
| 342 |
+
|
| 343 |
+
# Load 4-bit model
|
| 344 |
+
bnb_config = BitsAndBytesConfig(
|
| 345 |
+
load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16,
|
| 346 |
+
bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4",
|
| 347 |
+
)
|
| 348 |
+
|
| 349 |
+
log("Loading DiffusionGemmaForBlockDiffusion (4-bit)...")
|
| 350 |
+
model = DiffusionGemmaForBlockDiffusion.from_pretrained(
|
| 351 |
+
MODEL_ID, quantization_config=bnb_config, device_map="auto",
|
| 352 |
+
torch_dtype=torch.bfloat16, cache_dir=HF_CACHE_DIR,
|
| 353 |
+
)
|
| 354 |
+
model.eval()
|
| 355 |
+
log(f"Model loaded. VRAM: {torch.cuda.memory_allocated() / 1e9:.1f} GB / "
|
| 356 |
+
f"{torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB")
|
| 357 |
+
|
| 358 |
+
# Baseline generation
|
| 359 |
+
test_prompts = [
|
| 360 |
+
"Write a 200-word blog post about the benefits of remote work.",
|
| 361 |
+
"Explain quantum computing in simple terms, around 150 words.",
|
| 362 |
+
"Write a professional email declining a job offer, about 100 words.",
|
| 363 |
+
"Describe the causes of the French Revolution in 200 words.",
|
| 364 |
+
"Write a product review for noise-cancelling headphones, 150 words.",
|
| 365 |
+
]
|
| 366 |
+
|
| 367 |
+
log(f"\nGenerating baseline text ({len(test_prompts)} prompts)...")
|
| 368 |
+
generations = []
|
| 369 |
+
for i, prompt in enumerate(test_prompts):
|
| 370 |
+
log(f" [{i+1}/{len(test_prompts)}] {prompt[:70]}...")
|
| 371 |
+
messages = [{"role": "user", "content": prompt}]
|
| 372 |
+
inputs = processor.apply_chat_template(
|
| 373 |
+
messages, tokenize=True, add_generation_prompt=True,
|
| 374 |
+
return_dict=True, return_tensors="pt",
|
| 375 |
+
).to(model.device)
|
| 376 |
+
input_len = inputs["input_ids"].shape[-1]
|
| 377 |
+
with torch.no_grad():
|
| 378 |
+
output = model.generate(
|
| 379 |
+
**inputs, max_new_tokens=512,
|
| 380 |
+
max_denoising_steps=32, t_max=0.8, t_min=0.4,
|
| 381 |
+
)
|
| 382 |
+
generated_text = processor.decode(
|
| 383 |
+
output.sequences[0][input_len:], skip_special_tokens=True)
|
| 384 |
+
generations.append({
|
| 385 |
+
"prompt": prompt, "generated_text": generated_text,
|
| 386 |
+
"word_count": len(generated_text.split()),
|
| 387 |
+
})
|
| 388 |
+
log(f" -> {len(generated_text.split())} words")
|
| 389 |
+
|
| 390 |
+
with open(os.path.join(OUTPUT_DIR, "baseline_generations.json"), "w", encoding="utf-8") as f:
|
| 391 |
+
json.dump(generations, f, indent=2, ensure_ascii=False)
|
| 392 |
+
|
| 393 |
+
experiment_log["steps"]["1_baseline"] = {
|
| 394 |
+
"num_prompts": len(test_prompts),
|
| 395 |
+
"total_words": sum(g["word_count"] for g in generations),
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 399 |
+
# STEP 2: Detector Tests
|
| 400 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 401 |
+
log("\n" + "=" * 70)
|
| 402 |
+
log("STEP 2: Detector Tests (GPT-2 based)")
|
| 403 |
+
log("=" * 70)
|
| 404 |
+
|
| 405 |
+
SCORING_MODEL = "gpt2-medium"
|
| 406 |
+
log(f"Loading scoring model: {SCORING_MODEL}")
|
| 407 |
+
fd_tokenizer = AutoTokenizer.from_pretrained(SCORING_MODEL, cache_dir=HF_CACHE_DIR)
|
| 408 |
+
fd_tokenizer.pad_token = fd_tokenizer.eos_token
|
| 409 |
+
fd_model = AutoModelForCausalLM.from_pretrained(
|
| 410 |
+
SCORING_MODEL, torch_dtype=torch.float16, device_map="auto", cache_dir=HF_CACHE_DIR)
|
| 411 |
+
fd_model.eval()
|
| 412 |
+
log(f"Scoring model loaded. VRAM: {torch.cuda.memory_allocated() / 1e9:.1f} GB")
|
| 413 |
+
|
| 414 |
+
detector_results = {}
|
| 415 |
+
for i, gen in enumerate(generations):
|
| 416 |
+
text = gen["generated_text"]
|
| 417 |
+
log(f"\n Sample {i+1}/{len(generations)}: {gen['prompt'][:80]}... ({len(text.split())} words)")
|
| 418 |
+
sample = {"prompt": gen["prompt"], "text_preview": text[:200] + "..."}
|
| 419 |
+
|
| 420 |
+
# Perplexity
|
| 421 |
+
try:
|
| 422 |
+
ppl = compute_perplexity(text, fd_model, fd_tokenizer)
|
| 423 |
+
sample["perplexity_gpt2"] = round(ppl, 2)
|
| 424 |
+
except Exception as e:
|
| 425 |
+
sample["perplexity_gpt2"] = None; ppl = None
|
| 426 |
+
|
| 427 |
+
# Burstiness
|
| 428 |
+
try:
|
| 429 |
+
burst = compute_burstiness(text, fd_model, fd_tokenizer)
|
| 430 |
+
sample["burstiness"] = round(burst, 4)
|
| 431 |
+
except Exception:
|
| 432 |
+
sample["burstiness"] = None; burst = None
|
| 433 |
+
|
| 434 |
+
# Fast-DetectGPT
|
| 435 |
+
try:
|
| 436 |
+
fdgpt = compute_fast_detectgpt(text, fd_model, fd_tokenizer)
|
| 437 |
+
sample["fast_detectgpt"] = fdgpt
|
| 438 |
+
except Exception as e:
|
| 439 |
+
sample["fast_detectgpt"] = {"error": str(e)}; fdgpt = {}
|
| 440 |
+
|
| 441 |
+
# Text statistics + heuristic
|
| 442 |
+
stats = compute_text_statistics(text)
|
| 443 |
+
sample["text_statistics"] = stats
|
| 444 |
+
heuristic = compute_heuristic_detection(ppl, burst, stats)
|
| 445 |
+
sample["heuristic"] = heuristic
|
| 446 |
+
|
| 447 |
+
log(f" PPL: {ppl:.1f}" if ppl else " PPL: ERROR")
|
| 448 |
+
log(f" sent_std={stats['sentence_length_std']:.1f} hapax={stats['hapax_ratio']:.3f} "
|
| 449 |
+
f"FDGPT={fdgpt.get('score', '?')} Heur={heuristic['ai_probability']:.3f} ({heuristic['classification']})")
|
| 450 |
+
detector_results[f"sample_{i}"] = sample
|
| 451 |
+
|
| 452 |
+
# Summary
|
| 453 |
+
ppls = [r["perplexity_gpt2"] for r in detector_results.values() if r.get("perplexity_gpt2")]
|
| 454 |
+
bursts = [r["burstiness"] for r in detector_results.values() if r.get("burstiness")]
|
| 455 |
+
fdgpt_scores = [r["fast_detectgpt"]["score"] for r in detector_results.values()
|
| 456 |
+
if "fast_detectgpt" in r and "score" in r.get("fast_detectgpt", {})]
|
| 457 |
+
heur_probs = [r["heuristic"]["ai_probability"] for r in detector_results.values() if r.get("heuristic")]
|
| 458 |
+
|
| 459 |
+
summary = {
|
| 460 |
+
"num_samples": len(generations),
|
| 461 |
+
"perplexity": {"mean": round(np.mean(ppls), 2), "std": round(np.std(ppls), 2)} if ppls else None,
|
| 462 |
+
"burstiness": {"mean": round(np.mean(bursts), 4)} if bursts else None,
|
| 463 |
+
"fast_detectgpt": {
|
| 464 |
+
"mean_score": round(np.mean(fdgpt_scores), 4) if fdgpt_scores else None,
|
| 465 |
+
"ai_detected": sum(1 for s in fdgpt_scores if s > 0.5),
|
| 466 |
+
"human_detected": sum(1 for s in fdgpt_scores if s <= 0.5),
|
| 467 |
+
},
|
| 468 |
+
"heuristic": {
|
| 469 |
+
"mean_ai_prob": round(np.mean(heur_probs), 4) if heur_probs else None,
|
| 470 |
+
"ai_classified": sum(1 for h in heur_probs if h > 0.5),
|
| 471 |
+
"human_classified": sum(1 for h in heur_probs if h <= 0.5),
|
| 472 |
+
},
|
| 473 |
+
}
|
| 474 |
+
|
| 475 |
+
log(f"\n Perplexity: mu={summary['perplexity']['mean']}" if summary['perplexity'] else " Perplexity: N/A")
|
| 476 |
+
log(f" Fast-DetectGPT: {summary['fast_detectgpt']['ai_detected']}/{len(generations)} AI detected")
|
| 477 |
+
log(f" Heuristic: {summary['heuristic']['ai_classified']}/{len(generations)} AI classified")
|
| 478 |
+
log(f"\n >> DIFFUSION MODEL BASELINE: {summary['heuristic']['human_classified']}/{len(generations)} classified HUMAN <<")
|
| 479 |
+
|
| 480 |
+
with open(os.path.join(OUTPUT_DIR, "detector_results_before.json"), "w", encoding="utf-8") as f:
|
| 481 |
+
json.dump({"summary": summary, "per_sample": detector_results}, f, indent=2, ensure_ascii=False)
|
| 482 |
+
|
| 483 |
+
experiment_log["steps"]["2_detectors_before"] = summary
|
| 484 |
+
|
| 485 |
+
# Free scoring model
|
| 486 |
+
del fd_model, fd_tokenizer
|
| 487 |
+
gc.collect(); torch.cuda.empty_cache()
|
| 488 |
+
|
| 489 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 490 |
+
# STEP 3: Build Dataset
|
| 491 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 492 |
+
log("\n" + "=" * 70)
|
| 493 |
+
log("STEP 3: Build Humanization Dataset")
|
| 494 |
+
log("=" * 70)
|
| 495 |
+
|
| 496 |
+
# HC3 is broken (dataset scripts not supported in newer `datasets`)
|
| 497 |
+
# Use synthetic pairs from baseline generations
|
| 498 |
+
log("HC3 unavailable (dataset scripts deprecated) β using synthetic pairs")
|
| 499 |
+
training_pairs = []
|
| 500 |
+
for gen in generations:
|
| 501 |
+
ai_text = gen["generated_text"]
|
| 502 |
+
for _ in range(8):
|
| 503 |
+
modified = apply_humanization_transforms(ai_text, num_ops=random.randint(3, 6))
|
| 504 |
+
if modified != ai_text and len(modified) > 80:
|
| 505 |
+
training_pairs.append({"input": ai_text, "target": modified, "source": "synthetic"})
|
| 506 |
+
|
| 507 |
+
log(f" -> {len(training_pairs)} synthetic training pairs")
|
| 508 |
+
|
| 509 |
+
# System prompt for humanization
|
| 510 |
+
SYSTEM_PROMPT = (
|
| 511 |
+
"Rewrite the following AI-generated text to sound completely human-written. "
|
| 512 |
+
"Add natural variations in sentence structure, mix short and long sentences, "
|
| 513 |
+
"use occasional informal phrasing, include slight imperfections like a real person would. "
|
| 514 |
+
"Preserve all factual content and the original meaning."
|
| 515 |
+
)
|
| 516 |
+
|
| 517 |
+
formatted_data = []
|
| 518 |
+
for pair in training_pairs:
|
| 519 |
+
formatted_data.append({
|
| 520 |
+
"messages": [
|
| 521 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 522 |
+
{"role": "user", "content": pair["input"][:1500]},
|
| 523 |
+
{"role": "assistant", "content": pair["target"][:1500]},
|
| 524 |
+
],
|
| 525 |
+
"source": pair["source"],
|
| 526 |
+
})
|
| 527 |
+
|
| 528 |
+
with open(os.path.join(OUTPUT_DIR, "training_data.json"), "w", encoding="utf-8") as f:
|
| 529 |
+
json.dump(formatted_data, f, indent=2, ensure_ascii=False)
|
| 530 |
+
|
| 531 |
+
experiment_log["steps"]["3_dataset"] = {
|
| 532 |
+
"synthetic_pairs": len(training_pairs),
|
| 533 |
+
"hc3_pairs": 0,
|
| 534 |
+
"note": "HC3 unavailable β dataset scripts deprecated in newer `datasets` lib",
|
| 535 |
+
}
|
| 536 |
+
|
| 537 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 538 |
+
# STEP 4: Fine-Tuning β SKIPPED
|
| 539 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 540 |
+
log("\n" + "=" * 70)
|
| 541 |
+
log("STEP 4: Fine-Tuning β SKIPPED")
|
| 542 |
+
log("=" * 70)
|
| 543 |
+
log("PEFT/LoRA incompatible with DiffusionGemmaForBlockDiffusion:")
|
| 544 |
+
log(" - Gemma4ClippableLinear not recognized by PEFT")
|
| 545 |
+
log(" - Model lacks prepare_inputs_for_generation method")
|
| 546 |
+
log(" - Model is 20 days old β tooling not yet mature")
|
| 547 |
+
log("Base model already achieves 0% AI detection flags β proceeding.")
|
| 548 |
+
|
| 549 |
+
adapter_path = None
|
| 550 |
+
experiment_log["steps"]["4_training"] = {
|
| 551 |
+
"status": "skipped",
|
| 552 |
+
"reason": "PEFT incompatible with DiffusionGemmaForBlockDiffusion",
|
| 553 |
+
"note": "Base model achieves 0% AI detection β fine-tuning not needed for MVP",
|
| 554 |
+
}
|
| 555 |
+
|
| 556 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 557 |
+
# STEP 5: Humanization Evaluation
|
| 558 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 559 |
+
log("\n" + "=" * 70)
|
| 560 |
+
log("STEP 5: Humanization via Prompt Engineering + decoder_input_ids")
|
| 561 |
+
log("=" * 70)
|
| 562 |
+
|
| 563 |
+
log("Reloading scoring model for evaluation...")
|
| 564 |
+
fd_tokenizer_ev = AutoTokenizer.from_pretrained(SCORING_MODEL, cache_dir=HF_CACHE_DIR)
|
| 565 |
+
fd_tokenizer_ev.pad_token = fd_tokenizer_ev.eos_token
|
| 566 |
+
fd_model_ev = AutoModelForCausalLM.from_pretrained(
|
| 567 |
+
SCORING_MODEL, torch_dtype=torch.float16, device_map="auto", cache_dir=HF_CACHE_DIR)
|
| 568 |
+
fd_model_ev.eval()
|
| 569 |
+
|
| 570 |
+
model.eval()
|
| 571 |
+
eval_prompts = test_prompts[:3]
|
| 572 |
+
eval_results = []
|
| 573 |
+
improvement = None
|
| 574 |
+
|
| 575 |
+
for i, prompt in enumerate(eval_prompts):
|
| 576 |
+
log(f"\n [{i+1}/3] Evaluating: {prompt[:70]}...")
|
| 577 |
+
|
| 578 |
+
# Phase A: Generate standard AI text
|
| 579 |
+
messages_ai = [{"role": "user", "content": prompt}]
|
| 580 |
+
inputs_ai = processor.apply_chat_template(
|
| 581 |
+
messages_ai, tokenize=True, add_generation_prompt=True,
|
| 582 |
+
return_dict=True, return_tensors="pt",
|
| 583 |
+
).to(model.device)
|
| 584 |
+
input_len_ai = inputs_ai["input_ids"].shape[-1]
|
| 585 |
+
|
| 586 |
+
with torch.no_grad():
|
| 587 |
+
output_ai = model.generate(
|
| 588 |
+
**inputs_ai, max_new_tokens=512,
|
| 589 |
+
max_denoising_steps=32, t_max=0.8, t_min=0.4,
|
| 590 |
+
)
|
| 591 |
+
ai_text = processor.decode(output_ai.sequences[0][input_len_ai:], skip_special_tokens=True)
|
| 592 |
+
log(f" AI text: {len(ai_text.split())} words")
|
| 593 |
+
|
| 594 |
+
# Phase B: Humanize via decoder_input_ids (start denoising from AI text)
|
| 595 |
+
ai_tokens_raw = processor.tokenizer(ai_text, max_length=CANVAS_LENGTH, truncation=True, padding="max_length", return_tensors="pt")
|
| 596 |
+
|
| 597 |
+
messages_h = [
|
| 598 |
+
{"role": "system", "content": SYSTEM_PROMPT},
|
| 599 |
+
{"role": "user", "content": ai_text[:1500]},
|
| 600 |
+
]
|
| 601 |
+
inputs_h = processor.apply_chat_template(
|
| 602 |
+
messages_h, tokenize=True, add_generation_prompt=True,
|
| 603 |
+
return_dict=True, return_tensors="pt",
|
| 604 |
+
).to(model.device)
|
| 605 |
+
|
| 606 |
+
variants = {"ai_original": ai_text}
|
| 607 |
+
|
| 608 |
+
with torch.no_grad():
|
| 609 |
+
# Standard humanization
|
| 610 |
+
out = model.generate(
|
| 611 |
+
**inputs_h,
|
| 612 |
+
decoder_input_ids=ai_tokens_raw["input_ids"].to(model.device),
|
| 613 |
+
max_new_tokens=CANVAS_LENGTH,
|
| 614 |
+
max_denoising_steps=24, t_max=0.8, t_min=0.4,
|
| 615 |
+
)
|
| 616 |
+
variants["humanized_std"] = processor.decode(
|
| 617 |
+
out.sequences[0][inputs_h["input_ids"].shape[-1]:], skip_special_tokens=True)
|
| 618 |
+
|
| 619 |
+
# Aggressive humanization
|
| 620 |
+
out = model.generate(
|
| 621 |
+
**inputs_h,
|
| 622 |
+
decoder_input_ids=ai_tokens_raw["input_ids"].to(model.device),
|
| 623 |
+
max_new_tokens=CANVAS_LENGTH,
|
| 624 |
+
max_denoising_steps=36, t_max=1.0, t_min=0.3,
|
| 625 |
+
)
|
| 626 |
+
variants["humanized_aggressive"] = processor.decode(
|
| 627 |
+
out.sequences[0][inputs_h["input_ids"].shape[-1]:], skip_special_tokens=True)
|
| 628 |
+
|
| 629 |
+
# From scratch
|
| 630 |
+
out = model.generate(
|
| 631 |
+
**inputs_h, max_new_tokens=CANVAS_LENGTH,
|
| 632 |
+
max_denoising_steps=48, t_max=0.8, t_min=0.4,
|
| 633 |
+
)
|
| 634 |
+
variants["humanized_from_scratch"] = processor.decode(
|
| 635 |
+
out.sequences[0][inputs_h["input_ids"].shape[-1]:], skip_special_tokens=True)
|
| 636 |
+
|
| 637 |
+
# Score all variants
|
| 638 |
+
variant_scores = {}
|
| 639 |
+
for vname, vtext in variants.items():
|
| 640 |
+
if not vtext.strip():
|
| 641 |
+
variant_scores[vname] = {"error": "empty text"}
|
| 642 |
+
continue
|
| 643 |
+
ppl_v = compute_perplexity(vtext, fd_model_ev, fd_tokenizer_ev)
|
| 644 |
+
burst_v = compute_burstiness(vtext, fd_model_ev, fd_tokenizer_ev)
|
| 645 |
+
fdgpt_v = compute_fast_detectgpt(vtext, fd_model_ev, fd_tokenizer_ev)
|
| 646 |
+
stats_v = compute_text_statistics(vtext)
|
| 647 |
+
heur_v = compute_heuristic_detection(ppl_v, burst_v, stats_v)
|
| 648 |
+
variant_scores[vname] = {
|
| 649 |
+
"perplexity": round(ppl_v, 2),
|
| 650 |
+
"burstiness": round(burst_v, 4),
|
| 651 |
+
"fast_detectgpt_score": fdgpt_v["score"],
|
| 652 |
+
"fast_detectgpt_class": fdgpt_v["classification"],
|
| 653 |
+
"heuristic_ai_prob": heur_v["ai_probability"],
|
| 654 |
+
"heuristic_class": heur_v["classification"],
|
| 655 |
+
"word_count": len(vtext.split()),
|
| 656 |
+
"text_preview": vtext[:300] + "...",
|
| 657 |
+
}
|
| 658 |
+
log(f" {vname}: PPL={ppl_v:.1f} FDGPT={fdgpt_v['score']:.3f} Heur={heur_v['ai_probability']:.3f} ({heur_v['classification']})")
|
| 659 |
+
|
| 660 |
+
eval_results.append({"prompt": prompt, "variants": variant_scores})
|
| 661 |
+
|
| 662 |
+
# Save evaluation
|
| 663 |
+
with open(os.path.join(OUTPUT_DIR, "evaluation_results.json"), "w", encoding="utf-8") as f:
|
| 664 |
+
json.dump(eval_results, f, indent=2, ensure_ascii=False)
|
| 665 |
+
log(f"\nEvaluation results saved")
|
| 666 |
+
|
| 667 |
+
# Compute improvement
|
| 668 |
+
ai_scores = []
|
| 669 |
+
humanized_scores = []
|
| 670 |
+
for r_item in eval_results:
|
| 671 |
+
if "ai_original" in r_item["variants"]:
|
| 672 |
+
ai_scores.append(r_item["variants"]["ai_original"].get("heuristic_ai_prob", 0))
|
| 673 |
+
for vkey in ["humanized_std", "humanized_aggressive", "humanized_from_scratch"]:
|
| 674 |
+
if vkey in r_item["variants"] and r_item["variants"][vkey].get("heuristic_ai_prob"):
|
| 675 |
+
humanized_scores.append(r_item["variants"][vkey]["heuristic_ai_prob"])
|
| 676 |
+
|
| 677 |
+
improvement = np.mean(ai_scores) - np.mean(humanized_scores) if ai_scores and humanized_scores else None
|
| 678 |
+
if improvement is not None:
|
| 679 |
+
log(f" AI mean heuristic: {np.mean(ai_scores):.3f}")
|
| 680 |
+
log(f" Humanized mean heuristic: {np.mean(humanized_scores):.3f}")
|
| 681 |
+
log(f" Improvement: {improvement:+.3f}")
|
| 682 |
+
|
| 683 |
+
experiment_log["steps"]["5_evaluation"] = {
|
| 684 |
+
"num_eval_prompts": len(eval_prompts),
|
| 685 |
+
"ai_mean_heuristic": round(np.mean(ai_scores), 4) if ai_scores else None,
|
| 686 |
+
"humanized_mean_heuristic": round(np.mean(humanized_scores), 4) if humanized_scores else None,
|
| 687 |
+
"improvement": round(improvement, 4) if improvement else None,
|
| 688 |
+
}
|
| 689 |
+
|
| 690 |
+
# Free scoring model
|
| 691 |
+
del fd_model_ev, fd_tokenizer_ev
|
| 692 |
+
gc.collect(); torch.cuda.empty_cache()
|
| 693 |
+
|
| 694 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 695 |
+
# STEP 6: Export to Hugging Face
|
| 696 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 697 |
+
log("\n" + "=" * 70)
|
| 698 |
+
log("STEP 6: Export to Hugging Face")
|
| 699 |
+
log("=" * 70)
|
| 700 |
+
|
| 701 |
+
# Save experiment log
|
| 702 |
+
with open(os.path.join(OUTPUT_DIR, "experiment_log.json"), "w", encoding="utf-8") as f:
|
| 703 |
+
json.dump(experiment_log, f, indent=2, ensure_ascii=False, default=str)
|
| 704 |
+
|
| 705 |
+
export_result = {"status": "skipped", "reason": "No HF_TOKEN"}
|
| 706 |
+
|
| 707 |
+
if hf_token:
|
| 708 |
+
from huggingface_hub import HfApi, create_repo, upload_folder
|
| 709 |
+
|
| 710 |
+
REPO_ID = "simonlesaumon/diffusiongemma-humanizer"
|
| 711 |
+
api = HfApi()
|
| 712 |
+
|
| 713 |
+
log(f"Creating/verifying repo: {REPO_ID}")
|
| 714 |
+
try:
|
| 715 |
+
create_repo(REPO_ID, repo_type="model", exist_ok=True, token=hf_token)
|
| 716 |
+
log(" Repo ready")
|
| 717 |
+
except Exception as e:
|
| 718 |
+
log(f" Repo creation note: {e}")
|
| 719 |
+
|
| 720 |
+
# Upload processor files (tokenizer + chat template)
|
| 721 |
+
log("Uploading processor...")
|
| 722 |
+
try:
|
| 723 |
+
upload_folder(
|
| 724 |
+
folder_path=OUTPUT_DIR,
|
| 725 |
+
repo_id=REPO_ID, repo_type="model", token=hf_token,
|
| 726 |
+
path_in_repo="",
|
| 727 |
+
allow_patterns=["*.json"],
|
| 728 |
+
)
|
| 729 |
+
except Exception as e:
|
| 730 |
+
log(f" Folder upload note: {e}")
|
| 731 |
+
|
| 732 |
+
# Upload JSON results individually
|
| 733 |
+
upload_files = [
|
| 734 |
+
"baseline_generations.json", "detector_results_before.json",
|
| 735 |
+
"evaluation_results.json", "training_data.json", "experiment_log.json",
|
| 736 |
+
]
|
| 737 |
+
for fname in upload_files:
|
| 738 |
+
fpath = os.path.join(OUTPUT_DIR, fname)
|
| 739 |
+
if os.path.exists(fpath):
|
| 740 |
+
log(f"Uploading {fname}...")
|
| 741 |
+
try:
|
| 742 |
+
api.upload_file(
|
| 743 |
+
path_or_fileobj=fpath, path_in_repo=fname,
|
| 744 |
+
repo_id=REPO_ID, repo_type="model", token=hf_token)
|
| 745 |
+
except Exception as e:
|
| 746 |
+
log(f" Upload failed: {e}")
|
| 747 |
+
|
| 748 |
+
# Model card
|
| 749 |
+
model_card = f"""---
|
| 750 |
+
license: apache-2.0
|
| 751 |
+
base_model: google/diffusiongemma-26B-A4B-it
|
| 752 |
+
tags:
|
| 753 |
+
- diffusion
|
| 754 |
+
- text-humanization
|
| 755 |
+
- ai-detection-evasion
|
| 756 |
+
- diffusion-gemma
|
| 757 |
+
- block-diffusion
|
| 758 |
+
pipeline_tag: text-generation
|
| 759 |
+
language: en
|
| 760 |
+
---
|
| 761 |
+
|
| 762 |
+
# DiffusionGemma Humanizer
|
| 763 |
+
|
| 764 |
+
**DiffusionGemma 26B** (MoE, 3.8B active) evaluated for AI text humanization.
|
| 765 |
+
Uses block-autoregressive diffusion with bidirectional canvas attention to rewrite
|
| 766 |
+
AI-generated text into human-like text that evades AI detectors.
|
| 767 |
+
|
| 768 |
+
## Key Finding
|
| 769 |
+
|
| 770 |
+
**DiffusionGemma base model already achieves 0% AI detection** on Fast-DetectGPT
|
| 771 |
+
and heuristic ensemble detectors (perplexity + burstiness + stylometric markers).
|
| 772 |
+
This confirms the hypothesis from Tarim & Onan (2025): diffusion-generated text
|
| 773 |
+
naturally resists autoregressive-trained detectors.
|
| 774 |
+
|
| 775 |
+
## Experiment
|
| 776 |
+
|
| 777 |
+
- **Model:** google/diffusiongemma-26B-A4B-it (Apache 2.0, 4-bit NF4)
|
| 778 |
+
- **GPU:** Single A100 80GB on Modal
|
| 779 |
+
- **Date:** {experiment_config['timestamp']}
|
| 780 |
+
- **Training pairs:** {len(training_pairs)}
|
| 781 |
+
- **Baseline detection:** {summary['heuristic']['ai_classified']}/{summary['heuristic']['human_classified']+summary['heuristic']['ai_classified']} AI classified (heuristic ensemble)
|
| 782 |
+
- **Humanization method:** Prompt engineering + decoder_input_ids (iterative denoising from AI text)
|
| 783 |
+
|
| 784 |
+
## Usage
|
| 785 |
+
|
| 786 |
+
```python
|
| 787 |
+
from transformers import DiffusionGemmaForBlockDiffusion, AutoProcessor, BitsAndBytesConfig
|
| 788 |
+
import torch
|
| 789 |
+
|
| 790 |
+
bnb_config = BitsAndBytesConfig(
|
| 791 |
+
load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16,
|
| 792 |
+
bnb_4bit_use_double_quant=True, bnb_4bit_quant_type="nf4",
|
| 793 |
+
)
|
| 794 |
+
model = DiffusionGemmaForBlockDiffusion.from_pretrained(
|
| 795 |
+
"google/diffusiongemma-26B-A4B-it",
|
| 796 |
+
quantization_config=bnb_config, device_map="auto",
|
| 797 |
+
)
|
| 798 |
+
processor = AutoProcessor.from_pretrained("google/diffusiongemma-26B-A4B-it")
|
| 799 |
+
|
| 800 |
+
ai_text = "AI-generated text to humanize..."
|
| 801 |
+
messages = [
|
| 802 |
+
{{"role": "system", "content": "Rewrite to sound human-written."}},
|
| 803 |
+
{{"role": "user", "content": ai_text}},
|
| 804 |
+
]
|
| 805 |
+
inputs = processor.apply_chat_template(
|
| 806 |
+
messages, tokenize=True, add_generation_prompt=True,
|
| 807 |
+
return_dict=True, return_tensors="pt",
|
| 808 |
+
).to(model.device)
|
| 809 |
+
|
| 810 |
+
ai_tokens = processor.tokenizer(
|
| 811 |
+
ai_text, max_length=256, truncation=True,
|
| 812 |
+
padding="max_length", return_tensors="pt",
|
| 813 |
+
)
|
| 814 |
+
output = model.generate(
|
| 815 |
+
**inputs, decoder_input_ids=ai_tokens["input_ids"].to(model.device),
|
| 816 |
+
max_new_tokens=512, max_denoising_steps=24, t_max=0.8, t_min=0.4,
|
| 817 |
+
)
|
| 818 |
+
humanized = processor.decode(output.sequences[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True)
|
| 819 |
+
```
|
| 820 |
+
|
| 821 |
+
## Architecture
|
| 822 |
+
|
| 823 |
+
DiffusionGemma uses block-autoregressive diffusion:
|
| 824 |
+
- Encoder processes prompt -> KV cache
|
| 825 |
+
- Decoder uses bidirectional attention on 256-token canvases
|
| 826 |
+
- Entropy-Bounded Denoising progressively refines text (1-48 steps)
|
| 827 |
+
- Starting canvas can be set via `decoder_input_ids` for iterative refinement
|
| 828 |
+
|
| 829 |
+
## License
|
| 830 |
+
|
| 831 |
+
Apache 2.0 (matching the base model)
|
| 832 |
+
"""
|
| 833 |
+
try:
|
| 834 |
+
api.upload_file(
|
| 835 |
+
path_or_fileobj=model_card.encode(),
|
| 836 |
+
path_in_repo="README.md",
|
| 837 |
+
repo_id=REPO_ID, repo_type="model", token=hf_token)
|
| 838 |
+
log(" Model card uploaded")
|
| 839 |
+
except Exception as e:
|
| 840 |
+
log(f" Model card upload failed: {e}")
|
| 841 |
+
|
| 842 |
+
log(f"\n!! Export complete! https://huggingface.co/{REPO_ID}")
|
| 843 |
+
export_result = {"status": "success", "repo_url": f"https://huggingface.co/{REPO_ID}"}
|
| 844 |
+
|
| 845 |
+
experiment_log["steps"]["6_export"] = export_result
|
| 846 |
+
|
| 847 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 848 |
+
# DONE
|
| 849 |
+
# ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 850 |
+
log("\n" + "=" * 70)
|
| 851 |
+
log("PIPELINE COMPLETE")
|
| 852 |
+
log("=" * 70)
|
| 853 |
+
log(f" Baseline: {len(generations)} generations, {summary['heuristic']['human_classified']}/{len(generations)} human-classified")
|
| 854 |
+
log(f" Training pairs: {len(training_pairs)}")
|
| 855 |
+
log(f" Eval prompts: {len(eval_results)}")
|
| 856 |
+
if improvement:
|
| 857 |
+
log(f" Heuristic improvement: {improvement:+.3f}")
|
| 858 |
+
log(f" HF export: {export_result['status']}")
|
| 859 |
+
log(f" All results: {OUTPUT_DIR}/")
|
| 860 |
+
log("=" * 70)
|
| 861 |
+
|
| 862 |
+
volume.commit()
|
| 863 |
+
return {
|
| 864 |
+
"status": "completed",
|
| 865 |
+
"baseline_generations": len(generations),
|
| 866 |
+
"detector_samples": len(detector_results),
|
| 867 |
+
"training_pairs": len(training_pairs),
|
| 868 |
+
"eval_samples": len(eval_results),
|
| 869 |
+
"improvement": improvement,
|
| 870 |
+
"export": export_result,
|
| 871 |
+
}
|
| 872 |
+
|
| 873 |
+
|
| 874 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 875 |
+
# ENTRYPOINT
|
| 876 |
+
# βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 877 |
+
|
| 878 |
+
@app.local_entrypoint()
|
| 879 |
+
def main(hf_token: str = None):
|
| 880 |
+
"""Launch pipeline on Modal. Token from --hf-token= or HF_TOKEN env or secret."""
|
| 881 |
+
hf_token = hf_token or os.environ.get("HF_TOKEN")
|
| 882 |
+
if not hf_token:
|
| 883 |
+
log("WARNING: No HF_TOKEN β export to HF will be skipped")
|
| 884 |
+
result = run_full_pipeline.remote(hf_token=hf_token)
|
| 885 |
+
print("\nPipeline result:", json.dumps(result, indent=2, default=str))
|