Spaces:
Running
Running
Update Space with full professional weblog
Browse files- index.html +437 -18
index.html
CHANGED
|
@@ -1,19 +1,438 @@
|
|
| 1 |
-
<!
|
| 2 |
-
<html>
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
</html>
|
|
|
|
| 1 |
+
<!DOCTYPE html>
|
| 2 |
+
<html lang="en" class="scroll-smooth">
|
| 3 |
+
<head>
|
| 4 |
+
<meta charset="UTF-8">
|
| 5 |
+
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
| 6 |
+
<title>BlockDiffuse: Fully Parallel Latent Space Reasoning Generation with Diffusion Transformers</title>
|
| 7 |
+
<meta name="description" content="Official research blog & technical report for BlockDiffuse: Non-autoregressive 100-token block generation in continuous latent space using Rectified Flow Matching and DiT.">
|
| 8 |
+
<meta name="keywords" content="Diffusion Transformer, Flow Matching, Large Language Models, Non-Autoregressive, Qwen2.5, Deep Learning">
|
| 9 |
+
<meta property="og:title" content="BlockDiffuse: Parallel 100-Token Reasoning in Latent Space">
|
| 10 |
+
<meta property="og:description" content="Overcoming the autoregressive sequential decoding bottleneck via Diffusion Transformers and Rectified Flow Matching.">
|
| 11 |
+
|
| 12 |
+
<!-- Tailwind CSS CDN -->
|
| 13 |
+
<script src="https://cdn.tailwindcss.com"></script>
|
| 14 |
+
<!-- MathJax for TeX equations -->
|
| 15 |
+
<script src="https://polyfill.io/v3/polyfill.min.js?features=es6"></script>
|
| 16 |
+
<script id="MathJax-script" async src="https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"></script>
|
| 17 |
+
<!-- Font Awesome Icons -->
|
| 18 |
+
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.4.0/css/all.min.css">
|
| 19 |
+
<!-- Google Fonts -->
|
| 20 |
+
<link rel="preconnect" href="https://fonts.googleapis.com">
|
| 21 |
+
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
| 22 |
+
<link href="https://fonts.googleapis.com/css2?family=Fira+Code:wght@400;500;600&family=Inter:wght@300;400;500;600;700;800&family=Newsreader:ital,opsz,wght@0,6..72,400;0,6..72,600;1,6..72,400&display=swap" rel="stylesheet">
|
| 23 |
+
|
| 24 |
+
<script>
|
| 25 |
+
tailwind.config = {
|
| 26 |
+
darkMode: 'class',
|
| 27 |
+
theme: {
|
| 28 |
+
extend: {
|
| 29 |
+
fontFamily: {
|
| 30 |
+
sans: ['Inter', 'sans-serif'],
|
| 31 |
+
serif: ['Newsreader', 'serif'],
|
| 32 |
+
mono: ['Fira Code', 'monospace'],
|
| 33 |
+
},
|
| 34 |
+
colors: {
|
| 35 |
+
brand: {
|
| 36 |
+
50: '#f0fdf4',
|
| 37 |
+
100: '#dcfce7',
|
| 38 |
+
400: '#4ade80',
|
| 39 |
+
500: '#22c55e',
|
| 40 |
+
600: '#16a34a',
|
| 41 |
+
700: '#15803d',
|
| 42 |
+
},
|
| 43 |
+
cyber: {
|
| 44 |
+
cyan: '#38bdf8',
|
| 45 |
+
purple: '#a855f7',
|
| 46 |
+
pink: '#ec4899',
|
| 47 |
+
dark: '#090d16',
|
| 48 |
+
card: '#0f172a',
|
| 49 |
+
border: '#1e293b'
|
| 50 |
+
}
|
| 51 |
+
}
|
| 52 |
+
}
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
</script>
|
| 56 |
+
<style>
|
| 57 |
+
.gradient-text {
|
| 58 |
+
background: linear-gradient(135deg, #38bdf8 0%, #a855f7 50%, #ec4899 100%);
|
| 59 |
+
-webkit-background-clip: text;
|
| 60 |
+
-webkit-text-fill-color: transparent;
|
| 61 |
+
}
|
| 62 |
+
.code-gradient {
|
| 63 |
+
background: linear-gradient(180deg, rgba(15,23,42,0.9) 0%, rgba(9,13,22,0.95) 100%);
|
| 64 |
+
}
|
| 65 |
+
.glass-card {
|
| 66 |
+
background: rgba(15, 23, 42, 0.75);
|
| 67 |
+
backdrop-filter: blur(12px);
|
| 68 |
+
border: 1px solid rgba(255, 255, 255, 0.08);
|
| 69 |
+
}
|
| 70 |
+
.pipeline-arrow {
|
| 71 |
+
position: relative;
|
| 72 |
+
}
|
| 73 |
+
.pipeline-arrow::after {
|
| 74 |
+
content: "➔";
|
| 75 |
+
position: absolute;
|
| 76 |
+
right: -1.25rem;
|
| 77 |
+
top: 50%;
|
| 78 |
+
transform: translateY(-50%);
|
| 79 |
+
color: #38bdf8;
|
| 80 |
+
font-size: 1.2rem;
|
| 81 |
+
}
|
| 82 |
+
@media (max-width: 768px) {
|
| 83 |
+
.pipeline-arrow::after {
|
| 84 |
+
content: "⬇";
|
| 85 |
+
right: 50%;
|
| 86 |
+
top: auto;
|
| 87 |
+
bottom: -1.5rem;
|
| 88 |
+
transform: translateX(50%);
|
| 89 |
+
}
|
| 90 |
+
}
|
| 91 |
+
</style>
|
| 92 |
+
</head>
|
| 93 |
+
<body class="bg-[#070b12] text-slate-200 font-sans antialiased selection:bg-cyan-500 selection:text-black">
|
| 94 |
+
|
| 95 |
+
<!-- Navigation Header -->
|
| 96 |
+
<header class="sticky top-0 z-50 glass-card border-b border-slate-800">
|
| 97 |
+
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 h-16 flex items-center justify-between">
|
| 98 |
+
<div class="flex items-center space-x-3">
|
| 99 |
+
<div class="h-9 w-9 rounded-lg bg-gradient-to-tr from-cyan-500 via-indigo-500 to-pink-500 flex items-center justify-center text-white font-black text-xl shadow-lg shadow-cyan-500/20">
|
| 100 |
+
B
|
| 101 |
+
</div>
|
| 102 |
+
<span class="text-xl font-bold tracking-tight text-white font-mono">Block<span class="text-cyan-400">Diffuse</span></span>
|
| 103 |
+
<span class="text-xs bg-slate-800 border border-slate-700 text-cyan-400 px-2 py-0.5 rounded-full font-mono">v1.0-Research</span>
|
| 104 |
+
</div>
|
| 105 |
+
|
| 106 |
+
<nav class="hidden md:flex items-center space-x-8 text-sm font-medium text-slate-400">
|
| 107 |
+
<a href="#motivation" class="hover:text-cyan-400 transition">Motivation</a>
|
| 108 |
+
<a href="#architecture" class="hover:text-cyan-400 transition">Architecture</a>
|
| 109 |
+
<a href="#flow-matching" class="hover:text-cyan-400 transition">Flow Matching</a>
|
| 110 |
+
<a href="#benchmarks" class="hover:text-cyan-400 transition">Benchmarks</a>
|
| 111 |
+
<a href="#quickstart" class="hover:text-cyan-400 transition">Code & Inference</a>
|
| 112 |
+
</nav>
|
| 113 |
+
|
| 114 |
+
<div class="flex items-center space-x-3">
|
| 115 |
+
<a href="https://huggingface.co/tahamajs/BlockDiffuse" target="_blank" class="flex items-center space-x-2 bg-yellow-500/10 hover:bg-yellow-500/20 border border-yellow-500/30 text-yellow-400 px-3.5 py-1.5 rounded-md text-xs font-semibold tracking-wide transition shadow-sm">
|
| 116 |
+
<span>🤗</span>
|
| 117 |
+
<span>Hugging Face</span>
|
| 118 |
+
</a>
|
| 119 |
+
<a href="https://github.com/Hooshaai/BlockDiffuse" target="_blank" class="flex items-center space-x-2 bg-slate-800 hover:bg-slate-700 border border-slate-700 text-white px-3.5 py-1.5 rounded-md text-xs font-semibold tracking-wide transition shadow-sm">
|
| 120 |
+
<i class="fa-brands fa-github text-sm"></i>
|
| 121 |
+
<span>GitHub</span>
|
| 122 |
+
</a>
|
| 123 |
+
</div>
|
| 124 |
+
</div>
|
| 125 |
+
</header>
|
| 126 |
+
|
| 127 |
+
<!-- Hero Section -->
|
| 128 |
+
<section class="relative pt-24 pb-20 overflow-hidden border-b border-slate-800/80">
|
| 129 |
+
<div class="absolute inset-0 bg-[radial-gradient(ellipse_80%_60%_at_50%_-20%,rgba(56,189,248,0.15),rgba(255,255,255,0))]"></div>
|
| 130 |
+
<div class="max-w-5xl mx-auto px-4 sm:px-6 lg:px-8 text-center relative z-10">
|
| 131 |
+
<div class="inline-flex items-center space-x-2 px-3 py-1 rounded-full bg-cyan-500/10 border border-cyan-500/20 text-cyan-300 text-xs font-medium mb-6 font-mono">
|
| 132 |
+
<span class="flex h-2 w-2 rounded-full bg-cyan-400 animate-pulse"></span>
|
| 133 |
+
<span>Fully Parallel Non-Autoregressive Generation</span>
|
| 134 |
+
</div>
|
| 135 |
+
<h1 class="text-4xl sm:text-6xl lg:text-7xl font-extrabold tracking-tight text-white mb-6 leading-tight">
|
| 136 |
+
Generating 100 Tokens at Once in <span class="gradient-text">Continuous Latent Space</span>
|
| 137 |
+
</h1>
|
| 138 |
+
<p class="text-lg sm:text-xl text-slate-300 max-w-3xl mx-auto leading-relaxed mb-10 font-normal">
|
| 139 |
+
Bypassing the token-by-token autoregressive bottleneck. <strong>BlockDiffuse</strong> couples a <strong>Diffusion Transformer (DiT)</strong> with a frozen decoder-only Base LLM via <strong>Rectified Flow Matching</strong> to synthesize entire multi-token reasoning blocks in parallel.
|
| 140 |
+
</p>
|
| 141 |
+
|
| 142 |
+
<!-- Quick Metrics Header -->
|
| 143 |
+
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 max-w-4xl mx-auto pt-6">
|
| 144 |
+
<div class="glass-card p-4 rounded-xl border border-slate-800">
|
| 145 |
+
<div class="text-3xl font-extrabold text-cyan-400 font-mono">100</div>
|
| 146 |
+
<div class="text-xs text-slate-400 mt-1 uppercase tracking-wider font-semibold">Tokens / Block</div>
|
| 147 |
+
</div>
|
| 148 |
+
<div class="glass-card p-4 rounded-xl border border-slate-800">
|
| 149 |
+
<div class="text-3xl font-extrabold text-indigo-400 font-mono">8</div>
|
| 150 |
+
<div class="text-xs text-slate-400 mt-1 uppercase tracking-wider font-semibold">ODE Steps (DPM)</div>
|
| 151 |
+
</div>
|
| 152 |
+
<div class="glass-card p-4 rounded-xl border border-slate-800">
|
| 153 |
+
<div class="text-3xl font-extrabold text-emerald-400 font-mono">1.73s</div>
|
| 154 |
+
<div class="text-xs text-slate-400 mt-1 uppercase tracking-wider font-semibold">100-Token Latency</div>
|
| 155 |
+
</div>
|
| 156 |
+
<div class="glass-card p-4 rounded-xl border border-slate-800">
|
| 157 |
+
<div class="text-3xl font-extrabold text-pink-400 font-mono">156.3</div>
|
| 158 |
+
<div class="text-xs text-slate-400 mt-1 uppercase tracking-wider font-semibold">Tokens / Sec (Multi-Block)</div>
|
| 159 |
+
</div>
|
| 160 |
+
</div>
|
| 161 |
+
</div>
|
| 162 |
+
</section>
|
| 163 |
+
|
| 164 |
+
<!-- Main Content Container -->
|
| 165 |
+
<main class="max-w-4xl mx-auto px-4 sm:px-6 lg:px-8 py-16 space-y-20">
|
| 166 |
+
|
| 167 |
+
<!-- 1. The Bottleneck & Motivation -->
|
| 168 |
+
<section id="motivation" class="space-y-6">
|
| 169 |
+
<div class="flex items-center space-x-3 text-cyan-400 font-mono text-xs uppercase tracking-widest">
|
| 170 |
+
<span>// 01</span>
|
| 171 |
+
<span class="h-px w-8 bg-cyan-400/40"></span>
|
| 172 |
+
<span>The Problem</span>
|
| 173 |
+
</div>
|
| 174 |
+
<h2 class="text-3xl font-bold text-white tracking-tight">The Sequential Decoding Bottleneck</h2>
|
| 175 |
+
<p class="text-slate-300 leading-relaxed text-base">
|
| 176 |
+
Modern Large Language Models (LLMs) rely on causal autoregressive decoding: to generate an answer of \( N \) tokens, the model must execute \( N \) sequential forward passes through billions of parameters:
|
| 177 |
+
</p>
|
| 178 |
+
<div class="glass-card p-5 rounded-lg border border-slate-800 text-center font-mono text-sm text-cyan-300 overflow-x-auto">
|
| 179 |
+
\( P(y_{1:N} \mid x) = \prod_{i=1}^{N} P(y_i \mid y_{<i}, x) \)
|
| 180 |
+
</div>
|
| 181 |
+
<p class="text-slate-300 leading-relaxed text-base">
|
| 182 |
+
For reasoning-intensive tasks (e.g. GSM8K, MATH, complex algorithmic reasoning), models emit hundreds of "Chain-of-Thought" tokens before answering. Even with KV-caching, generation latency is bounded by <strong>memory bandwidth</strong> and strict sequential serialization, keeping GPU compute utilization low.
|
| 183 |
+
</p>
|
| 184 |
+
|
| 185 |
+
<div class="grid grid-cols-1 md:grid-cols-2 gap-4 pt-2">
|
| 186 |
+
<div class="p-5 rounded-xl bg-red-950/20 border border-red-900/30">
|
| 187 |
+
<div class="flex items-center space-x-2 text-red-400 font-semibold mb-2">
|
| 188 |
+
<i class="fa-solid fa-triangle-exclamation"></i>
|
| 189 |
+
<span>Traditional Autoregressive (AR)</span>
|
| 190 |
+
</div>
|
| 191 |
+
<ul class="text-xs text-slate-400 space-y-2 leading-relaxed">
|
| 192 |
+
<li>• <strong>Sequential:</strong> 1 token per forward pass; latency scales linearly with sequence length \( \mathcal{O}(N) \).</li>
|
| 193 |
+
<li>• <strong>Bandwidth-bound:</strong> High memory reads per token, stalling tensor cores.</li>
|
| 194 |
+
<li>• <strong>No global foresight:</strong> Cannot revise earlier reasoning decisions in the same block.</li>
|
| 195 |
+
</ul>
|
| 196 |
+
</div>
|
| 197 |
+
<div class="p-5 rounded-xl bg-emerald-950/20 border border-emerald-900/30">
|
| 198 |
+
<div class="flex items-center space-x-2 text-emerald-400 font-semibold mb-2">
|
| 199 |
+
<i class="fa-solid fa-bolt"></i>
|
| 200 |
+
<span>BlockDiffuse (Non-AR Latent Diffusion)</span>
|
| 201 |
+
</div>
|
| 202 |
+
<ul class="text-xs text-slate-400 space-y-2 leading-relaxed">
|
| 203 |
+
<li>• <strong>Fully Parallel:</strong> 100 tokens synthesized concurrently in a fixed number of ODE steps (e.g. 8 steps).</li>
|
| 204 |
+
<li>• <strong>Compute-saturating:</strong> Full batch tensor-core utilization.</li>
|
| 205 |
+
<li>• <strong>Holistic Plan:</strong> DiT cross-attends over the entire block trajectory simultaneously.</li>
|
| 206 |
+
</ul>
|
| 207 |
+
</div>
|
| 208 |
+
</div>
|
| 209 |
+
</section>
|
| 210 |
+
|
| 211 |
+
<!-- 2. Architectural Blueprint -->
|
| 212 |
+
<section id="architecture" class="space-y-6">
|
| 213 |
+
<div class="flex items-center space-x-3 text-cyan-400 font-mono text-xs uppercase tracking-widest">
|
| 214 |
+
<span>// 02</span>
|
| 215 |
+
<span class="h-px w-8 bg-cyan-400/40"></span>
|
| 216 |
+
<span>Architecture</span>
|
| 217 |
+
</div>
|
| 218 |
+
<h2 class="text-3xl font-bold text-white tracking-tight">BlockDiffuse Hybrid Architecture</h2>
|
| 219 |
+
<p class="text-slate-300 leading-relaxed text-base">
|
| 220 |
+
BlockDiffuse decouples prompt understanding and semantic projection from diffusion trajectory generation. Rather than training a language model from scratch, we harness the representation space of <strong>Qwen/Qwen2.5-0.5B-Instruct</strong>:
|
| 221 |
+
</p>
|
| 222 |
+
|
| 223 |
+
<!-- Architectural Flow Diagram -->
|
| 224 |
+
<div class="glass-card p-6 rounded-xl border border-slate-800 space-y-6">
|
| 225 |
+
<div class="grid grid-cols-1 md:grid-cols-4 gap-4 items-center">
|
| 226 |
+
<div class="p-4 rounded-lg bg-slate-900/90 border border-slate-800 text-center pipeline-arrow">
|
| 227 |
+
<span class="text-xs font-mono text-indigo-400 block mb-1">Prompt Processing</span>
|
| 228 |
+
<div class="font-bold text-sm text-white">Frozen Base LLM</div>
|
| 229 |
+
<span class="text-[11px] text-slate-400">Layer 0 → 12 (Qwen2.5)</span>
|
| 230 |
+
</div>
|
| 231 |
+
<div class="p-4 rounded-lg bg-slate-900/90 border border-slate-800 text-center pipeline-arrow">
|
| 232 |
+
<span class="text-xs font-mono text-cyan-400 block mb-1">Denoising ODE</span>
|
| 233 |
+
<div class="font-bold text-sm text-white">BlockDiffuse DiT</div>
|
| 234 |
+
<span class="text-[11px] text-slate-400">8 Layers, 14 Heads (896-d)</span>
|
| 235 |
+
</div>
|
| 236 |
+
<div class="p-4 rounded-lg bg-slate-900/90 border border-slate-800 text-center pipeline-arrow">
|
| 237 |
+
<span class="text-xs font-mono text-pink-400 block mb-1">Residual Adapter</span>
|
| 238 |
+
<div class="font-bold text-sm text-white">Deep Proj Head</div>
|
| 239 |
+
<span class="text-[11px] text-slate-400">3-Layer SwiGLU MLP</span>
|
| 240 |
+
</div>
|
| 241 |
+
<div class="p-4 rounded-lg bg-slate-900/90 border border-slate-800 text-center">
|
| 242 |
+
<span class="text-xs font-mono text-emerald-400 block mb-1">Output Projection</span>
|
| 243 |
+
<div class="font-bold text-sm text-white">Token Logits</div>
|
| 244 |
+
<span class="text-[11px] text-slate-400">100 Parallel Tokens</span>
|
| 245 |
+
</div>
|
| 246 |
+
</div>
|
| 247 |
+
|
| 248 |
+
<div class="border-t border-slate-800/80 pt-4 text-xs text-slate-400 leading-relaxed font-mono">
|
| 249 |
+
<p><strong class="text-cyan-300">Prompt Latents:</strong> Extracted from layer 12 of Qwen2.5 (\( h_{\text{mid}} \in \mathbb{R}^{B \times L_p \times 896} \)).</p>
|
| 250 |
+
<p><strong class="text-pink-300">DiT Transfer Learning:</strong> Blocks initialized with weights copied directly from layers 6–11 of Qwen2.5-0.5B to preserve self-attention priors.</p>
|
| 251 |
+
</div>
|
| 252 |
+
</div>
|
| 253 |
+
</section>
|
| 254 |
+
|
| 255 |
+
<!-- 3. Mathematical Formulation (Rectified Flow) -->
|
| 256 |
+
<section id="flow-matching" class="space-y-6">
|
| 257 |
+
<div class="flex items-center space-x-3 text-cyan-400 font-mono text-xs uppercase tracking-widest">
|
| 258 |
+
<span>// 03</span>
|
| 259 |
+
<span class="h-px w-8 bg-cyan-400/40"></span>
|
| 260 |
+
<span>Mathematics</span>
|
| 261 |
+
</div>
|
| 262 |
+
<h2 class="text-3xl font-bold text-white tracking-tight">Rectified Flow Matching in Latent Space</h2>
|
| 263 |
+
<p class="text-slate-300 leading-relaxed text-base">
|
| 264 |
+
Let \( z_1 \in \mathbb{R}^{B \times 100 \times d} \) denote the target sequence latents and \( z_0 \sim \mathcal{N}(0, I) \) denote standard Gaussian noise. We construct straight-line probability paths between noise and data:
|
| 265 |
+
</p>
|
| 266 |
+
<div class="glass-card p-5 rounded-lg border border-slate-800 font-mono text-sm text-center text-cyan-300 overflow-x-auto">
|
| 267 |
+
\( z_t = (1 - t) z_0 + t z_1, \quad t \in [0, 1] \)
|
| 268 |
+
</div>
|
| 269 |
+
<p class="text-slate-300 leading-relaxed text-base">
|
| 270 |
+
The corresponding ground truth vector field is constant along trajectories: \( v_t = \frac{d z_t}{d t} = z_1 - z_0 \). The DiT predicts velocity \( v_\theta(z_t, t, c) \) conditioned on prompt context \( c \).
|
| 271 |
+
</p>
|
| 272 |
+
|
| 273 |
+
<h3 class="text-xl font-bold text-white pt-2">Composite Multi-Objective Training Loss</h3>
|
| 274 |
+
<p class="text-slate-300 leading-relaxed text-sm">
|
| 275 |
+
To guarantee semantic coherence and avoid mode collapse in discrete space, BlockDiffuse trains on five synergistically weighted objectives:
|
| 276 |
+
</p>
|
| 277 |
+
<div class="glass-card p-5 rounded-lg border border-slate-800 text-sm font-mono text-slate-200 overflow-x-auto">
|
| 278 |
+
\[
|
| 279 |
+
\mathcal{L}_{\text{total}} = \lambda_{\text{FM}} \mathcal{L}_{\text{FM}} + \lambda_{\text{disp}} \mathcal{L}_{\text{disp}} + \lambda_{\text{KL}} \mathcal{L}_{\text{KL}} + \lambda_{\text{CE}} \mathcal{L}_{\text{CE}} + \lambda_{\text{NN}} \mathcal{L}_{\text{NN}}
|
| 280 |
+
\]
|
| 281 |
+
</div>
|
| 282 |
+
|
| 283 |
+
<div class="grid grid-cols-1 sm:grid-cols-2 gap-3 text-xs">
|
| 284 |
+
<div class="p-3.5 rounded-lg bg-slate-900/60 border border-slate-800">
|
| 285 |
+
<span class="font-mono text-cyan-400 font-bold block mb-1">1. Velocity MSE (\( \mathcal{L}_{\text{FM}} \))</span>
|
| 286 |
+
<p class="text-slate-400">\( \mathbb{E}_{t, z_0, z_1} [\| v_\theta(z_t, t, c) - (z_1 - z_0) \|^2] \). Directs diffusion trajectory.</p>
|
| 287 |
+
</div>
|
| 288 |
+
<div class="p-3.5 rounded-lg bg-slate-900/60 border border-slate-800">
|
| 289 |
+
<span class="font-mono text-indigo-400 font-bold block mb-1">2. Dispersive Loss (\( \mathcal{L}_{\text{disp}} \))</span>
|
| 290 |
+
<p class="text-slate-400">Repels adjacent token vectors to prevent collapse into repetitive token orbits.</p>
|
| 291 |
+
</div>
|
| 292 |
+
<div class="p-3.5 rounded-lg bg-slate-900/60 border border-slate-800">
|
| 293 |
+
<span class="font-mono text-pink-400 font-bold block mb-1">3. Teacher KL (\( \mathcal{L}_{\text{KL}} \))</span>
|
| 294 |
+
<p class="text-slate-400">Distills distribution from the frozen teacher LLM head over predicted \( \hat{z}_1 \).</p>
|
| 295 |
+
</div>
|
| 296 |
+
<div class="p-3.5 rounded-lg bg-slate-900/60 border border-slate-800">
|
| 297 |
+
<span class="font-mono text-emerald-400 font-bold block mb-1">4. Token CE (\( \mathcal{L}_{\text{CE}} \)) & NN InfoNCE</span>
|
| 298 |
+
<p class="text-slate-400">Chunked cross-entropy with gradient checkpointing + InfoNCE nearest-neighbor metric learning.</p>
|
| 299 |
+
</div>
|
| 300 |
+
</div>
|
| 301 |
+
</section>
|
| 302 |
+
|
| 303 |
+
<!-- 4. Benchmarks & Performance -->
|
| 304 |
+
<section id="benchmarks" class="space-y-6">
|
| 305 |
+
<div class="flex items-center space-x-3 text-cyan-400 font-mono text-xs uppercase tracking-widest">
|
| 306 |
+
<span>// 04</span>
|
| 307 |
+
<span class="h-px w-8 bg-cyan-400/40"></span>
|
| 308 |
+
<span>Empirical Verification</span>
|
| 309 |
+
</div>
|
| 310 |
+
<h2 class="text-3xl font-bold text-white tracking-tight">Benchmark Results & Telemetry</h2>
|
| 311 |
+
<p class="text-slate-300 leading-relaxed text-base">
|
| 312 |
+
All benchmarks measured on a single <strong>NVIDIA GeForce RTX 4070 Laptop GPU (8GB VRAM)</strong> utilizing DPM-Solver with second-order multistep integration:
|
| 313 |
+
</p>
|
| 314 |
+
|
| 315 |
+
<!-- Table -->
|
| 316 |
+
<div class="overflow-x-auto rounded-xl border border-slate-800">
|
| 317 |
+
<table class="w-full text-left text-sm text-slate-300">
|
| 318 |
+
<thead class="bg-slate-900/80 text-xs uppercase font-mono text-cyan-400 border-b border-slate-800">
|
| 319 |
+
<tr>
|
| 320 |
+
<th class="py-3 px-4">Mode</th>
|
| 321 |
+
<th class="py-3 px-4">Generated Tokens</th>
|
| 322 |
+
<th class="py-3 px-4">ODE Steps / Block</th>
|
| 323 |
+
<th class="py-3 px-4">Latency</th>
|
| 324 |
+
<th class="py-3 px-4">Throughput</th>
|
| 325 |
+
<th class="py-3 px-4">VRAM</th>
|
| 326 |
+
</tr>
|
| 327 |
+
</thead>
|
| 328 |
+
<tbody class="divide-y divide-slate-800/60 font-mono text-xs">
|
| 329 |
+
<tr class="hover:bg-slate-800/30">
|
| 330 |
+
<td class="py-3.5 px-4 font-bold text-white">Single-Block Parallel</td>
|
| 331 |
+
<td class="py-3.5 px-4">100 tokens</td>
|
| 332 |
+
<td class="py-3.5 px-4">8 steps (DPM-Solver)</td>
|
| 333 |
+
<td class="py-3.5 px-4 text-emerald-400 font-semibold">1,730.60 ms</td>
|
| 334 |
+
<td class="py-3.5 px-4 text-cyan-400 font-semibold">57.78 tok/s</td>
|
| 335 |
+
<td class="py-3.5 px-4 text-slate-400">3,674 MB</td>
|
| 336 |
+
</tr>
|
| 337 |
+
<tr class="hover:bg-slate-800/30 bg-slate-900/30">
|
| 338 |
+
<td class="py-3.5 px-4 font-bold text-white">Multi-Block Autoregressive</td>
|
| 339 |
+
<td class="py-3.5 px-4">200 tokens (2 blocks)</td>
|
| 340 |
+
<td class="py-3.5 px-4">8 steps / block</td>
|
| 341 |
+
<td class="py-3.5 px-4 text-emerald-400 font-semibold">1,279.20 ms</td>
|
| 342 |
+
<td class="py-3.5 px-4 text-cyan-400 font-semibold">156.35 tok/s</td>
|
| 343 |
+
<td class="py-3.5 px-4 text-slate-400">3,789 MB</td>
|
| 344 |
+
</tr>
|
| 345 |
+
</tbody>
|
| 346 |
+
</table>
|
| 347 |
+
</div>
|
| 348 |
+
|
| 349 |
+
<!-- Training Loss Curve Highlights -->
|
| 350 |
+
<div class="glass-card p-5 rounded-xl border border-slate-800 text-xs font-mono space-y-2">
|
| 351 |
+
<div class="flex items-center justify-between text-slate-300">
|
| 352 |
+
<span>Loss Convergence: Step 0 → Step 17,000</span>
|
| 353 |
+
<span class="text-emerald-400 font-bold">↓ 96% Loss Reduction</span>
|
| 354 |
+
</div>
|
| 355 |
+
<div class="w-full bg-slate-900 rounded-full h-2.5 overflow-hidden">
|
| 356 |
+
<div class="bg-gradient-to-r from-cyan-500 to-emerald-400 h-2.5 rounded-full" style="width: 96%"></div>
|
| 357 |
+
</div>
|
| 358 |
+
<div class="flex justify-between text-[11px] text-slate-400 pt-1">
|
| 359 |
+
<span>Start: \(\mathcal{L}_{\text{tot}} \approx 81.87\)</span>
|
| 360 |
+
<span>Final: \(\mathcal{L}_{\text{tot}} = 3.22\) (Velocity MSE: \(3.75\))</span>
|
| 361 |
+
</div>
|
| 362 |
+
</div>
|
| 363 |
+
</section>
|
| 364 |
+
|
| 365 |
+
<!-- 5. Quickstart & Interactive Run -->
|
| 366 |
+
<section id="quickstart" class="space-y-6">
|
| 367 |
+
<div class="flex items-center space-x-3 text-cyan-400 font-mono text-xs uppercase tracking-widest">
|
| 368 |
+
<span>// 05</span>
|
| 369 |
+
<span class="h-px w-8 bg-cyan-400/40"></span>
|
| 370 |
+
<span>Get Started</span>
|
| 371 |
+
</div>
|
| 372 |
+
<h2 class="text-3xl font-bold text-white tracking-tight">Run Inference in 3 Steps</h2>
|
| 373 |
+
<p class="text-slate-300 leading-relaxed text-base">
|
| 374 |
+
You can replicate our results or integrate BlockDiffuse into your research pipeline directly from the open-source repository:
|
| 375 |
+
</p>
|
| 376 |
+
|
| 377 |
+
<!-- Code Block -->
|
| 378 |
+
<div class="code-gradient rounded-xl border border-slate-800 overflow-hidden text-xs font-mono shadow-2xl">
|
| 379 |
+
<div class="flex items-center justify-between px-4 py-2.5 bg-slate-900/90 border-b border-slate-800 text-slate-400">
|
| 380 |
+
<div class="flex space-x-1.5">
|
| 381 |
+
<div class="w-3 h-3 rounded-full bg-red-500/80"></div>
|
| 382 |
+
<div class="w-3 h-3 rounded-full bg-yellow-500/80"></div>
|
| 383 |
+
<div class="w-3 h-3 rounded-full bg-emerald-500/80"></div>
|
| 384 |
+
</div>
|
| 385 |
+
<span>bash</span>
|
| 386 |
+
</div>
|
| 387 |
+
<pre class="p-4 text-slate-200 overflow-x-auto leading-relaxed"><code><span class="text-slate-500"># 1. Clone the repository</span>
|
| 388 |
+
git clone https://github.com/Hooshaai/BlockDiffuse.git
|
| 389 |
+
<span class="text-cyan-400">cd</span> BlockDiffuse
|
| 390 |
+
|
| 391 |
+
<span class="text-slate-500"># 2. Install dependencies</span>
|
| 392 |
+
pip install -r requirements.txt
|
| 393 |
+
|
| 394 |
+
<span class="text-slate-500"># 3. Generate 100 parallel tokens via DPM-Solver & TFE</span>
|
| 395 |
+
python inference.py \
|
| 396 |
+
--model Qwen/Qwen2.5-0.5B-Instruct \
|
| 397 |
+
--checkpoint ./checkpoints_improved/blockdiffuse_final.pt \
|
| 398 |
+
--prompt "<span class="text-emerald-300"><|im_start|>system\nYou are a helpful assistant.<|im_end|>\n<|im_start|>user\nA bookstore has 140 books on Monday. They sell 45 and get 80. How many remain?<|im_end|>\n<|im_start|>assistant\n</span>" \
|
| 399 |
+
--steps 8 \
|
| 400 |
+
--solver dpm_solver \
|
| 401 |
+
--use_tfe \
|
| 402 |
+
--tfe_seeds 3</code></pre>
|
| 403 |
+
</div>
|
| 404 |
+
</section>
|
| 405 |
+
|
| 406 |
+
<!-- 6. BibTeX Citation -->
|
| 407 |
+
<section class="space-y-4 pt-4 border-t border-slate-800">
|
| 408 |
+
<h3 class="text-xl font-bold text-white">Citation</h3>
|
| 409 |
+
<div class="code-gradient p-4 rounded-xl border border-slate-800 font-mono text-xs text-slate-300 overflow-x-auto">
|
| 410 |
+
<pre><code>@article{blockdiffuse2026,
|
| 411 |
+
title={BlockDiffuse: Fully Parallel Latent Space Reasoning Generation with Diffusion Transformers},
|
| 412 |
+
author={Hooshaai Research},
|
| 413 |
+
journal={GitHub / HuggingFace Technical Report},
|
| 414 |
+
year={2026},
|
| 415 |
+
url={https://github.com/Hooshaai/BlockDiffuse}
|
| 416 |
+
}</code></pre>
|
| 417 |
+
</div>
|
| 418 |
+
</section>
|
| 419 |
+
|
| 420 |
+
</main>
|
| 421 |
+
|
| 422 |
+
<!-- Footer -->
|
| 423 |
+
<footer class="border-t border-slate-800/80 bg-[#05080e] py-12 text-slate-500 text-xs">
|
| 424 |
+
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 flex flex-col md:flex-row items-center justify-between gap-4">
|
| 425 |
+
<div class="flex items-center space-x-2">
|
| 426 |
+
<span class="font-bold text-slate-300 font-mono">BlockDiffuse</span>
|
| 427 |
+
<span>© 2026 Hooshaai Research. Licensed under Apache 2.0.</span>
|
| 428 |
+
</div>
|
| 429 |
+
<div class="flex space-x-6 font-mono text-xs">
|
| 430 |
+
<a href="https://github.com/Hooshaai/BlockDiffuse" class="hover:text-cyan-400 transition">GitHub Repo</a>
|
| 431 |
+
<a href="https://huggingface.co/tahamajs/BlockDiffuse" class="hover:text-cyan-400 transition">HuggingFace Model</a>
|
| 432 |
+
<a href="https://github.com/Hooshaai/BlockDiffuse/issues" class="hover:text-cyan-400 transition">Issues</a>
|
| 433 |
+
</div>
|
| 434 |
+
</div>
|
| 435 |
+
</footer>
|
| 436 |
+
|
| 437 |
+
</body>
|
| 438 |
</html>
|