Spaces:
Running
Running
File size: 87,643 Bytes
e841931 210c8a5 e841931 edf1fe5 e841931 05cce7b edf1fe5 e841931 edf1fe5 e841931 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 edf1fe5 a49cf03 e841931 a49cf03 e841931 7048de1 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 | <!DOCTYPE html>
<html lang="en" class="dark">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>AsyncTensorRLHF — High-Throughput Asynchronous RLHF Explorer</title>
<!-- Tailwind CSS -->
<script src="https://cdn.tailwindcss.com"></script>
<!-- Chart.js -->
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
<!-- Lucide Icons -->
<script src="https://unpkg.com/lucide@latest"></script>
<!-- KaTeX for Math -->
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.16.8/dist/katex.min.css">
<script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.8/dist/katex.min.js"></script>
<script defer src="https://cdn.jsdelivr.net/npm/katex@0.16.8/dist/contrib/auto-render.min.js" onload="renderMathInElement(document.body, {delimiters: [{left: '$$', right: '$$', display: true}, {left: '$', right: '$', display: false}]});"></script>
<script>
tailwind.config = {
darkMode: 'class',
theme: {
extend: {
colors: {
brand: { 50: '#eef2ff', 500: '#6366f1', 600: '#4f46e5', 700: '#4338ca' }
}
}
}
}
</script>
</head>
<body class="bg-slate-950 text-slate-100 min-h-screen font-sans antialiased selection:bg-indigo-500 selection:text-white">
<!-- Navigation -->
<nav class="border-b border-slate-800/80 bg-slate-900/60 backdrop-blur sticky top-0 z-50">
<div class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 h-16 flex items-center justify-between">
<div class="flex items-center space-x-3">
<div class="w-9 h-9 rounded-lg bg-gradient-to-tr from-indigo-600 to-violet-500 flex items-center justify-center font-bold text-white shadow-lg shadow-indigo-500/30">
⚡
</div>
<div>
<span class="font-bold text-lg tracking-tight bg-clip-text text-transparent bg-gradient-to-r from-white via-slate-200 to-indigo-300">AsyncTensorRLHF</span>
<span class="text-xs px-2 py-0.5 ml-2 rounded-full bg-indigo-500/20 text-indigo-400 border border-indigo-500/30">v1.0.0</span>
</div>
</div>
<a href="paper.pdf" target="_blank" class="flex items-center space-x-2 text-sm bg-indigo-600 hover:bg-indigo-500 text-white px-3.5 py-1.5 rounded-lg shadow-sm transition font-medium">
<i data-lucide="file-text" class="w-4 h-4"></i>
<span>Read Paper (PDF)</span>
</a>
<a href="https://github.com/Hooshaai/AsyncTensorRLHF" target="_blank" class="flex items-center space-x-2 text-sm text-slate-400 hover:text-white px-3 py-1.5 rounded-md hover:bg-slate-800 transition">
<i data-lucide="github" class="w-4 h-4"></i>
<span>GitHub</span>
</a>
<a href="https://huggingface.co/Hooshaai/Qwen2.5-0.5B-AsyncTensorRLHF" target="_blank" class="flex items-center space-x-2 text-sm bg-purple-600 hover:bg-purple-500 text-white px-3 py-1.5 rounded-lg shadow-sm transition font-medium">
<span>🚀 Trained Qwen Model</span>
</a>
<a href="https://huggingface.co/tahamajs/AsyncTensorRLHF" target="_blank" class="flex items-center space-x-2 text-sm bg-slate-800 hover:bg-slate-700 text-slate-200 px-3 py-1.5 rounded-lg shadow-sm transition font-medium">
<span>🤗 Model Card</span>
</a>
</div>
</div>
</nav>
<!-- Hero Section -->
<header class="py-14 px-4 sm:px-6 lg:px-8 max-w-7xl mx-auto text-center">
<div class="inline-flex items-center space-x-2 px-3 py-1 rounded-full bg-indigo-500/10 border border-indigo-500/20 text-indigo-400 text-xs font-semibold uppercase tracking-wider mb-6">
<span>Research & Benchmark Release</span>
<span class="w-1.5 h-1.5 rounded-full bg-indigo-400"></span>
<span>NVIDIA RTX 4070 Verified</span>
</div>
<h1 class="text-4xl sm:text-5xl lg:text-6xl font-extrabold tracking-tight text-white mb-6 leading-tight">
High-Throughput Asynchronous RLHF<br/>
<span class="bg-clip-text text-transparent bg-gradient-to-r from-indigo-400 via-purple-300 to-pink-400">with In-VRAM Tensor-Native Rewards</span>
</h1>
<p class="max-w-3xl mx-auto text-slate-400 text-lg sm:text-xl leading-relaxed">
Decoupling autoregressive generation from policy gradient updates. Eliminates CPU string SerDes overhead via zero-copy GPU tensor rewards and guarantees bounded off-policy staleness with <strong>M2PO</strong> & <strong>GRPO</strong>.
</p>
<!-- Key Metrics Grid -->
<div class="grid grid-cols-2 sm:grid-cols-4 gap-4 max-w-4xl mx-auto mt-10">
<div class="p-4 rounded-xl bg-slate-900/80 border border-slate-800/80 shadow-md">
<div class="text-2xl sm:text-3xl font-black text-indigo-400">8.05M</div>
<div class="text-xs text-slate-400 mt-1 uppercase tracking-wider font-medium">GRPO Tokens / sec</div>
</div>
<div class="p-4 rounded-xl bg-slate-900/80 border border-slate-800/80 shadow-md">
<div class="text-2xl sm:text-3xl font-black text-emerald-400">411k</div>
<div class="text-xs text-slate-400 mt-1 uppercase tracking-wider font-medium">Buffer Samples / sec</div>
</div>
<div class="p-4 rounded-xl bg-slate-900/80 border border-slate-800/80 shadow-md">
<div class="text-2xl sm:text-3xl font-black text-purple-400">62.1%</div>
<div class="text-xs text-slate-400 mt-1 uppercase tracking-wider font-medium">Variance Reduction (M2PO)</div>
</div>
<div class="p-4 rounded-xl bg-slate-900/80 border border-slate-800/80 shadow-md">
<div class="text-2xl sm:text-3xl font-black text-amber-400">41 / 41</div>
<div class="text-xs text-slate-400 mt-1 uppercase tracking-wider font-medium">Unit & E2E Tests Passing</div>
</div>
</div>
</header>
<!-- Interactive Explorer Tabs -->
<main class="max-w-7xl mx-auto px-4 sm:px-6 lg:px-8 pb-20">
<div class="bg-slate-900/90 border border-slate-800 rounded-2xl p-6 sm:p-8 shadow-2xl">
<!-- Navigation Tabs -->
<div class="flex space-x-2 border-b border-slate-800 pb-4 mb-8 overflow-x-auto">
<button onclick="switchTab('tab-reward')" id="btn-tab-reward" class="tab-btn px-4 py-2 rounded-lg text-sm font-semibold transition bg-indigo-600 text-white flex items-center space-x-2">
<i data-lucide="cpu" class="w-4 h-4"></i>
<span>1. In-VRAM Reward Simulator</span>
</button>
<button onclick="switchTab('tab-losses')" id="btn-tab-losses" class="tab-btn px-4 py-2 rounded-lg text-sm font-semibold transition text-slate-400 hover:text-white flex items-center space-x-2">
<i data-lucide="trending-down" class="w-4 h-4"></i>
<span>2. Policy Loss & Staleness (M2PO)</span>
</button>
<button onclick="switchTab('tab-benchmarks')" id="btn-tab-benchmarks" class="tab-btn px-4 py-2 rounded-lg text-sm font-semibold transition text-slate-400 hover:text-white flex items-center space-x-2">
<i data-lucide="bar-chart-2" class="w-4 h-4"></i>
<span>3. Empirical GPU Benchmarks</span>
</button>
<button onclick="switchTab('tab-arch')" id="btn-tab-arch" class="tab-btn px-4 py-2 rounded-lg text-sm font-semibold transition text-slate-400 hover:text-white flex items-center space-x-2">
<i data-lucide="layers" class="w-4 h-4"></i>
<span>4. Architecture & Pipeline</span>
</button>
<button onclick="switchTab('tab-paper')" id="btn-tab-paper" class="tab-btn px-4 py-2 rounded-lg text-sm font-semibold transition text-slate-400 hover:text-white flex items-center space-x-2">
<i data-lucide="file-text" class="w-4 h-4"></i>
<span>5. Research Paper</span>
</button>
</div>
<!-- Tab 1: Reward Simulator -->
<div id="tab-reward" class="tab-content block">
<div class="grid grid-cols-1 lg:grid-cols-2 gap-8">
<div>
<h3 class="text-xl font-bold text-white mb-2 flex items-center space-x-2">
<span>Interactive Tensor-Native Pattern Matching</span>
</h3>
<p class="text-sm text-slate-400 mb-6">
Simulates the GPU sliding-window convolution (<code>.unfold()</code>) executed directly on token ID tensors inside VRAM without CPU string decoding.
</p>
<div class="space-y-4">
<div>
<label class="block text-xs font-semibold uppercase tracking-wider text-slate-400 mb-1">Generated Token Sequence (Tokens)</label>
<input type="text" id="input-seq" value="10, 20, 30, 40, 99, 50, 60" class="w-full bg-slate-950 border border-slate-800 rounded-lg px-3.5 py-2 text-sm text-indigo-200 font-mono focus:outline-none focus:border-indigo-500">
<p class="text-xs text-slate-500 mt-1">Token ID 99 is EOS. Tokens after EOS are automatically masked out.</p>
</div>
<div>
<label class="block text-xs font-semibold uppercase tracking-wider text-slate-400 mb-1">Target Answer Pattern (Tokens)</label>
<input type="text" id="input-pat" value="20, 30, 40" class="w-full bg-slate-950 border border-slate-800 rounded-lg px-3.5 py-2 text-sm text-indigo-200 font-mono focus:outline-none focus:border-indigo-500">
</div>
<div class="flex items-center space-x-4">
<button onclick="runRewardSimulation()" class="bg-indigo-600 hover:bg-indigo-500 text-white px-4 py-2 rounded-lg font-semibold text-sm transition flex items-center space-x-2 shadow-lg shadow-indigo-500/20">
<i data-lucide="play" class="w-4 h-4"></i>
<span>Execute In-VRAM Match</span>
</button>
<button onclick="setSample('match')" class="text-xs text-slate-400 hover:text-white px-2 py-1 bg-slate-800 rounded">Preset: Valid Match</button>
<button onclick="setSample('post-eos')" class="text-xs text-slate-400 hover:text-white px-2 py-1 bg-slate-800 rounded">Preset: After EOS (0.0)</button>
</div>
</div>
</div>
<!-- Simulation Visualizer Panel -->
<div class="bg-slate-950/80 border border-slate-800/80 rounded-xl p-5 flex flex-col justify-between">
<div>
<div class="flex items-center justify-between border-b border-slate-800 pb-3 mb-4">
<span class="text-xs font-bold uppercase tracking-wider text-slate-400">Tensor Memory Representation</span>
<span id="reward-badge" class="px-2.5 py-0.5 rounded-full text-xs font-bold bg-emerald-500/20 text-emerald-400 border border-emerald-500/30">Reward: 1.0</span>
</div>
<div class="text-xs text-slate-400 mb-2">Token Sequence Tape (GPU VRAM):</div>
<div id="token-tape" class="flex flex-wrap gap-1.5 mb-6 font-mono text-sm"></div>
<div class="text-xs text-slate-400 mb-2">Unfolded Sliding Window Match Details:</div>
<div id="window-details" class="font-mono text-xs text-slate-300 space-y-1 bg-slate-900/90 p-3 rounded-lg border border-slate-800/60 max-h-40 overflow-y-auto"></div>
</div>
<div class="mt-4 pt-3 border-t border-slate-800/60 flex items-center justify-between text-xs text-slate-400">
<span>PCIe Data Bus Transfer: <strong class="text-emerald-400">0.0 Bytes (Zero Copy)</strong></span>
<span>Latency: <strong class="text-indigo-400">< 0.05 ms</strong></span>
</div>
</div>
</div>
</div>
<!-- Tab 2: Losses & Staleness -->
<div id="tab-losses" class="tab-content hidden">
<div class="grid grid-cols-1 lg:grid-cols-3 gap-8">
<div class="lg:col-span-1 space-y-5">
<h3 class="text-xl font-bold text-white">M2PO Second-Moment Trust Region</h3>
<p class="text-sm text-slate-400">
When asynchronous rollout lag causes policy drift, the second moment <span class="font-mono">E[r^2]</span> spikes. M2PO bounds gradient variance by dynamically masking destructive outliers.
</p>
<div class="space-y-4">
<div>
<div class="flex justify-between text-xs font-semibold mb-1 text-slate-300">
<span>PPO Clip Parameter (ε)</span>
<span id="label-eps" class="font-mono text-indigo-400">0.20</span>
</div>
<input type="range" id="slider-eps" min="0.05" max="0.5" step="0.05" value="0.2" class="w-full accent-indigo-500" oninput="updateLossGraph()">
</div>
<div>
<div class="flex justify-between text-xs font-semibold mb-1 text-slate-300">
<span>M2PO Threshold (γ)</span>
<span id="label-m2" class="font-mono text-indigo-400">2.0</span>
</div>
<input type="range" id="slider-m2" min="1.0" max="5.0" step="0.5" value="2.0" class="w-full accent-indigo-500" oninput="updateLossGraph()">
</div>
<div>
<div class="flex justify-between text-xs font-semibold mb-1 text-slate-300">
<span>Asynchronous Staleness (τ)</span>
<span id="label-tau" class="font-mono text-indigo-400">5 versions</span>
</div>
<input type="range" id="slider-tau" min="0" max="10" step="1" value="5" class="w-full accent-indigo-500" oninput="updateLossGraph()">
</div>
</div>
<div class="p-3.5 rounded-lg bg-indigo-950/40 border border-indigo-500/20 text-xs text-indigo-200">
<strong class="text-white block mb-1">Empirical Result at τ = 8:</strong>
M2PO second-moment masking cuts gradient variance by <strong>62.1%</strong> compared to standard PPO, eliminating catastrophic policy divergence.
</div>
</div>
<div class="lg:col-span-2 bg-slate-950 p-4 rounded-xl border border-slate-800">
<h4 class="text-sm font-bold text-slate-300 mb-2">Importance Weight r(θ) vs. Loss Objective</h4>
<div class="h-64 sm:h-72">
<canvas id="lossChart"></canvas>
</div>
</div>
</div>
</div>
<!-- Tab 3: Benchmarks -->
<div id="tab-benchmarks" class="tab-content hidden">
<h3 class="text-xl font-bold text-white mb-2">Hardware Benchmarks (NVIDIA GeForce RTX 4070 Laptop GPU)</h3>
<p class="text-sm text-slate-400 mb-6">Empirical execution profile measured directly on PyTorch 2.6.0+cu124.</p>
<div class="grid grid-cols-1 lg:grid-cols-2 gap-8 mb-8">
<div class="bg-slate-950 p-4 rounded-xl border border-slate-800">
<h4 class="text-sm font-bold text-slate-300 mb-3">Policy Loss Throughput (Tokens / sec)</h4>
<div class="h-60">
<canvas id="throughputChart"></canvas>
</div>
</div>
<div class="bg-slate-950 p-4 rounded-xl border border-slate-800">
<h4 class="text-sm font-bold text-slate-300 mb-3">Staleness (τ) vs. M2PO Variance Reduction (%)</h4>
<div class="h-60">
<canvas id="varianceChart"></canvas>
</div>
</div>
</div>
<div class="overflow-x-auto">
<table class="w-full text-left text-xs border border-slate-800 rounded-lg overflow-hidden">
<thead class="bg-slate-800 text-slate-300 uppercase font-semibold">
<tr>
<th class="p-3">Workload Component</th>
<th class="p-3">Batch & Config</th>
<th class="p-3">Measured Latency / Speed</th>
<th class="p-3">Memory Footprint</th>
<th class="p-3">Status</th>
</tr>
</thead>
<tbody class="divide-y divide-slate-800 bg-slate-950/60 font-mono text-slate-300">
<tr>
<td class="p-3 text-white font-sans font-semibold">In-VRAM Tensor Reward</td>
<td class="p-3">B=64, L=256</td>
<td class="p-3 text-indigo-400 font-bold">8.20 ms (55.9 ms for B=128)</td>
<td class="p-3">0 MB host transfers</td>
<td class="p-3"><span class="px-2 py-0.5 rounded bg-emerald-500/20 text-emerald-400 font-sans">PASSED</span></td>
</tr>
<tr>
<td class="p-3 text-white font-sans font-semibold">GRPO Group Loss</td>
<td class="p-3">B=64, G=4</td>
<td class="p-3 text-emerald-400 font-bold">8,058,669 tokens/sec (2.03 ms)</td>
<td class="p-3">1.2 MB VRAM</td>
<td class="p-3"><span class="px-2 py-0.5 rounded bg-emerald-500/20 text-emerald-400 font-sans">PASSED</span></td>
</tr>
<tr>
<td class="p-3 text-white font-sans font-semibold">PPO Loss & Backward</td>
<td class="p-3">B=64, L=256</td>
<td class="p-3 text-emerald-400 font-bold">6,763,177 tokens/sec (2.42 ms)</td>
<td class="p-3">1.4 MB VRAM</td>
<td class="p-3"><span class="px-2 py-0.5 rounded bg-emerald-500/20 text-emerald-400 font-sans">PASSED</span></td>
</tr>
<tr>
<td class="p-3 text-white font-sans font-semibold">M2PO Second-Moment Loss</td>
<td class="p-3">B=64, L=256</td>
<td class="p-3 text-purple-400 font-bold">2,739,289 tokens/sec (5.98 ms)</td>
<td class="p-3">1.6 MB VRAM</td>
<td class="p-3"><span class="px-2 py-0.5 rounded bg-emerald-500/20 text-emerald-400 font-sans">PASSED</span></td>
</tr>
<tr>
<td class="p-3 text-white font-sans font-semibold">Bounded Replay Buffer Push</td>
<td class="p-3">Capacity=50,000</td>
<td class="p-3 text-amber-400 font-bold">312,283 ops / sec</td>
<td class="p-3">CPU/GPU shared</td>
<td class="p-3"><span class="px-2 py-0.5 rounded bg-emerald-500/20 text-emerald-400 font-sans">PASSED</span></td>
</tr>
<tr>
<td class="p-3 text-white font-sans font-semibold">Replay Buffer Sample</td>
<td class="p-3">Batch Size=64</td>
<td class="p-3 text-amber-400 font-bold">411,691 ops / sec</td>
<td class="p-3">Lock-free FIFO</td>
<td class="p-3"><span class="px-2 py-0.5 rounded bg-emerald-500/20 text-emerald-400 font-sans">PASSED</span></td>
</tr>
<tr>
<td class="p-3 text-white font-sans font-semibold">Complete Pipeline Footprint</td>
<td class="p-3">E2E Closed-Loop</td>
<td class="p-3 text-cyan-400 font-bold">< 3 ms per step</td>
<td class="p-3 font-bold text-emerald-400">17.00 MB VRAM total</td>
<td class="p-3"><span class="px-2 py-0.5 rounded bg-emerald-500/20 text-emerald-400 font-sans">PASSED</span></td>
</tr>
</tbody>
</table>
</div>
</div>
<!-- Tab 4: Architecture -->
<div id="tab-arch" class="tab-content hidden">
<h3 class="text-xl font-bold text-white mb-2">Architectural Blueprint</h3>
<p class="text-sm text-slate-400 mb-6">High-throughput asynchronous control flow and continuous streaming.</p>
<div class="grid grid-cols-1 md:grid-cols-3 gap-6">
<div class="p-5 rounded-xl bg-slate-950 border border-slate-800">
<div class="w-8 h-8 rounded bg-indigo-500/20 text-indigo-400 flex items-center justify-center font-bold mb-3">1</div>
<h4 class="font-bold text-white mb-2">Rollout Cluster</h4>
<p class="text-xs text-slate-400 leading-relaxed mb-3">
Workers continuously pull prompts from <code>PromptQueue</code> and generate completions using <code>vLLM</code> or <code>HFEngine</code> on dedicated GPUs.
</p>
<ul class="text-xs text-slate-500 space-y-1 font-mono">
<li>• Async continuous batching</li>
<li>• In-VRAM sliding window reward</li>
<li>• Zero CPU copy overhead</li>
</ul>
</div>
<div class="p-5 rounded-xl bg-slate-950 border border-slate-800">
<div class="w-8 h-8 rounded bg-purple-500/20 text-purple-400 flex items-center justify-center font-bold mb-3">2</div>
<h4 class="font-bold text-white mb-2">Versioned Replay Buffer</h4>
<p class="text-xs text-slate-400 leading-relaxed mb-3">
Acts as the asynchronous decouple boundary. Buffers experience tuples tagged with policy version <span class="font-mono">v</span>.
</p>
<ul class="text-xs text-slate-500 space-y-1 font-mono">
<li>• Max staleness eviction (τ > 5)</li>
<li>• 411k samples/sec throughput</li>
<li>• Group buffering for GRPO</li>
</ul>
</div>
<div class="p-5 rounded-xl bg-slate-950 border border-slate-800">
<div class="w-8 h-8 rounded bg-emerald-500/20 text-emerald-400 flex items-center justify-center font-bold mb-3">3</div>
<h4 class="font-bold text-white mb-2">Trainer Cluster</h4>
<p class="text-xs text-slate-400 leading-relaxed mb-3">
Consumes batches from the buffer and optimizes the model parameters using PPO, M2PO, or GRPO loss with gradient clipping.
</p>
<ul class="text-xs text-slate-500 space-y-1 font-mono">
<li>• Second-moment trust region</li>
<li>• 8.05M tokens/sec GRPO</li>
<li>• Asynchronous weight sync</li>
</ul>
</div>
</div>
</div>
<!-- Tab 5: Research Paper -->
<div id="tab-paper" class="tab-content hidden">
<div class="space-y-8">
<!-- Paper Header Card -->
<div class="bg-gradient-to-br from-slate-900 via-indigo-950/30 to-slate-900 p-6 sm:p-8 rounded-2xl border border-indigo-500/20 shadow-xl">
<div class="flex flex-wrap items-center justify-between gap-4 mb-4">
<div class="flex items-center space-x-2">
<span class="text-xs font-bold px-3 py-1 rounded-full bg-indigo-500/20 text-indigo-300 border border-indigo-500/30 uppercase tracking-wider">
Academic Research Paper
</span>
<span class="text-xs font-semibold px-2.5 py-0.5 rounded-full bg-emerald-500/20 text-emerald-300 border border-emerald-500/30">
IEEE 2-Column Format
</span>
<span class="text-xs font-semibold px-2.5 py-0.5 rounded-full bg-purple-500/20 text-purple-300 border border-purple-500/30">
6 Pages • March 2026
</span>
</div>
<!-- View Switcher Tabs -->
<div class="inline-flex items-center p-1 rounded-xl bg-slate-950 border border-slate-800 shadow-inner">
<button onclick="togglePaperView('html')" id="btn-paper-html" class="px-3.5 py-1.5 rounded-lg text-xs font-bold bg-indigo-600 text-white shadow-sm flex items-center space-x-2 transition">
<i data-lucide="book-open" class="w-3.5 h-3.5"></i>
<span>Interactive Paper</span>
</button>
<button onclick="togglePaperView('pdf')" id="btn-paper-pdf" class="px-3.5 py-1.5 rounded-lg text-xs font-bold text-slate-400 hover:text-white flex items-center space-x-2 transition">
<i data-lucide="file-text" class="w-3.5 h-3.5"></i>
<span>Original PDF</span>
</button>
</div>
</div>
<h2 class="text-2xl sm:text-3xl lg:text-4xl font-extrabold text-white tracking-tight leading-tight mb-4">
AsyncTensorRLHF: High-Throughput Asynchronous RLHF with In-VRAM Tensor-Native Rewards and Second-Moment Off-Policy Control
</h2>
<div class="flex flex-wrap items-center gap-y-2 gap-x-6 text-sm text-slate-300 mb-6 font-sans">
<div class="flex items-center space-x-2">
<span class="font-bold text-white text-base">Taha Majlesi</span>
<span class="text-xs px-2 py-0.5 rounded bg-slate-800 text-slate-300 border border-slate-700">Author & Lead Researcher</span>
</div>
<div class="text-slate-400 flex items-center space-x-1">
<i data-lucide="building" class="w-4 h-4 text-indigo-400"></i>
<span>Hooshaai Research</span>
</div>
<div class="text-slate-400 flex items-center space-x-1 font-mono text-xs">
<i data-lucide="mail" class="w-3.5 h-3.5 text-indigo-400"></i>
<span>tahamajs@gmail.com</span>
</div>
</div>
<!-- Quick Action Toolbar -->
<div class="flex flex-wrap items-center gap-3 pt-4 border-t border-slate-800/80">
<a href="paper.pdf" target="_blank" download="AsyncTensorRLHF_Paper.pdf" class="inline-flex items-center space-x-2 bg-indigo-600 hover:bg-indigo-500 text-white font-medium text-xs sm:text-sm px-4 py-2 rounded-lg shadow-lg shadow-indigo-600/30 transition">
<i data-lucide="download" class="w-4 h-4"></i>
<span>Download PDF (409 KB)</span>
</a>
<button onclick="copyBibtex()" class="inline-flex items-center space-x-2 bg-slate-800 hover:bg-slate-700 text-slate-200 font-medium text-xs sm:text-sm px-4 py-2 rounded-lg border border-slate-700 transition">
<i data-lucide="copy" class="w-4 h-4 text-indigo-400"></i>
<span id="copy-btn-text">Copy BibTeX Citation</span>
</button>
<a href="https://huggingface.co/Hooshaai/Qwen2.5-0.5B-AsyncTensorRLHF" target="_blank" class="inline-flex items-center space-x-2 bg-purple-600/20 hover:bg-purple-600/30 text-purple-300 font-medium text-xs sm:text-sm px-4 py-2 rounded-lg border border-purple-500/30 transition">
<i data-lucide="database" class="w-4 h-4"></i>
<span>Trained Qwen Weights</span>
</a>
<a href="https://github.com/Hooshaai/AsyncTensorRLHF" target="_blank" class="inline-flex items-center space-x-2 text-slate-400 hover:text-white font-medium text-xs sm:text-sm px-3 py-2 rounded-lg hover:bg-slate-800/60 transition">
<i data-lucide="github" class="w-4 h-4"></i>
<span>GitHub Repository</span>
</a>
</div>
</div>
<!-- VIEW 1: INTERACTIVE HTML PAPER -->
<div id="paper-view-html" class="space-y-10">
<!-- Section Quick Navigator (Sticky Sub-nav) -->
<div class="bg-slate-950/80 backdrop-blur border border-slate-800 rounded-xl p-3 flex items-center space-x-2 overflow-x-auto text-xs font-semibold">
<span class="text-slate-500 px-2 uppercase tracking-wider flex items-center space-x-1">
<i data-lucide="list" class="w-3.5 h-3.5"></i>
<span>Jump To:</span>
</span>
<a href="#sec-abstract" class="px-2.5 py-1 rounded hover:bg-slate-800 text-slate-300 hover:text-white transition whitespace-nowrap">Abstract</a>
<a href="#sec-intro" class="px-2.5 py-1 rounded hover:bg-slate-800 text-slate-300 hover:text-white transition whitespace-nowrap">1. Introduction</a>
<a href="#sec-related" class="px-2.5 py-1 rounded hover:bg-slate-800 text-slate-300 hover:text-white transition whitespace-nowrap">2. Related Work</a>
<a href="#sec-theory" class="px-2.5 py-1 rounded hover:bg-slate-800 text-slate-300 hover:text-white transition whitespace-nowrap">3. Theory & M2PO</a>
<a href="#sec-arch" class="px-2.5 py-1 rounded hover:bg-slate-800 text-slate-300 hover:text-white transition whitespace-nowrap">4. System Architecture</a>
<a href="#sec-eval" class="px-2.5 py-1 rounded hover:bg-slate-800 text-slate-300 hover:text-white transition whitespace-nowrap">5. Empirical Results</a>
<a href="#sec-disc" class="px-2.5 py-1 rounded hover:bg-slate-800 text-slate-300 hover:text-white transition whitespace-nowrap">6. Discussion</a>
<a href="#sec-ref" class="px-2.5 py-1 rounded hover:bg-slate-800 text-slate-300 hover:text-white transition whitespace-nowrap">References</a>
</div>
<!-- ABSTRACT & METRICS GRID -->
<section id="sec-abstract" class="p-6 sm:p-8 rounded-2xl bg-slate-950 border border-slate-800 shadow-lg">
<div class="flex items-center space-x-2 mb-4">
<div class="w-7 h-7 rounded-lg bg-indigo-500/20 text-indigo-400 flex items-center justify-center font-bold text-sm">
§
</div>
<h3 class="text-xl font-bold text-white uppercase tracking-wider text-sm">Abstract</h3>
</div>
<p class="text-slate-300 leading-relaxed text-justify text-sm sm:text-base mb-6 font-sans">
Post-training Reinforcement Learning from Human Feedback (RLHF) and Group Relative Policy Optimization (GRPO) represent foundational paradigms for aligning Large Language Models (LLMs) and eliciting deliberate chain-of-thought mathematical reasoning. However, standard RLHF infrastructure suffers from two fundamental bottlenecks: (1) <em>the CPU-GPU serialization wall</em>, wherein generated token sequences are repeatedly shuttled across the PCIe bus to host RAM for UTF-8 decoding, string parsing, and rule-based evaluation; and (2) <em>the synchronous lockstep barrier</em>, which forces autoregressive rollout generation and gradient backpropagation to alternate in strict lockstep, leaving GPUs idle for $40\text{--}60\%$ of cluster wall-clock time.
</p>
<p class="text-slate-300 leading-relaxed text-justify text-sm sm:text-base mb-6 font-sans">
We introduce <strong>AsyncTensorRLHF</strong>, a fully asynchronous, disaggregated RLHF architecture engineered for high-throughput post-training. AsyncTensorRLHF eliminates host-device serialization overhead by evaluating pattern matching and rule-verifiable rewards natively inside GPU VRAM via strided 1D tensor convolutions (<code>.unfold()</code>), achieving zero-copy reward evaluation. To decouple rollout generation from policy optimization while provably preventing policy collapse under off-policy staleness $\tau$, we introduce <strong>M2PO</strong> (Second-Moment Trust Region Policy Optimization), which adaptively constrains the empirical second moment of importance sampling ratios $\mathbb{E}[r_t^2(\theta)]$.
</p>
<p class="text-slate-300 leading-relaxed text-justify text-sm sm:text-base font-sans">
Empirical evaluations on an NVIDIA GeForce RTX 4070 Laptop GPU demonstrate that AsyncTensorRLHF achieves an unprecedented GRPO throughput of <strong>8.05 million tokens/second</strong>, a lock-free buffer processing rate of <strong>411,691 samples/second</strong>, and reduces gradient variance by <strong>62.1%</strong> under extreme off-policy staleness ($\tau = 8$). Furthermore, we validate end-to-end alignment by fine-tuning Qwen2.5-0.5B-Instruct on mathematical reasoning tasks in 108.36 seconds, open-sourcing the architecture, trained weights, and an interactive web explorer.
</p>
<!-- Highlight Metrics Strip -->
<div class="grid grid-cols-2 md:grid-cols-4 gap-4 mt-8 pt-6 border-t border-slate-800">
<div class="p-3.5 rounded-xl bg-slate-900 border border-slate-800">
<div class="text-2xl font-black text-indigo-400">8,058,669</div>
<div class="text-[11px] uppercase tracking-wider text-slate-400 mt-1 font-medium">GRPO Tokens / Sec</div>
</div>
<div class="p-3.5 rounded-xl bg-slate-900 border border-slate-800">
<div class="text-2xl font-black text-emerald-400">411,691</div>
<div class="text-[11px] uppercase tracking-wider text-slate-400 mt-1 font-medium">Buffer Items / Sec</div>
</div>
<div class="p-3.5 rounded-xl bg-slate-900 border border-slate-800">
<div class="text-2xl font-black text-amber-400">62.1%</div>
<div class="text-[11px] uppercase tracking-wider text-slate-400 mt-1 font-medium">Variance Reduction ($\tau=8$)</div>
</div>
<div class="p-3.5 rounded-xl bg-slate-900 border border-slate-800">
<div class="text-2xl font-black text-purple-400">108.36 s</div>
<div class="text-[11px] uppercase tracking-wider text-slate-400 mt-1 font-medium">Qwen 2.5 0.5B Fine-Tuning</div>
</div>
</div>
</section>
<!-- SECTION 1: INTRODUCTION -->
<section id="sec-intro" class="p-6 sm:p-8 rounded-2xl bg-slate-950 border border-slate-800 shadow-lg space-y-6">
<div class="border-b border-slate-800 pb-4">
<h3 class="text-2xl font-bold text-white flex items-center space-x-3">
<span class="text-indigo-400 font-mono">1.</span>
<span>Introduction & Motivation</span>
</h3>
</div>
<p class="text-slate-300 leading-relaxed text-justify text-sm sm:text-base font-sans">
Reinforcement Learning from Human Feedback (RLHF) and Group Relative Policy Optimization (GRPO) have become the dominant training paradigms for aligning foundation language models and unlocking autonomous multi-step reasoning capabilities. In particular, reasoning-centric RL requires generating millions of candidate tokens per prompt to search for verifiable solutions in domains such as mathematics and software synthesis.
</p>
<div class="grid grid-cols-1 md:grid-cols-2 gap-6 my-6">
<!-- Bottleneck 1 -->
<div class="p-5 rounded-xl bg-rose-950/20 border border-rose-500/30">
<div class="flex items-center space-x-2 text-rose-400 font-bold mb-2">
<i data-lucide="alert-triangle" class="w-5 h-5"></i>
<h4>Bottleneck 1: The PCIe SerDes Wall</h4>
</div>
<p class="text-xs sm:text-sm text-slate-300 leading-relaxed">
Rollout engines generate token ID tensors in GPU VRAM. To score verifiable rewards (e.g., ground-truth math extraction, compiler syntax checks), standard frameworks copy tensors to host RAM, invoke Python <code>tokenizer.decode()</code>, execute CPU regular expressions, and transfer scalars back across PCIe. In high-throughput regimes ($B \ge 64, L \ge 512$), this serialization roundtrip consumes up to <strong>30–50%</strong> of overall pipeline duration.
</p>
</div>
<!-- Bottleneck 2 -->
<div class="p-5 rounded-xl bg-amber-950/20 border border-amber-500/30">
<div class="flex items-center space-x-2 text-amber-400 font-bold mb-2">
<i data-lucide="clock" class="w-5 h-5"></i>
<h4>Bottleneck 2: The Synchronous Lockstep Barrier</h4>
</div>
<p class="text-xs sm:text-sm text-slate-300 leading-relaxed">
Standard PPO alternates strictly between rollout generation and policy optimization. While training GPUs compute forward and backward autograd graphs, rollout inference workers sit idle. Conversely, during autoregressive token generation, training GPUs remain dormant. This lockstep coupling degrades cluster-wide GPU utilization to <strong>40–60%</strong>.
</p>
</div>
</div>
<!-- Visual Gantt Timeline Comparison -->
<div class="p-6 rounded-xl bg-slate-900 border border-slate-800">
<div class="text-xs font-bold uppercase tracking-wider text-slate-400 mb-4 flex items-center space-x-2">
<i data-lucide="bar-chart-3" class="w-4 h-4 text-indigo-400"></i>
<span>Figure 1: Execution Schedule Comparison (Synchronous vs. AsyncTensorRLHF)</span>
</div>
<div class="space-y-4">
<!-- Synchronous Timeline -->
<div>
<div class="flex justify-between text-xs text-slate-400 mb-1">
<span class="font-bold text-rose-400">Traditional Synchronous PPO:</span>
<span class="text-rose-400 font-mono">~50% Compute Wasted in Idle Bubbles</span>
</div>
<div class="grid grid-cols-12 gap-1 h-8 text-[11px] font-bold text-center leading-8 rounded overflow-hidden">
<div class="col-span-3 bg-indigo-600 text-white rounded">Rollout (GPU)</div>
<div class="col-span-3 bg-rose-950 text-rose-300 border border-rose-800/80 dashed rounded">IDLE BUBBLE (50%)</div>
<div class="col-span-3 bg-indigo-600 text-white rounded">Rollout (GPU)</div>
<div class="col-span-3 bg-rose-950 text-rose-300 border border-rose-800/80 dashed rounded">IDLE BUBBLE (50%)</div>
</div>
<div class="grid grid-cols-12 gap-1 h-8 text-[11px] font-bold text-center leading-8 rounded overflow-hidden mt-1">
<div class="col-span-3 bg-rose-950 text-rose-300 border border-rose-800/80 dashed rounded">IDLE BUBBLE</div>
<div class="col-span-3 bg-emerald-600 text-white rounded">Train Step</div>
<div class="col-span-3 bg-rose-950 text-rose-300 border border-rose-800/80 dashed rounded">IDLE BUBBLE</div>
<div class="col-span-3 bg-emerald-600 text-white rounded">Train Step</div>
</div>
</div>
<!-- Asynchronous Timeline -->
<div class="pt-3 border-t border-slate-800">
<div class="flex justify-between text-xs text-slate-400 mb-1">
<span class="font-bold text-emerald-400">AsyncTensorRLHF (Fully Disaggregated):</span>
<span class="text-emerald-400 font-mono">100% GPU Duty Cycle • Zero Idle Bubbles</span>
</div>
<div class="grid grid-cols-12 gap-1 h-8 text-[11px] font-bold text-center leading-8 rounded overflow-hidden">
<div class="col-span-3 bg-indigo-600 text-white rounded">Rollout v0</div>
<div class="col-span-3 bg-indigo-600 text-white rounded">Rollout v1</div>
<div class="col-span-3 bg-indigo-600 text-white rounded">Rollout v2</div>
<div class="col-span-3 bg-indigo-600 text-white rounded">Rollout v3</div>
</div>
<div class="grid grid-cols-12 gap-1 h-8 text-[11px] font-bold text-center leading-8 rounded overflow-hidden mt-1">
<div class="col-span-2 bg-slate-900 border border-slate-800 text-slate-500 rounded">Buffer Ingest</div>
<div class="col-span-3 bg-emerald-600 text-white rounded">Train Step v1</div>
<div class="col-span-3 bg-emerald-600 text-white rounded">Train Step v2</div>
<div class="col-span-4 bg-emerald-600 text-white rounded">Train Step v3</div>
</div>
</div>
</div>
</div>
<!-- Principal Contributions -->
<h4 class="text-lg font-bold text-white mt-6 mb-3">Core Contributions</h4>
<div class="grid grid-cols-1 sm:grid-cols-2 gap-4">
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 flex items-start space-x-3">
<i data-lucide="check" class="w-5 h-5 text-emerald-400 shrink-0 mt-0.5"></i>
<div>
<strong class="text-white text-sm block mb-1">In-VRAM Tensor-Native Rewards</strong>
<p class="text-xs text-slate-400">Zero-copy 1D sliding-window convolutions (<code>.unfold()</code>) evaluate token sequences in GPU VRAM without host CPU serialization.</p>
</div>
</div>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 flex items-start space-x-3">
<i data-lucide="check" class="w-5 h-5 text-emerald-400 shrink-0 mt-0.5"></i>
<div>
<strong class="text-white text-sm block mb-1">M2PO Second-Moment Trust Region</strong>
<p class="text-xs text-slate-400">Constrains the empirical second moment $\mathbb{E}[r_t^2(\theta)] \le \gamma$, provably bounding off-policy gradient variance under asynchronous drift.</p>
</div>
</div>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 flex items-start space-x-3">
<i data-lucide="check" class="w-5 h-5 text-emerald-400 shrink-0 mt-0.5"></i>
<div>
<strong class="text-white text-sm block mb-1">Lock-Free Experience Buffers for GRPO</strong>
<p class="text-xs text-slate-400">Circular replay buffers that standardize group advantages in-place without a critic model and evict stale samples $\tau > \tau_{\max}$.</p>
</div>
</div>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 flex items-start space-x-3">
<i data-lucide="check" class="w-5 h-5 text-emerald-400 shrink-0 mt-0.5"></i>
<div>
<strong class="text-white text-sm block mb-1">Empirical Hardware Validation</strong>
<p class="text-xs text-slate-400">Delivers 8.05M tokens/sec under GRPO on an RTX 4070 GPU and completes end-to-end fine-tuning on Qwen2.5-0.5B-Instruct in 108.36 seconds.</p>
</div>
</div>
</div>
</section>
<!-- SECTION 2: RELATED WORK -->
<section id="sec-related" class="p-6 sm:p-8 rounded-2xl bg-slate-950 border border-slate-800 shadow-lg space-y-4">
<div class="border-b border-slate-800 pb-4">
<h3 class="text-2xl font-bold text-white flex items-center space-x-3">
<span class="text-indigo-400 font-mono">2.</span>
<span>Related Work</span>
</h3>
</div>
<div class="space-y-4 text-slate-300 text-sm sm:text-base leading-relaxed text-justify">
<p>
<strong class="text-indigo-300">Synchronous RLHF Systems:</strong> Early language model alignment frameworks derived from InstructGPT adapted Proximal Policy Optimization (PPO) using monolithic training graphs. Implementations such as DeepSpeed-Chat, early Hugging Face TRL, and NeMo-Aligner co-locate inference and training on shared worker pools, incurring heavy synchronization barriers and cluster-wide idle bubbles.
</p>
<p>
<strong class="text-indigo-300">High-Throughput Inference Engines:</strong> Modern serving frameworks, notably vLLM and SGLang, revolutionized LLM rollout throughput via PagedAttention, continuous iteration-level batching, and prefix caching. While HybridFlow and OpenRLHF integrated these serving engines into training loops, reward assignment and policy gradient updates remained tied to synchronous inter-node barriers.
</p>
<p>
<strong class="text-indigo-300">Asynchronous Reinforcement Learning:</strong> Asynchronous actor-learner architectures originated in deep RL with A3C and IMPALA, which introduced V-trace to correct for off-policy discrepancies. However, V-trace relies on discrete action spaces and tabular importance sampling. AsyncTensorRLHF extends off-policy stabilization to autoregressive language generation via Second-Moment Trust Region bounds (M2PO).
</p>
</div>
</section>
<!-- SECTION 3: THEORETICAL FOUNDATIONS -->
<section id="sec-theory" class="p-6 sm:p-8 rounded-2xl bg-slate-950 border border-slate-800 shadow-lg space-y-6">
<div class="border-b border-slate-800 pb-4">
<h3 class="text-2xl font-bold text-white flex items-center space-x-3">
<span class="text-indigo-400 font-mono">3.</span>
<span>Theoretical Foundations & Mathematical Formulations</span>
</h3>
</div>
<!-- 3.1 Policy Gradient under Staleness -->
<div>
<h4 class="text-lg font-bold text-white mb-2">3.1 Policy Gradients Under Asynchronous Staleness ($ au$)</h4>
<p class="text-slate-300 text-sm sm:text-base leading-relaxed mb-4">
Let $x \sim \mathcal{D}_x$ denote an input prompt and $y = (y_1, \dots, y_L)$ denote an autoregressive token sequence generated by policy $\pi_{\theta_{\text{old}}}$. In an asynchronous regime, the parameter state $\theta$ advances during rollout generation, such that the experience tuple $(x, y, r, \log \pi_{\theta_{\text{old}}}(y \mid x))$ is consumed when the active policy has advanced to version $\theta$, where staleness is defined as $\tau = \text{version}(\theta) - \text{version}(\theta_{\text{old}}) \ge 0$.
</p>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 text-center font-mono text-sm sm:text-base my-3">
$$\mathcal{J}(\theta) = \mathbb{E}_{x \sim \mathcal{D}_x, y \sim \pi_{\theta_{\text{old}}}}\left[ \frac{\pi_\theta(y \mid x)}{\pi_{\theta_{\text{old}}}(y \mid x)} A^{\pi_{\theta_{\text{old}}}}(x, y) \right]$$
</div>
<p class="text-slate-300 text-sm leading-relaxed mb-3">
The per-token importance weight ratio is defined as:
</p>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 text-center font-mono text-sm sm:text-base my-3">
$$r_t(\theta) = \frac{\pi_\theta(y_t \mid x, y_{<t})}{\pi_{\theta_{\text{old}}}(y_t \mid x, y_{<t})} = \exp\left( \log \pi_\theta(y_t) - \log \pi_{\theta_{\text{old}}}(y_t) \right)$$
</div>
<p class="text-slate-300 text-sm leading-relaxed mb-3">
As staleness $\tau$ increases, the divergence between $\pi_\theta$ and $\pi_{\theta_{\text{old}}}$ expands. Under the $\chi^2$-divergence formulation:
</p>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 text-center font-mono text-sm sm:text-base my-3">
$$\operatorname{Var}_{y \sim \pi_{\theta_{\text{old}}}}[r_t(\theta)] = \mathbb{E}\left[ r_t(\theta)^2 \right] - 1 \approx \exp\left( D_{\chi^2}(\pi_\theta \parallel \pi_{\theta_{\text{old}}}) \right) - 1$$
</div>
<p class="text-xs text-slate-400">
When $\tau \ge 3$, outlier importance ratios $r_t(\theta) \gg 1$ cause severe gradient variance spikes, destabilizing standard policy gradient updates.
</p>
</div>
<!-- 3.2 Standard Clipped PPO -->
<div class="pt-6 border-t border-slate-800">
<h4 class="text-lg font-bold text-white mb-2">3.2 Standard Clipped PPO Objective</h4>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 text-center font-mono text-sm sm:text-base my-3">
$$\mathcal{L}_{\text{PPO}}(\theta) = -\mathbb{E}\left[ \min\left( r_t(\theta) A_t, \; \operatorname{clip}(r_t(\theta), 1-\epsilon, 1+\epsilon) A_t \right) \right]$$
</div>
<p class="text-slate-300 text-sm leading-relaxed">
While effective for synchronous training where $\theta \approx \theta_{\text{old}}$, clipping fails under severe asynchronous drift: when $r_t(\theta)$ is far outside $[1-\epsilon, 1+\epsilon]$, gradients vanish on valid exploration tokens while extreme advantage products distort parameter updates.
</p>
</div>
<!-- 3.3 M2PO Second-Moment Trust Region -->
<div class="pt-6 border-t border-slate-800 space-y-4">
<h4 class="text-lg font-bold text-white">3.3 M2PO: Second-Moment Trust Region Optimization</h4>
<p class="text-slate-300 text-sm leading-relaxed">
To resolve off-policy instability without discarding asynchronous parallelism, we formulate <strong>M2PO</strong>. We impose an empirical second-moment bound on the importance ratio distribution:
</p>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 text-center font-mono text-sm sm:text-base my-3">
$$M_2 = \frac{1}{B \cdot L} \sum_{b=1}^B \sum_{t=1}^L r_{b,t}(\theta)^2 \le \gamma_{\text{threshold}}$$
</div>
<p class="text-slate-300 text-sm leading-relaxed">
Tokens that violate this second-moment bound are identified by the binary indicator mask:
</p>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 text-center font-mono text-sm sm:text-base my-3">
$$m_{b,t} = \mathbb{I}\left( r_{b,t}(\theta)^2 \le \gamma_{\text{threshold}} \right)$$
</div>
<p class="text-slate-300 text-sm leading-relaxed">
The M2PO surrogate loss is defined as:
</p>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 text-center font-mono text-sm sm:text-base my-3">
$$\mathcal{L}_{\text{M2PO}}(\theta) = -\frac{1}{Z_m} \sum_{b=1}^B \sum_{t=1}^L m_{b,t} \cdot \min\Big( r_{b,t}(\theta) A_{b,t}, \; \operatorname{clip}(r_{b,t}(\theta), 1-\epsilon, 1+\epsilon) A_{b,t} \Big)$$
</div>
<p class="text-xs text-slate-400">
where normalization factor $Z_m = \max\left(1, \sum_{b=1}^B \sum_{t=1}^L m_{b,t}\right)$.
</p>
<!-- Theorem 1 Card -->
<div class="p-5 rounded-xl bg-indigo-950/30 border border-indigo-500/30 space-y-2">
<div class="flex items-center space-x-2 text-indigo-400 font-bold text-sm">
<i data-lucide="shield-check" class="w-4 h-4"></i>
<span>Theorem 1: Trust-Region Equivalence of Second-Moment Constraint</span>
</div>
<p class="text-xs sm:text-sm text-slate-300 leading-relaxed">
Under a second-order Taylor expansion around $\theta_{\text{old}}$, the expected second-moment penalty $\mathbb{E}[(r_t(\theta) - 1)^2]$ is asymptotically equivalent to the Fisher Information Matrix (FIM) Riemannian metric:
</p>
<div class="p-3 rounded-lg bg-slate-950 text-center font-mono text-xs sm:text-sm text-indigo-300">
$$\mathbb{E}\left[ (r_t(\theta) - 1)^2 \right] = (\theta - \theta_{\text{old}})^T F_{\theta_{\text{old}}} (\theta - \theta_{\text{old}}) + \mathcal{O}(\|\theta - \theta_{\text{old}}\|^3)$$
</div>
<p class="text-xs text-slate-400">
where $F_{\theta_{\text{old}}} = \mathbb{E}[\nabla_\theta \log \pi_\theta \nabla_\theta \log \pi_\theta^T]$ is the Fisher Information Matrix. Consequently, constraining $r_t(\theta)^2 \le \gamma$ implicitly enforces a Riemannian trust region without requiring explicit Hessian-vector products.
</p>
</div>
<!-- Proposition 2 & Proof Card -->
<div class="p-5 rounded-xl bg-slate-900 border border-slate-800 space-y-3">
<div class="flex items-center space-x-2 text-emerald-400 font-bold text-sm">
<i data-lucide="award" class="w-4 h-4"></i>
<span>Proposition 2: Gradient Variance Boundedness Under M2PO</span>
</div>
<div class="p-3 rounded-lg bg-slate-950 text-center font-mono text-xs sm:text-sm text-emerald-300">
$$\operatorname{Var}\left( \nabla_\theta \mathcal{L}_{\text{M2PO}}(\theta) \right) \le \frac{\gamma_{\text{threshold}}}{Z_m} \cdot \max_{b,t} \| \nabla_\theta \log \pi_\theta(y_{b,t}) \|^2 \cdot \mathbb{E}[A_{b,t}^2]$$
</div>
<div class="text-xs text-slate-400 space-y-1">
<div class="font-semibold text-slate-300">Formal Proof Summary:</div>
<p>Let $g_{b,t}(\theta) = \nabla_\theta \log \pi_\theta(y_{b,t} \mid x_b, y_{b,<t})$ denote the score function. Applying the variance inequality $\operatorname{Var}(X) \le \mathbb{E}[X^2]$, we have:</p>
<div class="p-2 rounded bg-slate-950 font-mono text-[11px] text-slate-300">
$$\operatorname{Var}(\nabla_\theta \mathcal{L}_{\text{M2PO}}) \le \frac{1}{Z_m^2} \sum_{b,t} \mathbb{E}\left[ m_{b,t}^2 r_{b,t}^2 A_{b,t}^2 \|g_{b,t}\|^2 \right] \le \frac{\gamma_{\text{threshold}}}{Z_m} \mathbb{E}[A^2] \max \|g\|^2$$
</div>
<p>Because $m_{b,t} r_{b,t}^2 \le \gamma_{\text{threshold}}$ holds almost surely by construction of the indicator mask, off-policy gradient variance is strictly bounded irrespective of staleness drift $\tau$. $\blacksquare$</p>
</div>
</div>
</div>
<!-- 3.4 GRPO -->
<div class="pt-6 border-t border-slate-800">
<h4 class="text-lg font-bold text-white mb-2">3.4 Group Relative Policy Optimization (GRPO)</h4>
<p class="text-slate-300 text-sm leading-relaxed mb-3">
For verifiable reasoning tasks, AsyncTensorRLHF implements GRPO, which eliminates the memory footprint of a learned critic network. For prompt $x$, a group of $G$ outputs is evaluated with scalar rewards, and advantages are standardized across the candidate group:
</p>
<div class="grid grid-cols-1 sm:grid-cols-2 gap-4 my-3">
<div class="p-3 rounded-lg bg-slate-900 border border-slate-800 text-center font-mono text-xs sm:text-sm">
$$\mu_R = \frac{1}{G} \sum_{i=1}^G R^{(i)}, \quad \sigma_R = \sqrt{\frac{1}{G} \sum_{i=1}^G (R^{(i)} - \mu_R)^2} + \epsilon_{\text{eps}}$$
</div>
<div class="p-3 rounded-lg bg-slate-900 border border-slate-800 text-center font-mono text-xs sm:text-sm flex items-center justify-center">
$$\hat{A}^{(i)} = \frac{R^{(i)} - \mu_R}{\sigma_R}$$
</div>
</div>
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 text-center font-mono text-sm my-3">
$$\mathcal{L}_{\text{GRPO}}(\theta) = -\frac{1}{B G L} \sum_{b=1}^B \sum_{i=1}^G \sum_{t=1}^L \min\Big( r_{b,i,t}(\theta) \hat{A}_{b,i}, \; \operatorname{clip}(r_{b,i,t}(\theta), 1-\epsilon, 1+\epsilon) \hat{A}_{b,i} \Big)$$
</div>
</div>
</section>
<!-- SECTION 4: SYSTEM ARCHITECTURE -->
<section id="sec-arch" class="p-6 sm:p-8 rounded-2xl bg-slate-950 border border-slate-800 shadow-lg space-y-6">
<div class="border-b border-slate-800 pb-4">
<h3 class="text-2xl font-bold text-white flex items-center space-x-3">
<span class="text-indigo-400 font-mono">4.</span>
<span>System Architecture & Engineering Implementation</span>
</h3>
</div>
<!-- Interactive Architecture Diagram Card -->
<div class="p-6 rounded-xl bg-slate-900 border border-slate-800">
<div class="text-xs font-bold uppercase tracking-wider text-slate-400 mb-4 flex items-center space-x-2">
<i data-lucide="layers" class="w-4 h-4 text-indigo-400"></i>
<span>Figure 2: Architectural Layout & Dataflow Diagram</span>
</div>
<div class="grid grid-cols-1 md:grid-cols-4 gap-4 text-center text-xs">
<div class="p-4 rounded-lg bg-slate-950 border border-slate-800 md:col-span-4">
<div class="font-bold text-indigo-300 mb-1">Orchestrator Control Plane</div>
<div class="text-slate-400 text-[11px]">Version Counter $v$ • Async Dispatcher • Periodic Weight Broadcast (<code>weight_sync_fn</code>)</div>
</div>
<div class="p-4 rounded-lg bg-indigo-950/40 border border-indigo-500/30 md:col-span-2">
<div class="font-bold text-white mb-1">Rollout Inference Cluster</div>
<div class="text-indigo-300 text-[11px]">Continuous Generation via vLLM / HFEngine</div>
</div>
<div class="p-4 rounded-lg bg-indigo-900/40 border border-indigo-400/40 md:col-span-2">
<div class="font-bold text-white mb-1">In-VRAM Reward Engine</div>
<div class="text-indigo-200 text-[11px]">Strided <code>.unfold()</code> Zero-Copy Evaluation</div>
</div>
<div class="p-4 rounded-lg bg-amber-950/40 border border-amber-500/30 md:col-span-4">
<div class="font-bold text-amber-300 mb-1">Experience Subsystem</div>
<div class="text-slate-400 text-[11px]">Bounded Buffer (FIFO Queue) • Versioned Buffer (Staleness Eviction: $\tau \le \tau_{\max}$) • Group Buffer (GRPO)</div>
</div>
<div class="p-4 rounded-lg bg-emerald-950/40 border border-emerald-500/30 md:col-span-4">
<div class="font-bold text-emerald-300 mb-1">Trainer Cluster</div>
<div class="text-emerald-200 text-[11px]">TrainerWorker (PPO / M2PO / GRPO Loss) • AdamW Optimizer ($ heta_{t+1}$) • Version Bump</div>
</div>
</div>
</div>
<!-- 4.1 In-VRAM Tensor-Native Reward -->
<div>
<h4 class="text-lg font-bold text-white mb-2">4.1 In-VRAM Tensor-Native Reward Computation</h4>
<p class="text-slate-300 text-sm leading-relaxed mb-4">
Traditional RLHF pipelines incur severe serialization overhead during rule-based reward evaluation:
$$\text{GPU IDs} \xrightarrow{\text{PCIe}} \text{CPU RAM} \xrightarrow{\text{Decode}} \text{String} \xrightarrow{\text{Regex}} \text{Score} \xrightarrow{\text{PCIe}} \text{GPU VRAM}$$
AsyncTensorRLHF executes reward verification entirely on GPU tensors via parallel 1D sliding-window convolutions.
</p>
<!-- Algorithm 1 Code Block -->
<div class="rounded-xl overflow-hidden border border-slate-800 bg-slate-900">
<div class="px-4 py-2 bg-slate-950 border-b border-slate-800 flex items-center justify-between">
<span class="text-xs font-bold text-slate-300">Algorithm 1: In-VRAM Tensor-Native Reward Evaluation</span>
<span class="text-xs text-indigo-400 font-mono">torch.Tensor Zero-Copy</span>
</div>
<pre class="p-4 text-xs font-mono text-slate-300 overflow-x-auto leading-relaxed">
<span class="text-slate-500">// 1. Initialize output reward vector on device</span>
R = torch.zeros(B, dtype=torch.float32, device=Y.device)
<span class="text-slate-500">// 2. Vectorized first-EOS boundary detection across all batch items</span>
eos_mask = (Y == eos_token_id)
has_eos = eos_mask.any(dim=-1)
first_eos = torch.where(has_eos, eos_mask.int().argmax(dim=-1), L)
<span class="text-slate-500">// 3. In-VRAM Strided 1D Convolution via .unfold() (zero memory allocation)</span>
for b in range(B):
valid_len = first_eos[b].item()
target_pattern = patterns[b]
K = len(target_pattern)
if K == 0 or K > valid_len:
continue
<span class="text-indigo-400 font-bold">windows = Y[b, :valid_len].unfold(dimension=0, size=K, step=1)</span>
match = (windows == target_pattern).all(dim=-1).any()
R[b] = 1.0 if match else 0.0
return R</pre>
</div>
</div>
</section>
<!-- SECTION 5: EXPERIMENTAL RESULTS -->
<section id="sec-eval" class="p-6 sm:p-8 rounded-2xl bg-slate-950 border border-slate-800 shadow-lg space-y-6">
<div class="border-b border-slate-800 pb-4">
<h3 class="text-2xl font-bold text-white flex items-center space-x-3">
<span class="text-indigo-400 font-mono">5.</span>
<span>Experimental Evaluation & Hardware Benchmarks</span>
</h3>
</div>
<!-- Hardware Platform -->
<div class="p-4 rounded-xl bg-slate-900 border border-slate-800 text-xs text-slate-300">
<span class="font-bold text-white block mb-1">Hardware & Software Evaluation Platform:</span>
NVIDIA GeForce RTX 4070 Laptop GPU (8GB VRAM, Compute Capability 8.9, Ada Lovelace) • PyTorch 2.6.0+cu124 • CUDA 12.4 • Python 3.13.5 • Microsoft Windows 11 Host. 100% automated test pass rate (41/41 tests in 4.00s).
</div>
<!-- Table 1 -->
<div>
<h4 class="text-base font-bold text-white mb-2">Table 1: Policy Loss Computational Throughput ($B=64, L=256$, 30 iterations)</h4>
<div class="overflow-x-auto">
<table class="w-full text-xs sm:text-sm text-left border border-slate-800 rounded-lg overflow-hidden">
<thead class="bg-slate-900 text-slate-300 font-bold uppercase tracking-wider text-[11px]">
<tr>
<th class="p-3">Objective</th>
<th class="p-3 text-center">Latency (ms)</th>
<th class="p-3 text-center">Throughput (Tokens/sec)</th>
<th class="p-3 text-center">VRAM (MB)</th>
<th class="p-3 text-center">Status</th>
</tr>
</thead>
<tbody class="divide-y divide-slate-800">
<tr class="bg-slate-950 hover:bg-slate-900/60 transition font-medium">
<td class="p-3 font-bold text-emerald-400">GRPO (Group Size G=4)</td>
<td class="p-3 text-center font-mono">2.03 ms</td>
<td class="p-3 text-center font-bold font-mono text-emerald-300">8,058,669</td>
<td class="p-3 text-center font-mono text-slate-400">1.20 MB</td>
<td class="p-3 text-center"><span class="px-2 py-0.5 rounded bg-emerald-500/20 text-emerald-300 text-xs">PASSED</span></td>
</tr>
<tr class="bg-slate-950 hover:bg-slate-900/60 transition">
<td class="p-3 font-bold text-indigo-400">PPO (Standard Clipped)</td>
<td class="p-3 text-center font-mono">2.42 ms</td>
<td class="p-3 text-center font-mono text-indigo-300">6,763,177</td>
<td class="p-3 text-center font-mono text-slate-400">1.40 MB</td>
<td class="p-3 text-center"><span class="px-2 py-0.5 rounded bg-indigo-500/20 text-indigo-300 text-xs">PASSED</span></td>
</tr>
<tr class="bg-slate-950 hover:bg-slate-900/60 transition">
<td class="p-3 font-bold text-purple-400">M2PO (Trust Region)</td>
<td class="p-3 text-center font-mono">5.98 ms</td>
<td class="p-3 text-center font-mono text-purple-300">2,739,289</td>
<td class="p-3 text-center font-mono text-slate-400">1.60 MB</td>
<td class="p-3 text-center"><span class="px-2 py-0.5 rounded bg-purple-500/20 text-purple-300 text-xs">PASSED</span></td>
</tr>
</tbody>
</table>
</div>
</div>
<!-- Table 2 & Table 3 Grid -->
<div class="grid grid-cols-1 md:grid-cols-2 gap-6">
<!-- Table 2 -->
<div>
<h4 class="text-base font-bold text-white mb-2">Table 2: Replay Buffer Concurrency Benchmark</h4>
<div class="overflow-x-auto">
<table class="w-full text-xs text-left border border-slate-800 rounded-lg overflow-hidden">
<thead class="bg-slate-900 text-slate-300 font-bold uppercase tracking-wider text-[10px]">
<tr>
<th class="p-2.5">Operation</th>
<th class="p-2.5 text-center">Throughput</th>
<th class="p-2.5 text-center">Mean Latency</th>
</tr>
</thead>
<tbody class="divide-y divide-slate-800 bg-slate-950 font-mono text-xs">
<tr>
<td class="p-2.5 font-bold text-indigo-300 font-sans">Push (BoundedBuffer)</td>
<td class="p-2.5 text-center text-emerald-400 font-bold">312,283 ops/s</td>
<td class="p-2.5 text-center text-slate-400">0.0032 ms</td>
</tr>
<tr>
<td class="p-2.5 font-bold text-indigo-300 font-sans">Sample Batch (B=64)</td>
<td class="p-2.5 text-center text-emerald-400 font-bold">411,691 items/s</td>
<td class="p-2.5 text-center text-slate-400">0.0024 ms</td>
</tr>
</tbody>
</table>
</div>
</div>
<!-- Table 3 -->
<div>
<h4 class="text-base font-bold text-white mb-2">Table 3: Off-Policy Staleness ($ au$) Variance Cut</h4>
<div class="overflow-x-auto">
<table class="w-full text-xs text-left border border-slate-800 rounded-lg overflow-hidden">
<thead class="bg-slate-900 text-slate-300 font-bold uppercase tracking-wider text-[10px]">
<tr>
<th class="p-2.5 text-center">Staleness $ au$</th>
<th class="p-2.5 text-center">PPO Norm</th>
<th class="p-2.5 text-center">M2PO Norm</th>
<th class="p-2.5 text-center">Variance Cut</th>
</tr>
</thead>
<tbody class="divide-y divide-slate-800 bg-slate-950 font-mono text-xs">
<tr><td class="p-2 text-center text-slate-400">$ au = 0$</td><td class="p-2 text-center">0.0218</td><td class="p-2 text-center">0.0218</td><td class="p-2 text-center text-slate-500">0.0%</td></tr>
<tr><td class="p-2 text-center text-slate-400">$ au = 1$</td><td class="p-2 text-center">0.0212</td><td class="p-2 text-center">0.0212</td><td class="p-2 text-center text-slate-500">0.0%</td></tr>
<tr><td class="p-2 text-center text-slate-400">$ au = 2$</td><td class="p-2 text-center">0.0202</td><td class="p-2 text-center">0.0201</td><td class="p-2 text-center text-slate-400">0.3%</td></tr>
<tr><td class="p-2 text-center text-slate-400">$ au = 3$</td><td class="p-2 text-center">0.0221</td><td class="p-2 text-center">0.0204</td><td class="p-2 text-center text-amber-400">7.7%</td></tr>
<tr><td class="p-2 text-center text-slate-400">$ au = 5$</td><td class="p-2 text-center">0.0265</td><td class="p-2 text-center">0.0198</td><td class="p-2 text-center text-amber-400 font-bold">25.6%</td></tr>
<tr class="bg-indigo-950/20"><td class="p-2 text-center font-bold text-white">$ au = 8$</td><td class="p-2 text-center text-rose-400">0.0502</td><td class="p-2 text-center text-emerald-400 font-bold">0.0190</td><td class="p-2 text-center text-emerald-300 font-bold">62.1%</td></tr>
</tbody>
</table>
</div>
</div>
</div>
<!-- Real Model Alignment Case Study -->
<div class="p-5 rounded-xl bg-purple-950/20 border border-purple-500/30">
<div class="flex items-center space-x-2 text-purple-300 font-bold mb-2">
<i data-lucide="sparkles" class="w-5 h-5"></i>
<h4>Real Model Alignment Case Study: Qwen2.5-0.5B-Instruct</h4>
</div>
<div class="grid grid-cols-1 sm:grid-cols-3 gap-4 text-xs mt-3">
<div class="p-3 rounded-lg bg-slate-950 border border-slate-800">
<span class="text-slate-400 block mb-1">LoRA Architecture</span>
<span class="font-bold text-white font-mono">r=8, alpha=16 (1.08M params, 0.22%)</span>
</div>
<div class="p-3 rounded-lg bg-slate-950 border border-slate-800">
<span class="text-slate-400 block mb-1">Training Wall-Clock Time</span>
<span class="font-bold text-emerald-400 font-mono">108.36 seconds (6 iterations)</span>
</div>
<div class="p-3 rounded-lg bg-slate-950 border border-slate-800">
<span class="text-slate-400 block mb-1">Peak VRAM Allocation</span>
<span class="font-bold text-indigo-300 font-mono">1.97 GB / 8.0 GB VRAM</span>
</div>
</div>
</div>
</section>
<!-- SECTION 6 & 7: DISCUSSION & CONCLUSION -->
<section id="sec-disc" class="p-6 sm:p-8 rounded-2xl bg-slate-950 border border-slate-800 shadow-lg space-y-4">
<div class="border-b border-slate-800 pb-4">
<h3 class="text-2xl font-bold text-white flex items-center space-x-3">
<span class="text-indigo-400 font-mono">6.</span>
<span>Discussion, Limitations & Conclusion</span>
</h3>
</div>
<div class="space-y-4 text-slate-300 text-sm sm:text-base leading-relaxed text-justify">
<p>
While AsyncTensorRLHF delivers dramatic speedups for rule-verifiable rewards, subjective alignment tasks (such as open-ended conversational helpfulness) require neural reward models (RMs). In future work, we plan to implement in-VRAM embedding cross-attention kernels to support neural RMs without host CPU serialization. Additionally, extending M2PO second-moment masking to multi-node NCCL all-reduce clusters represents a promising research avenue.
</p>
<p>
We introduced <strong>AsyncTensorRLHF</strong>, an asynchronous RLHF framework that eliminates the CPU-GPU memory wall via in-VRAM tensor-native rewards and overcomes the synchronous lockstep barrier through decoupled rollout-trainer orchestration. By incorporating M2PO second-moment trust region bounds, AsyncTensorRLHF guarantees numerical stability across off-policy staleness drift, cutting gradient variance by up to 62.1% while delivering over 8 million tokens/second under GRPO. The complete ecosystem—including source code, unit test suites, fine-tuned Qwen2.5 checkpoints, and an interactive web explorer—is open-sourced for the community.
</p>
</div>
</section>
<!-- SECTION 8: REFERENCES -->
<section id="sec-ref" class="p-6 sm:p-8 rounded-2xl bg-slate-950 border border-slate-800 shadow-lg space-y-4">
<div class="border-b border-slate-800 pb-4">
<h3 class="text-xl font-bold text-white flex items-center space-x-3">
<span class="text-indigo-400 font-mono">§</span>
<span>References</span>
</h3>
</div>
<ol class="space-y-3 text-xs text-slate-400 leading-relaxed font-sans">
<li><strong class="text-slate-200">[1] J. Schulman, et al.</strong> "Proximal policy optimization algorithms." <em>arXiv:1707.06347</em>, 2017.</li>
<li><strong class="text-slate-200">[2] L. Ouyang, et al.</strong> "Training language models to follow instructions with human feedback." <em>NeurIPS</em>, 2022.</li>
<li><strong class="text-slate-200">[3] Z. Shao, et al.</strong> "DeepSeekMath: Pushing the limits of mathematical reasoning in open language models." <em>arXiv:2402.03300</em>, 2024.</li>
<li><strong class="text-slate-200">[4] DeepSeek-AI, et al.</strong> "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning." <em>arXiv:2501.12948</em>, 2025.</li>
<li><strong class="text-slate-200">[5] W. Kwon, et al.</strong> "Efficient memory management for large language model serving with PagedAttention." <em>SOSP</em>, 2023.</li>
<li><strong class="text-slate-200">[6] L. Zheng, et al.</strong> "SGLang: Efficient Execution of Structured Language Model Programs." <em>arXiv:2312.07104</em>, 2023.</li>
<li><strong class="text-slate-200">[7] Y. Sheng, et al.</strong> "HybridFlow: A flexible and efficient RLHF framework." <em>arXiv:2409.19256</em>, 2024.</li>
<li><strong class="text-slate-200">[8] E. J. Hu, et al.</strong> "LoRA: Low-rank adaptation of large language models." <em>ICLR</em>, 2022.</li>
<li><strong class="text-slate-200">[9] Qwen Team.</strong> "Qwen2.5 Technical Report." <em>arXiv:2412.15115</em>, 2024.</li>
<li><strong class="text-slate-200">[10] V. Mnih, et al.</strong> "Asynchronous methods for deep reinforcement learning." <em>ICML</em>, 2016.</li>
<li><strong class="text-slate-200">[11] L. Espeholt, et al.</strong> "IMPALA: Scalable distributed deep-RL with importance weighted actor-learner architectures." <em>ICML</em>, 2018.</li>
<li><strong class="text-slate-200">[12] T. Dao, et al.</strong> "FlashAttention: Fast and memory-efficient exact attention with IO-awareness." <em>NeurIPS</em>, 2022.</li>
<li><strong class="text-slate-200">[13] J. Schulman, et al.</strong> "Trust region policy optimization." <em>ICML</em>, 2015.</li>
<li><strong class="text-slate-200">[14] I. Loshchilov and F. Hutter.</strong> "Decoupled weight decay regularization." <em>ICLR</em>, 2019.</li>
<li><strong class="text-slate-200">[15] A. Paszke, et al.</strong> "PyTorch: An imperative style, high-performance deep learning library." <em>NeurIPS</em>, 2019.</li>
</ol>
</section>
</div>
<!-- VIEW 2: ORIGINAL PDF EMBEDDED VIEWER -->
<div id="paper-view-pdf" class="hidden p-6 rounded-2xl bg-slate-950 border border-slate-800 shadow-2xl">
<div class="flex items-center justify-between mb-4">
<div class="flex items-center space-x-2">
<i data-lucide="file" class="w-4 h-4 text-indigo-400"></i>
<h4 class="font-bold text-slate-200 text-sm uppercase tracking-wider">Compiled Document Preview</h4>
</div>
<span class="text-xs text-slate-400 font-mono">6 Pages • 409 KB PDF</span>
</div>
<div class="w-full h-[850px] rounded-xl overflow-hidden border border-slate-800 bg-slate-900 shadow-inner">
<iframe src="paper.pdf" class="w-full h-full" title="AsyncTensorRLHF Paper PDF"></iframe>
</div>
</div>
</div>
</div>
</div>
</main>
<footer class="border-t border-slate-800/80 py-8 text-center text-xs text-slate-500">
<div class="max-w-7xl mx-auto px-4 flex flex-col sm:flex-row items-center justify-between gap-4">
<div>Released under the Apache 2.0 License. Maintained by <a href="https://huggingface.co/Hooshaai" class="text-slate-400 hover:text-white">Hooshaai</a>.</div>
<div class="flex items-center space-x-4 font-mono">
<span>PyTorch 2.6+</span>
<span>CUDA 12.4</span>
<span>Python 3.10+</span>
</div>
</div>
</footer>
<script>
lucide.createIcons();
function togglePaperView(mode) {
const htmlView = document.getElementById('paper-view-html');
const pdfView = document.getElementById('paper-view-pdf');
const btnHtml = document.getElementById('btn-paper-html');
const btnPdf = document.getElementById('btn-paper-pdf');
if (mode === 'html') {
htmlView.classList.remove('hidden');
pdfView.classList.add('hidden');
btnHtml.className = 'px-3.5 py-1.5 rounded-lg text-xs font-bold bg-indigo-600 text-white shadow-sm flex items-center space-x-2 transition';
btnPdf.className = 'px-3.5 py-1.5 rounded-lg text-xs font-bold text-slate-400 hover:text-white flex items-center space-x-2 transition';
if (window.renderMathInElement) {
renderMathInElement(htmlView, {
delimiters: [
{ left: '$$', right: '$$', display: true },
{ left: '$', right: '$', display: false }
],
throwOnError: false
});
}
} else {
htmlView.classList.add('hidden');
pdfView.classList.remove('hidden');
btnPdf.className = 'px-3.5 py-1.5 rounded-lg text-xs font-bold bg-indigo-600 text-white shadow-sm flex items-center space-x-2 transition';
btnHtml.className = 'px-3.5 py-1.5 rounded-lg text-xs font-bold text-slate-400 hover:text-white flex items-center space-x-2 transition';
}
lucide.createIcons();
}
function copyBibtex() {
const bibtex = `@article{majlesi2026asynctensorrlhf,
title = {AsyncTensorRLHF: High-Throughput Asynchronous RLHF with In-VRAM Tensor-Native Rewards},
author = {Majlesi, Taha},
journal = {arXiv preprint arXiv:2603.XXXXX},
year = {2026},
url = {https://github.com/Hooshaai/AsyncTensorRLHF}
}`;
navigator.clipboard.writeText(bibtex).then(() => {
const btn = document.getElementById('copy-btn-text');
btn.innerText = 'Copied to Clipboard!';
setTimeout(() => { btn.innerText = 'Copy BibTeX Citation'; }, 2000);
});
}
function switchTab(tabId) {
document.querySelectorAll('.tab-content').forEach(el => el.classList.add('hidden'));
document.querySelectorAll('.tab-btn').forEach(btn => {
btn.classList.remove('bg-indigo-600', 'text-white');
btn.classList.add('text-slate-400');
});
document.getElementById(tabId).classList.remove('hidden');
const activeBtn = document.getElementById('btn-' + tabId);
activeBtn.classList.add('bg-indigo-600', 'text-white');
activeBtn.classList.remove('text-slate-400');
}
function setSample(type) {
if (type === 'match') {
document.getElementById('input-seq').value = '10, 20, 30, 40, 99, 50, 60';
document.getElementById('input-pat').value = '20, 30, 40';
} else {
document.getElementById('input-seq').value = '10, 20, 99, 30, 40, 50';
document.getElementById('input-pat').value = '30, 40';
}
runRewardSimulation();
}
function runRewardSimulation() {
const seqRaw = document.getElementById('input-seq').value.split(',').map(s => parseInt(s.trim())).filter(n => !isNaN(n));
const patRaw = document.getElementById('input-pat').value.split(',').map(s => parseInt(s.trim())).filter(n => !isNaN(n));
const eosId = 99;
const firstEosIdx = seqRaw.indexOf(eosId);
const validEnd = firstEosIdx === -1 ? seqRaw.length : firstEosIdx;
let isMatch = false;
let matchIdx = -1;
if (patRaw.length > 0 && patRaw.length <= validEnd) {
for (let i = 0; i <= validEnd - patRaw.length; i++) {
let sub = seqRaw.slice(i, i + patRaw.length);
if (sub.every((val, idx) => val === patRaw[idx])) {
isMatch = true;
matchIdx = i;
break;
}
}
}
// Render tape
const tape = document.getElementById('token-tape');
tape.innerHTML = '';
seqRaw.forEach((token, idx) => {
const span = document.createElement('span');
span.className = 'px-2 py-1 rounded border ';
if (token === eosId) {
span.className += 'bg-red-500/20 text-red-400 border-red-500/40 font-bold';
span.innerText = `${token} [EOS]`;
} else if (idx >= validEnd) {
span.className += 'bg-slate-900 text-slate-600 border-slate-800 line-through';
span.innerText = token;
} else if (isMatch && idx >= matchIdx && idx < matchIdx + patRaw.length) {
span.className += 'bg-emerald-500/20 text-emerald-300 border-emerald-500/50 font-bold';
span.innerText = token;
} else {
span.className += 'bg-slate-900 text-indigo-300 border-slate-800';
span.innerText = token;
}
tape.appendChild(span);
});
// Update badge
const badge = document.getElementById('reward-badge');
if (isMatch) {
badge.className = 'px-2.5 py-0.5 rounded-full text-xs font-bold bg-emerald-500/20 text-emerald-400 border border-emerald-500/30';
badge.innerText = 'Reward: 1.0 (Match)';
} else {
badge.className = 'px-2.5 py-0.5 rounded-full text-xs font-bold bg-rose-500/20 text-rose-400 border border-rose-500/30';
badge.innerText = 'Reward: 0.0 (No Match)';
}
// Update details
const details = document.getElementById('window-details');
details.innerHTML = `
<div>• Sequence length before EOS: <strong>${validEnd}</strong> tokens</div>
<div>• Pattern length: <strong>${patRaw.length}</strong> tokens</div>
<div>• Sliding window evaluations: <strong>${Math.max(0, validEnd - patRaw.length + 1)}</strong> checks</div>
<div>• Subsequence match located at index: <strong>${matchIdx !== -1 ? matchIdx : 'None'}</strong></div>
`;
}
// Chart.js Graphs
let lossChartInstance = null;
function initLossGraph() {
const ctx = document.getElementById('lossChart').getContext('2d');
const ratios = [];
for (let r = 0.5; r <= 2.5; r += 0.05) ratios.push(r);
lossChartInstance = new Chart(ctx, {
type: 'line',
data: {
labels: ratios.map(r => r.toFixed(2)),
datasets: [
{ label: 'Standard PPO Objective', data: [], borderColor: '#6366f1', borderWidth: 2, tension: 0.3 },
{ label: 'M2PO Constrained Objective', data: [], borderColor: '#10b981', borderWidth: 2, borderDash: [5, 5], tension: 0.3 }
]
},
options: {
responsive: true,
maintainAspectRatio: false,
scales: {
x: { title: { display: true, text: 'Importance Sampling Ratio r(θ)', color: '#94a3b8' }, grid: { color: '#1e293b' } },
y: { title: { display: true, text: 'Surrogate Objective Value', color: '#94a3b8' }, grid: { color: '#1e293b' } }
},
plugins: { legend: { labels: { color: '#cbd5e1' } } }
}
});
updateLossGraph();
}
function updateLossGraph() {
const eps = parseFloat(document.getElementById('slider-eps').value);
const m2 = parseFloat(document.getElementById('slider-m2').value);
const tau = parseInt(document.getElementById('slider-tau').value);
document.getElementById('label-eps').innerText = eps.toFixed(2);
document.getElementById('label-m2').innerText = m2.toFixed(1);
document.getElementById('label-tau').innerText = `${tau} versions`;
const ppoData = [];
const m2poData = [];
lossChartInstance.data.labels.forEach(label => {
const r = parseFloat(label);
const adv = 1.0;
const unclipped = r * adv;
const clipped = Math.min(Math.max(r, 1 - eps), 1 + eps) * adv;
const ppoVal = Math.min(unclipped, clipped);
// M2PO constraint: if r^2 > m2, penalized / masked
let m2poVal = ppoVal;
if (r * r > m2) {
m2poVal = ppoVal * Math.exp(-0.5 * (r * r - m2));
}
ppoData.push(ppoVal);
m2poData.push(m2poVal);
});
lossChartInstance.data.datasets[0].data = ppoData;
lossChartInstance.data.datasets[1].data = m2poData;
lossChartInstance.update();
}
function initBenchmarkCharts() {
// Throughput
const ctx1 = document.getElementById('throughputChart').getContext('2d');
new Chart(ctx1, {
type: 'bar',
data: {
labels: ['GRPO (G=4)', 'Standard PPO', 'M2PO Trust Region'],
datasets: [{
label: 'Tokens / sec',
data: [8058669, 6763177, 2739289],
backgroundColor: ['#10b981', '#6366f1', '#a855f7']
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: { legend: { display: false } },
scales: {
x: { grid: { color: '#1e293b' }, ticks: { color: '#cbd5e1' } },
y: { grid: { color: '#1e293b' }, ticks: { color: '#cbd5e1' } }
}
}
});
// Variance reduction
const ctx2 = document.getElementById('varianceChart').getContext('2d');
new Chart(ctx2, {
type: 'line',
data: {
labels: ['tau=0', 'tau=1', 'tau=2', 'tau=3', 'tau=5', 'tau=8'],
datasets: [{
label: 'Variance Reduction (%)',
data: [0.0, 0.0, 0.3, 7.7, 25.6, 62.1],
borderColor: '#f59e0b',
backgroundColor: 'rgba(245, 158, 11, 0.1)',
fill: true,
tension: 0.3,
borderWidth: 2
}]
},
options: {
responsive: true,
maintainAspectRatio: false,
plugins: { legend: { labels: { color: '#cbd5e1' } } },
scales: {
x: { grid: { color: '#1e293b' }, ticks: { color: '#cbd5e1' } },
y: { grid: { color: '#1e293b' }, ticks: { color: '#cbd5e1' }, title: { display: true, text: 'Variance Cut (%)', color: '#94a3b8' } }
}
}
});
}
window.onload = () => {
runRewardSimulation();
initLossGraph();
initBenchmarkCharts();
};
</script>
</body>
</html>
|