Upload source/files/YARNOps.cpp with huggingface_hub
Browse files- source/files/YARNOps.cpp +445 -0
source/files/YARNOps.cpp
ADDED
|
@@ -0,0 +1,445 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
#include "autograd/operations/RoPEOps.h"
|
| 3 |
+
#include "autograd/backward/RoPEBackward.h"
|
| 4 |
+
#include "autograd/AutogradContext.h"
|
| 5 |
+
#include "autograd/Variable.h"
|
| 6 |
+
#include "autograd/GraphRecorder.h"
|
| 7 |
+
#include "checkpointing/GradMode.h"
|
| 8 |
+
#include "ops/helpers/RoPEKernels.h"
|
| 9 |
+
#include "autograd/ops_template.h"
|
| 10 |
+
|
| 11 |
+
#include <stdexcept>
|
| 12 |
+
#include <cmath>
|
| 13 |
+
#include <cstdlib> // std::getenv, std::atof
|
| 14 |
+
#include <cstdio> // std::fprintf
|
| 15 |
+
#include "CPLog.h" // cplog::log_rank() — gate one-time notices to rank 0
|
| 16 |
+
#include <string> // std::to_string
|
| 17 |
+
#include <algorithm> // std::max, std::min
|
| 18 |
+
#include <vector> // std::vector (LongRoPE lambda)
|
| 19 |
+
#include "context_parallel/LongRoPEOps.h" // build_rope_cache_longrope declaration
|
| 20 |
+
|
| 21 |
+
namespace OwnTensor {
|
| 22 |
+
namespace autograd {
|
| 23 |
+
|
| 24 |
+
// ============================================================================
|
| 25 |
+
// build_rope_cache — YaRN (NTK-by-parts + attention temperature) cos/sin cache
|
| 26 |
+
//
|
| 27 |
+
// Canonical HF/Qwen3 YaRN. Extends context (e.g. 1024 -> 8192) by blending
|
| 28 |
+
// per-dimension inverse frequencies (extrapolate high-freq, interpolate low-freq
|
| 29 |
+
// via a dimension-index linear ramp between correction dims) and baking the
|
| 30 |
+
// attention temperature m = 0.1*ln(s)+1 into BOTH cos and sin. RoPE hits both Q
|
| 31 |
+
// and K, so m appears twice in each QK logit (m^2 = 1/t), matching YaRN's
|
| 32 |
+
// softmax temperature; the fused kernel's 1/sqrt(hd) scale is left untouched.
|
| 33 |
+
//
|
| 34 |
+
// Params are hardcoded defaults, each overridable by an env var. Default
|
| 35 |
+
// YARN_SCALE=1.0 reduces this to standard RoPE bit-identically (base_freq/s ==
|
| 36 |
+
// base_freq for all i, and m==1), preserving all existing callers.
|
| 37 |
+
//
|
| 38 |
+
// The cache is always built on CPU (fp32) then copied to the target device: the
|
| 39 |
+
// device kernel rope_build_cache_cuda has no per-dimension frequency hook, so we
|
| 40 |
+
// bypass it. Cost is negligible (built once at model init).
|
| 41 |
+
// ============================================================================
|
| 42 |
+
namespace {
|
| 43 |
+
inline float yarn_env_f(const char* key, float dflt) {
|
| 44 |
+
const char* v = std::getenv(key);
|
| 45 |
+
return v ? static_cast<float>(std::atof(v)) : dflt;
|
| 46 |
+
}
|
| 47 |
+
} // anonymous namespace
|
| 48 |
+
|
| 49 |
+
Tensor build_rope_cache(int64_t seq_len, int64_t head_dim, float base, DeviceIndex device)
|
| 50 |
+
{
|
| 51 |
+
if (head_dim % 2 != 0) {
|
| 52 |
+
throw std::invalid_argument("build_rope_cache: head_dim must be even");
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
// --- YaRN config (env-overridable; scale=1.0 => standard RoPE) ---
|
| 56 |
+
const float s = yarn_env_f("YARN_SCALE", 1.0f); // target_ctx / original_ctx
|
| 57 |
+
const float L = yarn_env_f("YARN_ORIG_MAXPOS", 1024.0f);// original pretrain context
|
| 58 |
+
const float b_fast = yarn_env_f("YARN_BETA_FAST", 32.0f); // "beta" (high-rotation bound)
|
| 59 |
+
const float b_slow = yarn_env_f("YARN_BETA_SLOW", 1.0f); // "alpha" (low-rotation bound)
|
| 60 |
+
|
| 61 |
+
// Attention temperature m (s==1 -> m==1 -> no scaling). YARN_NO_MSCALE=1
|
| 62 |
+
// forces m=1 for the fair-comparison "YaRN without temperature" arm of the
|
| 63 |
+
// LongRoPE study (default keeps m, so unset behavior is byte-identical).
|
| 64 |
+
const bool yarn_no_m = std::getenv("YARN_NO_MSCALE") != nullptr &&
|
| 65 |
+
std::atoi(std::getenv("YARN_NO_MSCALE")) != 0;
|
| 66 |
+
const float m = (!yarn_no_m && s > 1.0f) ? (0.1f * std::log(s) + 1.0f) : 1.0f;
|
| 67 |
+
|
| 68 |
+
const int64_t half = head_dim / 2;
|
| 69 |
+
|
| 70 |
+
// Canonical correction range: linear ramp over dimension index between the
|
| 71 |
+
// dims where the number of rotations equals b_fast (high-freq end) and
|
| 72 |
+
// b_slow (low-freq end). Matches HF find_correction_dim/find_correction_range.
|
| 73 |
+
auto find_dim = [&](float num_rot) {
|
| 74 |
+
return (static_cast<float>(head_dim)
|
| 75 |
+
* std::log(L / (num_rot * 2.0f * static_cast<float>(M_PI))))
|
| 76 |
+
/ (2.0f * std::log(base));
|
| 77 |
+
};
|
| 78 |
+
int64_t low = static_cast<int64_t>(std::floor(find_dim(b_fast)));
|
| 79 |
+
int64_t high = static_cast<int64_t>(std::ceil (find_dim(b_slow)));
|
| 80 |
+
low = std::max<int64_t>(low, 0);
|
| 81 |
+
high = std::min<int64_t>(high, half - 1); // clamp to half-1: loop indexes [0,half) only;
|
| 82 |
+
// deliberate deviation from HF head_dim-1 -- do not "fix"
|
| 83 |
+
float denom = static_cast<float>(high - low);
|
| 84 |
+
if (denom <= 0.0f) denom = 1e-3f; // avoid /0 when low==high
|
| 85 |
+
|
| 86 |
+
// (R5) Guard degenerate env input: L/(num_rot*2pi) <= 0 -> log(neg) = NaN.
|
| 87 |
+
if (s > 1.0f && (!std::isfinite(find_dim(b_fast)) || !std::isfinite(find_dim(b_slow)) || low > high)) {
|
| 88 |
+
throw std::runtime_error(
|
| 89 |
+
"build_rope_cache(YaRN): bad correction range -- check "
|
| 90 |
+
"YARN_BETA_FAST/YARN_BETA_SLOW/YARN_ORIG_MAXPOS (got low="
|
| 91 |
+
+ std::to_string(low) + " high=" + std::to_string(high) + ")");
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
// (R2) scale <-> seq_len consistency (warn, don't hard-fail).
|
| 95 |
+
if (s > 1.0f && std::abs(static_cast<float>(seq_len) - s * L) > 1.0f) {
|
| 96 |
+
if (cplog::log_rank()) std::fprintf(stderr,
|
| 97 |
+
"[YaRN][warn] seq_len(%lld) != scale(%.3f)*orig_maxpos(%.1f)=%.1f "
|
| 98 |
+
"-- cache scaled for a different context than it is sized for\n",
|
| 99 |
+
(long long)seq_len, s, L, s * L);
|
| 100 |
+
}
|
| 101 |
+
|
| 102 |
+
// (R3) Init log: makes the active config + the YARNOps-symbol-wins link
|
| 103 |
+
// dependency visible. Missing line / scale=1 => silent revert to std RoPE.
|
| 104 |
+
if (cplog::log_rank()) std::fprintf(stderr,
|
| 105 |
+
"[YaRN] build_rope_cache: seq_len=%lld hd=%lld scale=%.3f low=%lld high=%lld m=%.4f\n",
|
| 106 |
+
(long long)seq_len, (long long)head_dim, s, (long long)low, (long long)high, m);
|
| 107 |
+
|
| 108 |
+
// --- Build on CPU, fill, then copy to device (bypass the CUDA kernel). ---
|
| 109 |
+
TensorOptions cpu_opts = TensorOptions().with_dtype(Dtype::Float32)
|
| 110 |
+
.with_device(DeviceIndex(Device::CPU));
|
| 111 |
+
Tensor cpu(Shape{{seq_len, head_dim}}, cpu_opts);
|
| 112 |
+
float* p = cpu.data<float>();
|
| 113 |
+
|
| 114 |
+
for (int64_t pos = 0; pos < seq_len; ++pos) {
|
| 115 |
+
for (int64_t i = 0; i < half; ++i) {
|
| 116 |
+
float exponent = (2.0f * static_cast<float>(i)) / static_cast<float>(head_dim);
|
| 117 |
+
float base_freq = 1.0f / std::pow(base, exponent); // extrapolation freq
|
| 118 |
+
|
| 119 |
+
float ramp = (static_cast<float>(i) - static_cast<float>(low)) / denom;
|
| 120 |
+
ramp = ramp < 0.0f ? 0.0f : (ramp > 1.0f ? 1.0f : ramp);
|
| 121 |
+
// ramp=0 (i<low, high-freq) -> keep base_freq (extrapolate)
|
| 122 |
+
// ramp=1 (i>high, low-freq) -> base_freq/s (interpolate)
|
| 123 |
+
float inv_freq = (base_freq / s) * ramp + base_freq * (1.0f - ramp);
|
| 124 |
+
|
| 125 |
+
float angle = static_cast<float>(pos) * inv_freq;
|
| 126 |
+
p[pos * head_dim + i] = std::cos(angle) * m; // bake temperature m
|
| 127 |
+
p[pos * head_dim + i + half] = std::sin(angle) * m;
|
| 128 |
+
}
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
cpu.set_requires_grad(false);
|
| 132 |
+
if (device.device == Device::CUDA) {
|
| 133 |
+
Tensor cache = cpu.to(device);
|
| 134 |
+
cache.set_requires_grad(false);
|
| 135 |
+
return cache;
|
| 136 |
+
}
|
| 137 |
+
return cpu;
|
| 138 |
+
}
|
| 139 |
+
|
| 140 |
+
// ============================================================================
|
| 141 |
+
// build_rope_cache_longrope — LongRoPE (searched per-dim lambda + n_hat) cache.
|
| 142 |
+
//
|
| 143 |
+
// angle(n,i) = n * base_freq_i / (n < n_hat ? 1 : lambda_i). NeoX half-split
|
| 144 |
+
// layout identical to build_rope_cache above (cos in [0,half), sin in
|
| 145 |
+
// [half,head_dim)). NO YaRN attention-temperature m. lambda has length
|
| 146 |
+
// head_dim/2, monotone non-decreasing, each >= 1 (the search enforces this; we
|
| 147 |
+
// re-assert defensively). Built on CPU fp32 then copied to device, grad-free.
|
| 148 |
+
// New, additive symbol -- does not touch build_rope_cache or any caller.
|
| 149 |
+
// ============================================================================
|
| 150 |
+
namespace {
|
| 151 |
+
|
| 152 |
+
// One [seq_len, head_dim] LongRoPE plane, NeoX half-split (cos in [0,half),
|
| 153 |
+
// sin in [half,head_dim)). Extracted verbatim from build_rope_cache_longrope so
|
| 154 |
+
// the per-group builder cannot drift from the single-plane one.
|
| 155 |
+
void fill_longrope_plane(float* p, int64_t seq_len, int64_t head_dim, float base,
|
| 156 |
+
const std::vector<float>& lambda, int64_t n_hat, float mscale)
|
| 157 |
+
{
|
| 158 |
+
const int64_t half = head_dim / 2;
|
| 159 |
+
for (int64_t pos = 0; pos < seq_len; ++pos) {
|
| 160 |
+
const bool preserved = (pos < n_hat); // initial tokens keep original positions
|
| 161 |
+
for (int64_t i = 0; i < half; ++i) {
|
| 162 |
+
float exponent = (2.0f * static_cast<float>(i)) / static_cast<float>(head_dim);
|
| 163 |
+
float base_freq = 1.0f / std::pow(base, exponent);
|
| 164 |
+
float scale = preserved ? 1.0f : lambda[static_cast<size_t>(i)]; // lambda divides freq
|
| 165 |
+
float angle = static_cast<float>(pos) * base_freq / scale;
|
| 166 |
+
p[pos * head_dim + i] = std::cos(angle) * mscale; // NeoX half-split; mscale=1 => no temp
|
| 167 |
+
p[pos * head_dim + i + half] = std::sin(angle) * mscale;
|
| 168 |
+
}
|
| 169 |
+
}
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
// Shared shape/monotonicity validation. `tag` names the caller in the message.
|
| 173 |
+
void validate_longrope_lambda(const std::vector<float>& lambda, int64_t half,
|
| 174 |
+
const char* tag)
|
| 175 |
+
{
|
| 176 |
+
if (static_cast<int64_t>(lambda.size()) != half) {
|
| 177 |
+
throw std::invalid_argument(
|
| 178 |
+
std::string(tag) + ": lambda size (" + std::to_string(lambda.size()) +
|
| 179 |
+
") must equal head_dim/2 (" + std::to_string(half) + ")");
|
| 180 |
+
}
|
| 181 |
+
for (int64_t i = 0; i < half; ++i) {
|
| 182 |
+
if (!(lambda[static_cast<size_t>(i)] >= 1.0f)) {
|
| 183 |
+
throw std::invalid_argument(
|
| 184 |
+
std::string(tag) + ": lambda[" + std::to_string(i) + "] must be >= 1");
|
| 185 |
+
}
|
| 186 |
+
if (i > 0 && lambda[static_cast<size_t>(i)] < lambda[static_cast<size_t>(i - 1)]) {
|
| 187 |
+
throw std::invalid_argument(
|
| 188 |
+
std::string(tag) + ": lambda must be non-decreasing (violated at i=" +
|
| 189 |
+
std::to_string(i) + ")");
|
| 190 |
+
}
|
| 191 |
+
}
|
| 192 |
+
}
|
| 193 |
+
|
| 194 |
+
} // namespace
|
| 195 |
+
|
| 196 |
+
Tensor build_rope_cache_longrope(int64_t seq_len, int64_t head_dim, float base,
|
| 197 |
+
const std::vector<float>& lambda, int64_t n_hat,
|
| 198 |
+
DeviceIndex device, float mscale)
|
| 199 |
+
{
|
| 200 |
+
if (head_dim % 2 != 0) {
|
| 201 |
+
throw std::invalid_argument("build_rope_cache_longrope: head_dim must be even");
|
| 202 |
+
}
|
| 203 |
+
const int64_t half = head_dim / 2;
|
| 204 |
+
validate_longrope_lambda(lambda, half, "build_rope_cache_longrope");
|
| 205 |
+
if (n_hat < 0) {
|
| 206 |
+
throw std::invalid_argument("build_rope_cache_longrope: n_hat must be >= 0");
|
| 207 |
+
}
|
| 208 |
+
|
| 209 |
+
if (cplog::log_rank()) std::fprintf(stderr,
|
| 210 |
+
"[LongRoPE] build_rope_cache_longrope: seq_len=%lld hd=%lld n_hat=%lld "
|
| 211 |
+
"lam0=%.4f lam_last=%.4f mscale=%.4f\n",
|
| 212 |
+
(long long)seq_len, (long long)head_dim, (long long)n_hat,
|
| 213 |
+
(double)lambda[0], (double)lambda[static_cast<size_t>(half - 1)], (double)mscale);
|
| 214 |
+
|
| 215 |
+
TensorOptions cpu_opts = TensorOptions().with_dtype(Dtype::Float32)
|
| 216 |
+
.with_device(DeviceIndex(Device::CPU));
|
| 217 |
+
Tensor cpu(Shape{{seq_len, head_dim}}, cpu_opts);
|
| 218 |
+
float* p = cpu.data<float>();
|
| 219 |
+
|
| 220 |
+
fill_longrope_plane(p, seq_len, head_dim, base, lambda, n_hat, mscale);
|
| 221 |
+
|
| 222 |
+
cpu.set_requires_grad(false);
|
| 223 |
+
if (device.device == Device::CUDA) {
|
| 224 |
+
Tensor cache = cpu.to(device);
|
| 225 |
+
cache.set_requires_grad(false);
|
| 226 |
+
return cache;
|
| 227 |
+
}
|
| 228 |
+
return cpu;
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
// ---------------------------------------------------------------------------
|
| 232 |
+
// Ms-PoE per-KV-group cache: [n_groups, seq_len, head_dim]. Plane g is exactly
|
| 233 |
+
// build_rope_cache_longrope(lambdas[g]). See LongRoPEOps.h for why the ratio is
|
| 234 |
+
// per KV GROUP and not per head.
|
| 235 |
+
// ---------------------------------------------------------------------------
|
| 236 |
+
Tensor build_rope_cache_longrope_grouped(
|
| 237 |
+
int64_t seq_len, int64_t head_dim, float base,
|
| 238 |
+
const std::vector<std::vector<float>>& lambdas, int64_t n_hat,
|
| 239 |
+
DeviceIndex device, float mscale)
|
| 240 |
+
{
|
| 241 |
+
if (head_dim % 2 != 0) {
|
| 242 |
+
throw std::invalid_argument(
|
| 243 |
+
"build_rope_cache_longrope_grouped: head_dim must be even");
|
| 244 |
+
}
|
| 245 |
+
if (lambdas.empty()) {
|
| 246 |
+
throw std::invalid_argument(
|
| 247 |
+
"build_rope_cache_longrope_grouped: needs >= 1 lambda group");
|
| 248 |
+
}
|
| 249 |
+
if (n_hat < 0) {
|
| 250 |
+
throw std::invalid_argument(
|
| 251 |
+
"build_rope_cache_longrope_grouped: n_hat must be >= 0");
|
| 252 |
+
}
|
| 253 |
+
const int64_t half = head_dim / 2;
|
| 254 |
+
const int64_t n_groups = static_cast<int64_t>(lambdas.size());
|
| 255 |
+
for (int64_t g = 0; g < n_groups; ++g) {
|
| 256 |
+
validate_longrope_lambda(lambdas[static_cast<size_t>(g)], half,
|
| 257 |
+
"build_rope_cache_longrope_grouped");
|
| 258 |
+
}
|
| 259 |
+
|
| 260 |
+
if (cplog::log_rank()) {
|
| 261 |
+
std::fprintf(stderr,
|
| 262 |
+
"[Ms-PoE] build_rope_cache_longrope_grouped: groups=%lld seq_len=%lld "
|
| 263 |
+
"hd=%lld n_hat=%lld mscale=%.4f\n",
|
| 264 |
+
(long long)n_groups, (long long)seq_len, (long long)head_dim,
|
| 265 |
+
(long long)n_hat, (double)mscale);
|
| 266 |
+
for (int64_t g = 0; g < n_groups; ++g) {
|
| 267 |
+
const std::vector<float>& lam = lambdas[static_cast<size_t>(g)];
|
| 268 |
+
std::fprintf(stderr, "[Ms-PoE] group %lld: lam0=%.4f lam_last=%.4f\n",
|
| 269 |
+
(long long)g, (double)lam[0],
|
| 270 |
+
(double)lam[static_cast<size_t>(half - 1)]);
|
| 271 |
+
}
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
TensorOptions cpu_opts = TensorOptions().with_dtype(Dtype::Float32)
|
| 275 |
+
.with_device(DeviceIndex(Device::CPU));
|
| 276 |
+
Tensor cpu(Shape{{n_groups, seq_len, head_dim}}, cpu_opts);
|
| 277 |
+
float* p = cpu.data<float>();
|
| 278 |
+
|
| 279 |
+
const int64_t plane = seq_len * head_dim;
|
| 280 |
+
for (int64_t g = 0; g < n_groups; ++g) {
|
| 281 |
+
fill_longrope_plane(p + g * plane, seq_len, head_dim, base,
|
| 282 |
+
lambdas[static_cast<size_t>(g)], n_hat, mscale);
|
| 283 |
+
}
|
| 284 |
+
|
| 285 |
+
cpu.set_requires_grad(false);
|
| 286 |
+
if (device.device == Device::CUDA) {
|
| 287 |
+
Tensor cache = cpu.to(device);
|
| 288 |
+
cache.set_requires_grad(false);
|
| 289 |
+
return cache;
|
| 290 |
+
}
|
| 291 |
+
return cpu;
|
| 292 |
+
}
|
| 293 |
+
|
| 294 |
+
// ============================================================================
|
| 295 |
+
// rope — separate-tensor forward
|
| 296 |
+
// Operates on x of shape [B, T, H, D]; returns rotated tensor (out-of-place).
|
| 297 |
+
// ============================================================================
|
| 298 |
+
Tensor rope(const Tensor& x,
|
| 299 |
+
const Tensor& cos_sin_cache,
|
| 300 |
+
bool interleaved,
|
| 301 |
+
int64_t position_offset)
|
| 302 |
+
{
|
| 303 |
+
GraphRecordMode::record_forward("ROPE: rope");
|
| 304 |
+
|
| 305 |
+
if (x.ndim() != 4) {
|
| 306 |
+
throw std::invalid_argument("rope: input must be 4-D [B, T, H, D]");
|
| 307 |
+
}
|
| 308 |
+
if (x.dtype() != Dtype::Float32 || cos_sin_cache.dtype() != Dtype::Float32) {
|
| 309 |
+
throw std::invalid_argument("rope: only fp32 is supported in v1");
|
| 310 |
+
}
|
| 311 |
+
|
| 312 |
+
int64_t B = x.shape().dims[0];
|
| 313 |
+
int64_t T = x.shape().dims[1];
|
| 314 |
+
int64_t H = x.shape().dims[2];
|
| 315 |
+
int64_t D = x.shape().dims[3];
|
| 316 |
+
if (D % 2 != 0) throw std::invalid_argument("rope: head_dim must be even");
|
| 317 |
+
|
| 318 |
+
// Cache is [seq_len, head_dim]; every token's pos = t + position_offset must
|
| 319 |
+
// index a valid row. Validate before launch so we fail with a clear message
|
| 320 |
+
// instead of an out-of-bounds device read.
|
| 321 |
+
int64_t cache_seq_len = cos_sin_cache.shape().dims[0];
|
| 322 |
+
if (T + position_offset > cache_seq_len) {
|
| 323 |
+
throw std::invalid_argument(
|
| 324 |
+
"rope: T + position_offset (" + std::to_string(T + position_offset) +
|
| 325 |
+
") exceeds cos_sin_cache seq_len (" + std::to_string(cache_seq_len) +
|
| 326 |
+
"); rebuild cache with a larger seq_len");
|
| 327 |
+
}
|
| 328 |
+
|
| 329 |
+
// Out-of-place: clone input, rotate the clone in place. Clone is
|
| 330 |
+
// contiguous so strides are simple.
|
| 331 |
+
Tensor out = x.contiguous().clone();
|
| 332 |
+
|
| 333 |
+
int64_t token_stride = H * D; // contiguous [B,T,H,D]
|
| 334 |
+
int64_t head_stride = D;
|
| 335 |
+
|
| 336 |
+
if (x.device().device == Device::CUDA) {
|
| 337 |
+
cuda::rope_forward_cuda(
|
| 338 |
+
out.data<float>(),
|
| 339 |
+
/*k=*/nullptr,
|
| 340 |
+
cos_sin_cache.data<float>(),
|
| 341 |
+
B, T,
|
| 342 |
+
cache_seq_len,
|
| 343 |
+
/*nh_q=*/H, /*nh_k=*/0, D,
|
| 344 |
+
token_stride, head_stride,
|
| 345 |
+
/*k_token_stride=*/0, /*k_head_stride=*/0,
|
| 346 |
+
position_offset,
|
| 347 |
+
interleaved,
|
| 348 |
+
/*inverse=*/false);
|
| 349 |
+
} else {
|
| 350 |
+
throw std::runtime_error("rope: CPU path not yet wired (see RoPE.cpp)");
|
| 351 |
+
}
|
| 352 |
+
|
| 353 |
+
if (GradMode::is_enabled() && x.requires_grad()) {
|
| 354 |
+
auto grad_fn = std::make_shared<RoPEBackward>(
|
| 355 |
+
cos_sin_cache, interleaved, position_offset);
|
| 356 |
+
grad_fn->set_next_edge(0, get_grad_edge(x));
|
| 357 |
+
out.set_grad_fn(grad_fn);
|
| 358 |
+
out.set_requires_grad(true);
|
| 359 |
+
}
|
| 360 |
+
|
| 361 |
+
if (autograd::g_shape_debug) GraphRecordMode::attach_forward_shape(out.shape(), out.dtype());
|
| 362 |
+
return out;
|
| 363 |
+
}
|
| 364 |
+
|
| 365 |
+
// ============================================================================
|
| 366 |
+
// rope_packed_qk — packed-QKV forward
|
| 367 |
+
// Rotates Q (heads [0..nh_q)) and K (heads [nh_q..nh_q+nh_k)) of a packed
|
| 368 |
+
// [B, T, (nh_q+nh_k+nh_v)*D] tensor. V slice is copied through untouched.
|
| 369 |
+
// ============================================================================
|
| 370 |
+
Tensor rope_packed_qk(const Tensor& qkv,
|
| 371 |
+
int64_t nh_q, int64_t nh_k, int64_t nh_v,
|
| 372 |
+
int64_t head_dim,
|
| 373 |
+
const Tensor& cos_sin_cache,
|
| 374 |
+
bool interleaved,
|
| 375 |
+
int64_t position_offset)
|
| 376 |
+
{
|
| 377 |
+
GraphRecordMode::record_forward("ROPE: rope_packed_qk");
|
| 378 |
+
|
| 379 |
+
if (qkv.ndim() != 3) {
|
| 380 |
+
throw std::invalid_argument("rope_packed_qk: qkv must be 3-D [B, T, C]");
|
| 381 |
+
}
|
| 382 |
+
if (qkv.dtype() != Dtype::Float32 || cos_sin_cache.dtype() != Dtype::Float32) {
|
| 383 |
+
throw std::invalid_argument("rope_packed_qk: only fp32 is supported in v1");
|
| 384 |
+
}
|
| 385 |
+
if (head_dim % 2 != 0) throw std::invalid_argument("rope_packed_qk: head_dim must be even");
|
| 386 |
+
|
| 387 |
+
int64_t B = qkv.shape().dims[0];
|
| 388 |
+
int64_t T = qkv.shape().dims[1];
|
| 389 |
+
int64_t C = qkv.shape().dims[2];
|
| 390 |
+
int64_t expected_C = (nh_q + nh_k + nh_v) * head_dim;
|
| 391 |
+
if (C != expected_C) {
|
| 392 |
+
throw std::invalid_argument("rope_packed_qk: last dim does not match (nh_q+nh_k+nh_v)*head_dim");
|
| 393 |
+
}
|
| 394 |
+
|
| 395 |
+
// Cache is [seq_len, head_dim]; pos = t + position_offset must index a valid
|
| 396 |
+
// row. Validate before launch (clear error instead of an OOB device read).
|
| 397 |
+
int64_t cache_seq_len = cos_sin_cache.shape().dims[0];
|
| 398 |
+
if (T + position_offset > cache_seq_len) {
|
| 399 |
+
throw std::invalid_argument(
|
| 400 |
+
"rope_packed_qk: T + position_offset (" + std::to_string(T + position_offset) +
|
| 401 |
+
") exceeds cos_sin_cache seq_len (" + std::to_string(cache_seq_len) +
|
| 402 |
+
"); rebuild cache with a larger seq_len");
|
| 403 |
+
}
|
| 404 |
+
|
| 405 |
+
Tensor out = qkv.contiguous().clone();
|
| 406 |
+
|
| 407 |
+
// In packed layout: token stride spans the full Q|K|V row; head stride is D.
|
| 408 |
+
// Q starts at offset 0; K starts at offset nh_q * head_dim within each token.
|
| 409 |
+
int64_t token_stride = (nh_q + nh_k + nh_v) * head_dim;
|
| 410 |
+
int64_t head_stride = head_dim;
|
| 411 |
+
|
| 412 |
+
if (qkv.device().device == Device::CUDA) {
|
| 413 |
+
float* base = out.data<float>();
|
| 414 |
+
float* q_ptr = base;
|
| 415 |
+
float* k_ptr = base + nh_q * head_dim;
|
| 416 |
+
|
| 417 |
+
cuda::rope_forward_cuda(
|
| 418 |
+
q_ptr, k_ptr,
|
| 419 |
+
cos_sin_cache.data<float>(),
|
| 420 |
+
B, T,
|
| 421 |
+
cache_seq_len,
|
| 422 |
+
nh_q, nh_k, head_dim,
|
| 423 |
+
token_stride, head_stride,
|
| 424 |
+
token_stride, head_stride,
|
| 425 |
+
position_offset,
|
| 426 |
+
interleaved,
|
| 427 |
+
/*inverse=*/false);
|
| 428 |
+
} else {
|
| 429 |
+
throw std::runtime_error("rope_packed_qk: CPU path not yet wired (see RoPE.cpp)");
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
if (GradMode::is_enabled() && qkv.requires_grad()) {
|
| 433 |
+
auto grad_fn = std::make_shared<RoPEPackedBackward>(
|
| 434 |
+
cos_sin_cache, nh_q, nh_k, nh_v, head_dim, interleaved, position_offset);
|
| 435 |
+
grad_fn->set_next_edge(0, get_grad_edge(qkv));
|
| 436 |
+
out.set_grad_fn(grad_fn);
|
| 437 |
+
out.set_requires_grad(true);
|
| 438 |
+
}
|
| 439 |
+
|
| 440 |
+
if (autograd::g_shape_debug) GraphRecordMode::attach_forward_shape(out.shape(), out.dtype());
|
| 441 |
+
return out;
|
| 442 |
+
}
|
| 443 |
+
|
| 444 |
+
} // namespace autograd
|
| 445 |
+
} // namespace OwnTensor
|