unparallelled commited on
Commit
6b70795
·
verified ·
1 Parent(s): d2ced45

Upload source/files/YARNOps.cpp with huggingface_hub

Browse files
Files changed (1) hide show
  1. source/files/YARNOps.cpp +445 -0
source/files/YARNOps.cpp ADDED
@@ -0,0 +1,445 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ #include "autograd/operations/RoPEOps.h"
3
+ #include "autograd/backward/RoPEBackward.h"
4
+ #include "autograd/AutogradContext.h"
5
+ #include "autograd/Variable.h"
6
+ #include "autograd/GraphRecorder.h"
7
+ #include "checkpointing/GradMode.h"
8
+ #include "ops/helpers/RoPEKernels.h"
9
+ #include "autograd/ops_template.h"
10
+
11
+ #include <stdexcept>
12
+ #include <cmath>
13
+ #include <cstdlib> // std::getenv, std::atof
14
+ #include <cstdio> // std::fprintf
15
+ #include "CPLog.h" // cplog::log_rank() — gate one-time notices to rank 0
16
+ #include <string> // std::to_string
17
+ #include <algorithm> // std::max, std::min
18
+ #include <vector> // std::vector (LongRoPE lambda)
19
+ #include "context_parallel/LongRoPEOps.h" // build_rope_cache_longrope declaration
20
+
21
+ namespace OwnTensor {
22
+ namespace autograd {
23
+
24
+ // ============================================================================
25
+ // build_rope_cache — YaRN (NTK-by-parts + attention temperature) cos/sin cache
26
+ //
27
+ // Canonical HF/Qwen3 YaRN. Extends context (e.g. 1024 -> 8192) by blending
28
+ // per-dimension inverse frequencies (extrapolate high-freq, interpolate low-freq
29
+ // via a dimension-index linear ramp between correction dims) and baking the
30
+ // attention temperature m = 0.1*ln(s)+1 into BOTH cos and sin. RoPE hits both Q
31
+ // and K, so m appears twice in each QK logit (m^2 = 1/t), matching YaRN's
32
+ // softmax temperature; the fused kernel's 1/sqrt(hd) scale is left untouched.
33
+ //
34
+ // Params are hardcoded defaults, each overridable by an env var. Default
35
+ // YARN_SCALE=1.0 reduces this to standard RoPE bit-identically (base_freq/s ==
36
+ // base_freq for all i, and m==1), preserving all existing callers.
37
+ //
38
+ // The cache is always built on CPU (fp32) then copied to the target device: the
39
+ // device kernel rope_build_cache_cuda has no per-dimension frequency hook, so we
40
+ // bypass it. Cost is negligible (built once at model init).
41
+ // ============================================================================
42
+ namespace {
43
+ inline float yarn_env_f(const char* key, float dflt) {
44
+ const char* v = std::getenv(key);
45
+ return v ? static_cast<float>(std::atof(v)) : dflt;
46
+ }
47
+ } // anonymous namespace
48
+
49
+ Tensor build_rope_cache(int64_t seq_len, int64_t head_dim, float base, DeviceIndex device)
50
+ {
51
+ if (head_dim % 2 != 0) {
52
+ throw std::invalid_argument("build_rope_cache: head_dim must be even");
53
+ }
54
+
55
+ // --- YaRN config (env-overridable; scale=1.0 => standard RoPE) ---
56
+ const float s = yarn_env_f("YARN_SCALE", 1.0f); // target_ctx / original_ctx
57
+ const float L = yarn_env_f("YARN_ORIG_MAXPOS", 1024.0f);// original pretrain context
58
+ const float b_fast = yarn_env_f("YARN_BETA_FAST", 32.0f); // "beta" (high-rotation bound)
59
+ const float b_slow = yarn_env_f("YARN_BETA_SLOW", 1.0f); // "alpha" (low-rotation bound)
60
+
61
+ // Attention temperature m (s==1 -> m==1 -> no scaling). YARN_NO_MSCALE=1
62
+ // forces m=1 for the fair-comparison "YaRN without temperature" arm of the
63
+ // LongRoPE study (default keeps m, so unset behavior is byte-identical).
64
+ const bool yarn_no_m = std::getenv("YARN_NO_MSCALE") != nullptr &&
65
+ std::atoi(std::getenv("YARN_NO_MSCALE")) != 0;
66
+ const float m = (!yarn_no_m && s > 1.0f) ? (0.1f * std::log(s) + 1.0f) : 1.0f;
67
+
68
+ const int64_t half = head_dim / 2;
69
+
70
+ // Canonical correction range: linear ramp over dimension index between the
71
+ // dims where the number of rotations equals b_fast (high-freq end) and
72
+ // b_slow (low-freq end). Matches HF find_correction_dim/find_correction_range.
73
+ auto find_dim = [&](float num_rot) {
74
+ return (static_cast<float>(head_dim)
75
+ * std::log(L / (num_rot * 2.0f * static_cast<float>(M_PI))))
76
+ / (2.0f * std::log(base));
77
+ };
78
+ int64_t low = static_cast<int64_t>(std::floor(find_dim(b_fast)));
79
+ int64_t high = static_cast<int64_t>(std::ceil (find_dim(b_slow)));
80
+ low = std::max<int64_t>(low, 0);
81
+ high = std::min<int64_t>(high, half - 1); // clamp to half-1: loop indexes [0,half) only;
82
+ // deliberate deviation from HF head_dim-1 -- do not "fix"
83
+ float denom = static_cast<float>(high - low);
84
+ if (denom <= 0.0f) denom = 1e-3f; // avoid /0 when low==high
85
+
86
+ // (R5) Guard degenerate env input: L/(num_rot*2pi) <= 0 -> log(neg) = NaN.
87
+ if (s > 1.0f && (!std::isfinite(find_dim(b_fast)) || !std::isfinite(find_dim(b_slow)) || low > high)) {
88
+ throw std::runtime_error(
89
+ "build_rope_cache(YaRN): bad correction range -- check "
90
+ "YARN_BETA_FAST/YARN_BETA_SLOW/YARN_ORIG_MAXPOS (got low="
91
+ + std::to_string(low) + " high=" + std::to_string(high) + ")");
92
+ }
93
+
94
+ // (R2) scale <-> seq_len consistency (warn, don't hard-fail).
95
+ if (s > 1.0f && std::abs(static_cast<float>(seq_len) - s * L) > 1.0f) {
96
+ if (cplog::log_rank()) std::fprintf(stderr,
97
+ "[YaRN][warn] seq_len(%lld) != scale(%.3f)*orig_maxpos(%.1f)=%.1f "
98
+ "-- cache scaled for a different context than it is sized for\n",
99
+ (long long)seq_len, s, L, s * L);
100
+ }
101
+
102
+ // (R3) Init log: makes the active config + the YARNOps-symbol-wins link
103
+ // dependency visible. Missing line / scale=1 => silent revert to std RoPE.
104
+ if (cplog::log_rank()) std::fprintf(stderr,
105
+ "[YaRN] build_rope_cache: seq_len=%lld hd=%lld scale=%.3f low=%lld high=%lld m=%.4f\n",
106
+ (long long)seq_len, (long long)head_dim, s, (long long)low, (long long)high, m);
107
+
108
+ // --- Build on CPU, fill, then copy to device (bypass the CUDA kernel). ---
109
+ TensorOptions cpu_opts = TensorOptions().with_dtype(Dtype::Float32)
110
+ .with_device(DeviceIndex(Device::CPU));
111
+ Tensor cpu(Shape{{seq_len, head_dim}}, cpu_opts);
112
+ float* p = cpu.data<float>();
113
+
114
+ for (int64_t pos = 0; pos < seq_len; ++pos) {
115
+ for (int64_t i = 0; i < half; ++i) {
116
+ float exponent = (2.0f * static_cast<float>(i)) / static_cast<float>(head_dim);
117
+ float base_freq = 1.0f / std::pow(base, exponent); // extrapolation freq
118
+
119
+ float ramp = (static_cast<float>(i) - static_cast<float>(low)) / denom;
120
+ ramp = ramp < 0.0f ? 0.0f : (ramp > 1.0f ? 1.0f : ramp);
121
+ // ramp=0 (i<low, high-freq) -> keep base_freq (extrapolate)
122
+ // ramp=1 (i>high, low-freq) -> base_freq/s (interpolate)
123
+ float inv_freq = (base_freq / s) * ramp + base_freq * (1.0f - ramp);
124
+
125
+ float angle = static_cast<float>(pos) * inv_freq;
126
+ p[pos * head_dim + i] = std::cos(angle) * m; // bake temperature m
127
+ p[pos * head_dim + i + half] = std::sin(angle) * m;
128
+ }
129
+ }
130
+
131
+ cpu.set_requires_grad(false);
132
+ if (device.device == Device::CUDA) {
133
+ Tensor cache = cpu.to(device);
134
+ cache.set_requires_grad(false);
135
+ return cache;
136
+ }
137
+ return cpu;
138
+ }
139
+
140
+ // ============================================================================
141
+ // build_rope_cache_longrope — LongRoPE (searched per-dim lambda + n_hat) cache.
142
+ //
143
+ // angle(n,i) = n * base_freq_i / (n < n_hat ? 1 : lambda_i). NeoX half-split
144
+ // layout identical to build_rope_cache above (cos in [0,half), sin in
145
+ // [half,head_dim)). NO YaRN attention-temperature m. lambda has length
146
+ // head_dim/2, monotone non-decreasing, each >= 1 (the search enforces this; we
147
+ // re-assert defensively). Built on CPU fp32 then copied to device, grad-free.
148
+ // New, additive symbol -- does not touch build_rope_cache or any caller.
149
+ // ============================================================================
150
+ namespace {
151
+
152
+ // One [seq_len, head_dim] LongRoPE plane, NeoX half-split (cos in [0,half),
153
+ // sin in [half,head_dim)). Extracted verbatim from build_rope_cache_longrope so
154
+ // the per-group builder cannot drift from the single-plane one.
155
+ void fill_longrope_plane(float* p, int64_t seq_len, int64_t head_dim, float base,
156
+ const std::vector<float>& lambda, int64_t n_hat, float mscale)
157
+ {
158
+ const int64_t half = head_dim / 2;
159
+ for (int64_t pos = 0; pos < seq_len; ++pos) {
160
+ const bool preserved = (pos < n_hat); // initial tokens keep original positions
161
+ for (int64_t i = 0; i < half; ++i) {
162
+ float exponent = (2.0f * static_cast<float>(i)) / static_cast<float>(head_dim);
163
+ float base_freq = 1.0f / std::pow(base, exponent);
164
+ float scale = preserved ? 1.0f : lambda[static_cast<size_t>(i)]; // lambda divides freq
165
+ float angle = static_cast<float>(pos) * base_freq / scale;
166
+ p[pos * head_dim + i] = std::cos(angle) * mscale; // NeoX half-split; mscale=1 => no temp
167
+ p[pos * head_dim + i + half] = std::sin(angle) * mscale;
168
+ }
169
+ }
170
+ }
171
+
172
+ // Shared shape/monotonicity validation. `tag` names the caller in the message.
173
+ void validate_longrope_lambda(const std::vector<float>& lambda, int64_t half,
174
+ const char* tag)
175
+ {
176
+ if (static_cast<int64_t>(lambda.size()) != half) {
177
+ throw std::invalid_argument(
178
+ std::string(tag) + ": lambda size (" + std::to_string(lambda.size()) +
179
+ ") must equal head_dim/2 (" + std::to_string(half) + ")");
180
+ }
181
+ for (int64_t i = 0; i < half; ++i) {
182
+ if (!(lambda[static_cast<size_t>(i)] >= 1.0f)) {
183
+ throw std::invalid_argument(
184
+ std::string(tag) + ": lambda[" + std::to_string(i) + "] must be >= 1");
185
+ }
186
+ if (i > 0 && lambda[static_cast<size_t>(i)] < lambda[static_cast<size_t>(i - 1)]) {
187
+ throw std::invalid_argument(
188
+ std::string(tag) + ": lambda must be non-decreasing (violated at i=" +
189
+ std::to_string(i) + ")");
190
+ }
191
+ }
192
+ }
193
+
194
+ } // namespace
195
+
196
+ Tensor build_rope_cache_longrope(int64_t seq_len, int64_t head_dim, float base,
197
+ const std::vector<float>& lambda, int64_t n_hat,
198
+ DeviceIndex device, float mscale)
199
+ {
200
+ if (head_dim % 2 != 0) {
201
+ throw std::invalid_argument("build_rope_cache_longrope: head_dim must be even");
202
+ }
203
+ const int64_t half = head_dim / 2;
204
+ validate_longrope_lambda(lambda, half, "build_rope_cache_longrope");
205
+ if (n_hat < 0) {
206
+ throw std::invalid_argument("build_rope_cache_longrope: n_hat must be >= 0");
207
+ }
208
+
209
+ if (cplog::log_rank()) std::fprintf(stderr,
210
+ "[LongRoPE] build_rope_cache_longrope: seq_len=%lld hd=%lld n_hat=%lld "
211
+ "lam0=%.4f lam_last=%.4f mscale=%.4f\n",
212
+ (long long)seq_len, (long long)head_dim, (long long)n_hat,
213
+ (double)lambda[0], (double)lambda[static_cast<size_t>(half - 1)], (double)mscale);
214
+
215
+ TensorOptions cpu_opts = TensorOptions().with_dtype(Dtype::Float32)
216
+ .with_device(DeviceIndex(Device::CPU));
217
+ Tensor cpu(Shape{{seq_len, head_dim}}, cpu_opts);
218
+ float* p = cpu.data<float>();
219
+
220
+ fill_longrope_plane(p, seq_len, head_dim, base, lambda, n_hat, mscale);
221
+
222
+ cpu.set_requires_grad(false);
223
+ if (device.device == Device::CUDA) {
224
+ Tensor cache = cpu.to(device);
225
+ cache.set_requires_grad(false);
226
+ return cache;
227
+ }
228
+ return cpu;
229
+ }
230
+
231
+ // ---------------------------------------------------------------------------
232
+ // Ms-PoE per-KV-group cache: [n_groups, seq_len, head_dim]. Plane g is exactly
233
+ // build_rope_cache_longrope(lambdas[g]). See LongRoPEOps.h for why the ratio is
234
+ // per KV GROUP and not per head.
235
+ // ---------------------------------------------------------------------------
236
+ Tensor build_rope_cache_longrope_grouped(
237
+ int64_t seq_len, int64_t head_dim, float base,
238
+ const std::vector<std::vector<float>>& lambdas, int64_t n_hat,
239
+ DeviceIndex device, float mscale)
240
+ {
241
+ if (head_dim % 2 != 0) {
242
+ throw std::invalid_argument(
243
+ "build_rope_cache_longrope_grouped: head_dim must be even");
244
+ }
245
+ if (lambdas.empty()) {
246
+ throw std::invalid_argument(
247
+ "build_rope_cache_longrope_grouped: needs >= 1 lambda group");
248
+ }
249
+ if (n_hat < 0) {
250
+ throw std::invalid_argument(
251
+ "build_rope_cache_longrope_grouped: n_hat must be >= 0");
252
+ }
253
+ const int64_t half = head_dim / 2;
254
+ const int64_t n_groups = static_cast<int64_t>(lambdas.size());
255
+ for (int64_t g = 0; g < n_groups; ++g) {
256
+ validate_longrope_lambda(lambdas[static_cast<size_t>(g)], half,
257
+ "build_rope_cache_longrope_grouped");
258
+ }
259
+
260
+ if (cplog::log_rank()) {
261
+ std::fprintf(stderr,
262
+ "[Ms-PoE] build_rope_cache_longrope_grouped: groups=%lld seq_len=%lld "
263
+ "hd=%lld n_hat=%lld mscale=%.4f\n",
264
+ (long long)n_groups, (long long)seq_len, (long long)head_dim,
265
+ (long long)n_hat, (double)mscale);
266
+ for (int64_t g = 0; g < n_groups; ++g) {
267
+ const std::vector<float>& lam = lambdas[static_cast<size_t>(g)];
268
+ std::fprintf(stderr, "[Ms-PoE] group %lld: lam0=%.4f lam_last=%.4f\n",
269
+ (long long)g, (double)lam[0],
270
+ (double)lam[static_cast<size_t>(half - 1)]);
271
+ }
272
+ }
273
+
274
+ TensorOptions cpu_opts = TensorOptions().with_dtype(Dtype::Float32)
275
+ .with_device(DeviceIndex(Device::CPU));
276
+ Tensor cpu(Shape{{n_groups, seq_len, head_dim}}, cpu_opts);
277
+ float* p = cpu.data<float>();
278
+
279
+ const int64_t plane = seq_len * head_dim;
280
+ for (int64_t g = 0; g < n_groups; ++g) {
281
+ fill_longrope_plane(p + g * plane, seq_len, head_dim, base,
282
+ lambdas[static_cast<size_t>(g)], n_hat, mscale);
283
+ }
284
+
285
+ cpu.set_requires_grad(false);
286
+ if (device.device == Device::CUDA) {
287
+ Tensor cache = cpu.to(device);
288
+ cache.set_requires_grad(false);
289
+ return cache;
290
+ }
291
+ return cpu;
292
+ }
293
+
294
+ // ============================================================================
295
+ // rope — separate-tensor forward
296
+ // Operates on x of shape [B, T, H, D]; returns rotated tensor (out-of-place).
297
+ // ============================================================================
298
+ Tensor rope(const Tensor& x,
299
+ const Tensor& cos_sin_cache,
300
+ bool interleaved,
301
+ int64_t position_offset)
302
+ {
303
+ GraphRecordMode::record_forward("ROPE: rope");
304
+
305
+ if (x.ndim() != 4) {
306
+ throw std::invalid_argument("rope: input must be 4-D [B, T, H, D]");
307
+ }
308
+ if (x.dtype() != Dtype::Float32 || cos_sin_cache.dtype() != Dtype::Float32) {
309
+ throw std::invalid_argument("rope: only fp32 is supported in v1");
310
+ }
311
+
312
+ int64_t B = x.shape().dims[0];
313
+ int64_t T = x.shape().dims[1];
314
+ int64_t H = x.shape().dims[2];
315
+ int64_t D = x.shape().dims[3];
316
+ if (D % 2 != 0) throw std::invalid_argument("rope: head_dim must be even");
317
+
318
+ // Cache is [seq_len, head_dim]; every token's pos = t + position_offset must
319
+ // index a valid row. Validate before launch so we fail with a clear message
320
+ // instead of an out-of-bounds device read.
321
+ int64_t cache_seq_len = cos_sin_cache.shape().dims[0];
322
+ if (T + position_offset > cache_seq_len) {
323
+ throw std::invalid_argument(
324
+ "rope: T + position_offset (" + std::to_string(T + position_offset) +
325
+ ") exceeds cos_sin_cache seq_len (" + std::to_string(cache_seq_len) +
326
+ "); rebuild cache with a larger seq_len");
327
+ }
328
+
329
+ // Out-of-place: clone input, rotate the clone in place. Clone is
330
+ // contiguous so strides are simple.
331
+ Tensor out = x.contiguous().clone();
332
+
333
+ int64_t token_stride = H * D; // contiguous [B,T,H,D]
334
+ int64_t head_stride = D;
335
+
336
+ if (x.device().device == Device::CUDA) {
337
+ cuda::rope_forward_cuda(
338
+ out.data<float>(),
339
+ /*k=*/nullptr,
340
+ cos_sin_cache.data<float>(),
341
+ B, T,
342
+ cache_seq_len,
343
+ /*nh_q=*/H, /*nh_k=*/0, D,
344
+ token_stride, head_stride,
345
+ /*k_token_stride=*/0, /*k_head_stride=*/0,
346
+ position_offset,
347
+ interleaved,
348
+ /*inverse=*/false);
349
+ } else {
350
+ throw std::runtime_error("rope: CPU path not yet wired (see RoPE.cpp)");
351
+ }
352
+
353
+ if (GradMode::is_enabled() && x.requires_grad()) {
354
+ auto grad_fn = std::make_shared<RoPEBackward>(
355
+ cos_sin_cache, interleaved, position_offset);
356
+ grad_fn->set_next_edge(0, get_grad_edge(x));
357
+ out.set_grad_fn(grad_fn);
358
+ out.set_requires_grad(true);
359
+ }
360
+
361
+ if (autograd::g_shape_debug) GraphRecordMode::attach_forward_shape(out.shape(), out.dtype());
362
+ return out;
363
+ }
364
+
365
+ // ============================================================================
366
+ // rope_packed_qk — packed-QKV forward
367
+ // Rotates Q (heads [0..nh_q)) and K (heads [nh_q..nh_q+nh_k)) of a packed
368
+ // [B, T, (nh_q+nh_k+nh_v)*D] tensor. V slice is copied through untouched.
369
+ // ============================================================================
370
+ Tensor rope_packed_qk(const Tensor& qkv,
371
+ int64_t nh_q, int64_t nh_k, int64_t nh_v,
372
+ int64_t head_dim,
373
+ const Tensor& cos_sin_cache,
374
+ bool interleaved,
375
+ int64_t position_offset)
376
+ {
377
+ GraphRecordMode::record_forward("ROPE: rope_packed_qk");
378
+
379
+ if (qkv.ndim() != 3) {
380
+ throw std::invalid_argument("rope_packed_qk: qkv must be 3-D [B, T, C]");
381
+ }
382
+ if (qkv.dtype() != Dtype::Float32 || cos_sin_cache.dtype() != Dtype::Float32) {
383
+ throw std::invalid_argument("rope_packed_qk: only fp32 is supported in v1");
384
+ }
385
+ if (head_dim % 2 != 0) throw std::invalid_argument("rope_packed_qk: head_dim must be even");
386
+
387
+ int64_t B = qkv.shape().dims[0];
388
+ int64_t T = qkv.shape().dims[1];
389
+ int64_t C = qkv.shape().dims[2];
390
+ int64_t expected_C = (nh_q + nh_k + nh_v) * head_dim;
391
+ if (C != expected_C) {
392
+ throw std::invalid_argument("rope_packed_qk: last dim does not match (nh_q+nh_k+nh_v)*head_dim");
393
+ }
394
+
395
+ // Cache is [seq_len, head_dim]; pos = t + position_offset must index a valid
396
+ // row. Validate before launch (clear error instead of an OOB device read).
397
+ int64_t cache_seq_len = cos_sin_cache.shape().dims[0];
398
+ if (T + position_offset > cache_seq_len) {
399
+ throw std::invalid_argument(
400
+ "rope_packed_qk: T + position_offset (" + std::to_string(T + position_offset) +
401
+ ") exceeds cos_sin_cache seq_len (" + std::to_string(cache_seq_len) +
402
+ "); rebuild cache with a larger seq_len");
403
+ }
404
+
405
+ Tensor out = qkv.contiguous().clone();
406
+
407
+ // In packed layout: token stride spans the full Q|K|V row; head stride is D.
408
+ // Q starts at offset 0; K starts at offset nh_q * head_dim within each token.
409
+ int64_t token_stride = (nh_q + nh_k + nh_v) * head_dim;
410
+ int64_t head_stride = head_dim;
411
+
412
+ if (qkv.device().device == Device::CUDA) {
413
+ float* base = out.data<float>();
414
+ float* q_ptr = base;
415
+ float* k_ptr = base + nh_q * head_dim;
416
+
417
+ cuda::rope_forward_cuda(
418
+ q_ptr, k_ptr,
419
+ cos_sin_cache.data<float>(),
420
+ B, T,
421
+ cache_seq_len,
422
+ nh_q, nh_k, head_dim,
423
+ token_stride, head_stride,
424
+ token_stride, head_stride,
425
+ position_offset,
426
+ interleaved,
427
+ /*inverse=*/false);
428
+ } else {
429
+ throw std::runtime_error("rope_packed_qk: CPU path not yet wired (see RoPE.cpp)");
430
+ }
431
+
432
+ if (GradMode::is_enabled() && qkv.requires_grad()) {
433
+ auto grad_fn = std::make_shared<RoPEPackedBackward>(
434
+ cos_sin_cache, nh_q, nh_k, nh_v, head_dim, interleaved, position_offset);
435
+ grad_fn->set_next_edge(0, get_grad_edge(qkv));
436
+ out.set_grad_fn(grad_fn);
437
+ out.set_requires_grad(true);
438
+ }
439
+
440
+ if (autograd::g_shape_debug) GraphRecordMode::attach_forward_shape(out.shape(), out.dtype());
441
+ return out;
442
+ }
443
+
444
+ } // namespace autograd
445
+ } // namespace OwnTensor