File size: 9,139 Bytes
67c4f10
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
#pragma once
#include <cuda_runtime.h>
#include <cmath>
#include <cstdint>

#define WARP_SIZE 32

// -------------------------------------------------------------
// 1. Dual-POV Frame Preprocessing Kernel
// -------------------------------------------------------------
// Ingests HWC uint8 raw agentview & wristview frames (256x256x3)
// Performs Robosuite [::-1, ::-1] camera flip and [-1, 1] normalization
// Writes contiguous [6, 256, 256] float tensor directly to GPU memory
__global__ void preprocess_frames_kernel(
    const uint8_t* __restrict__ agent_img,
    const uint8_t* __restrict__ wrist_img,
    float* __restrict__ out_obs,
    int H, int W
) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    int total_pixels = H * W;
    if (idx >= total_pixels) return;

    int r = idx / W;
    int c = idx % W;

    // Robosuite [::-1, ::-1] flip
    int flipped_r = H - 1 - r;
    int flipped_c = W - 1 - c;
    int in_idx = (flipped_r * W + flipped_c) * 3;

    // Agent (Channels 0, 1, 2)
    out_obs[0 * total_pixels + idx] = ((float)agent_img[in_idx + 0] / 255.0f) * 2.0f - 1.0f;
    out_obs[1 * total_pixels + idx] = ((float)agent_img[in_idx + 1] / 255.0f) * 2.0f - 1.0f;
    out_obs[2 * total_pixels + idx] = ((float)agent_img[in_idx + 2] / 255.0f) * 2.0f - 1.0f;

    // Wrist (Channels 3, 4, 5)
    out_obs[3 * total_pixels + idx] = ((float)wrist_img[in_idx + 0] / 255.0f) * 2.0f - 1.0f;
    out_obs[4 * total_pixels + idx] = ((float)wrist_img[in_idx + 1] / 255.0f) * 2.0f - 1.0f;
    out_obs[5 * total_pixels + idx] = ((float)wrist_img[in_idx + 2] / 255.0f) * 2.0f - 1.0f;
}

// -------------------------------------------------------------
// 2. LayerNorm Kernel (dim=256)
// -------------------------------------------------------------
__global__ void layernorm_256_kernel(
    const float* __restrict__ in,
    const float* __restrict__ gamma,
    const float* __restrict__ beta,
    float* __restrict__ out,
    int num_rows,
    float eps = 1e-5f
) {
    int row = blockIdx.x;
    if (row >= num_rows) return;

    int tid = threadIdx.x;
    const float* in_row = in + row * 256;
    float* out_row = out + row * 256;

    float val = (tid < 256) ? in_row[tid] : 0.0f;

    // Compute mean
    __shared__ float s_mean;
    __shared__ float s_var;

    // Warp-level reduction for 256 floats (8 warps of 32 threads)
    float sum = val;
    #pragma unroll
    for (int offset = 16; offset > 0; offset /= 2) {
        sum += __shfl_down_sync(0xffffffff, sum, offset);
    }

    __shared__ float warp_sums[8];
    int lane = tid % 32;
    int warp_id = tid / 32;
    if (lane == 0) warp_sums[warp_id] = sum;
    __syncthreads();

    if (tid == 0) {
        float total_sum = 0.0f;
        for (int i = 0; i < 8; ++i) total_sum += warp_sums[i];
        s_mean = total_sum / 256.0f;
    }
    __syncthreads();

    float mean = s_mean;
    float diff = val - mean;
    float sq_diff = diff * diff;

    #pragma unroll
    for (int offset = 16; offset > 0; offset /= 2) {
        sq_diff += __shfl_down_sync(0xffffffff, sq_diff, offset);
    }
    if (lane == 0) warp_sums[warp_id] = sq_diff;
    __syncthreads();

    if (tid == 0) {
        float total_sq = 0.0f;
        for (int i = 0; i < 8; ++i) total_sq += warp_sums[i];
        s_var = total_sq / 256.0f;
    }
    __syncthreads();

    float inv_std = rsqrtf(s_var + eps);
    float g = (gamma != nullptr) ? gamma[tid] : 1.0f;
    float b = (beta != nullptr) ? beta[tid] : 0.0f;

    out_row[tid] = (diff * inv_std) * g + b;
}

// -------------------------------------------------------------
// 3. GELU Activation Kernel
// -------------------------------------------------------------
__global__ void gelu_kernel(const float* __restrict__ in, float* __restrict__ out, int N) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    if (idx >= N) return;
    float x = in[idx];
    // Exact PyTorch GELU: x * 0.5 * (1 + erf(x / sqrt(2)))
    out[idx] = x * 0.5f * (1.0f + erff(x * 0.7071067811865475f));
}

// -------------------------------------------------------------
// 4. SiLU Activation Kernel
// -------------------------------------------------------------
__global__ void silu_kernel(const float* __restrict__ in, float* __restrict__ out, int N) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    if (idx >= N) return;
    float x = in[idx];
    out[idx] = x / (1.0f + expf(-x));
}

// -------------------------------------------------------------
// 5. FiLM Modulation Kernel: out = norm_h * (1.0 + scale) + shift
// -------------------------------------------------------------
__global__ void film_modulate_256_kernel(
    const float* __restrict__ h_norm,
    const float* __restrict__ film_cond, // [512] -> scale (0:256), shift (256:512)
    float* __restrict__ out,
    int num_tokens
) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    if (idx >= num_tokens * 256) return;

    int d = idx % 256;

    float scale = film_cond[d];
    float shift = film_cond[256 + d];

    out[idx] = h_norm[idx] * (1.0f + scale) + shift;
}

// -------------------------------------------------------------
// 6. Softmax Kernel (num_rows, num_cols <= 32)
// -------------------------------------------------------------
__global__ void softmax_small_kernel(
    float* __restrict__ data,
    int num_rows,
    int num_cols
) {
    int row = blockIdx.x;
    if (row >= num_rows) return;

    int tid = threadIdx.x;
    float* row_data = data + row * num_cols;

    float val = (tid < num_cols) ? row_data[tid] : -1e20f;

    // Max reduction
    float max_val = val;
    #pragma unroll
    for (int offset = 16; offset > 0; offset /= 2) {
        max_val = fmaxf(max_val, __shfl_down_sync(0xffffffff, max_val, offset));
    }
    max_val = __shfl_sync(0xffffffff, max_val, 0);

    float exp_val = (tid < num_cols) ? expf(val - max_val) : 0.0f;

    // Sum reduction
    float sum = exp_val;
    #pragma unroll
    for (int offset = 16; offset > 0; offset /= 2) {
        sum += __shfl_down_sync(0xffffffff, sum, offset);
    }
    sum = __shfl_sync(0xffffffff, sum, 0);

    if (tid < num_cols) {
        row_data[tid] = exp_val / (sum + 1e-12f);
    }
}

// -------------------------------------------------------------
// 7. Euler Integration Step: z = z + dt * v
// -------------------------------------------------------------
__global__ void euler_step_kernel(
    float* __restrict__ z,
    const float* __restrict__ v,
    float dt,
    int N
) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    if (idx >= N) return;
    z[idx] += dt * v[idx];
}

// -------------------------------------------------------------
// 8. Action Denormalization: a = ((z + 1)/2) * (max - min) + min
// -------------------------------------------------------------
__global__ void denormalize_actions_kernel(
    const float* __restrict__ z_chunk,
    const float* __restrict__ act_min,
    const float* __restrict__ act_max,
    float* __restrict__ out_actions,
    int chunk_size,
    int act_dim
) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    if (idx >= chunk_size * act_dim) return;

    int dim_idx = idx % act_dim;
    float z = z_chunk[idx];
    float mn = act_min[dim_idx];
    float mx = act_max[dim_idx];

    // Standard denormalization
    out_actions[idx] = ((z + 1.0f) * 0.5f) * (mx - mn) + mn;
}

// -------------------------------------------------------------
// 9. Sinusoidal Timestep Embedding (dim=256)
// -------------------------------------------------------------
__global__ void sinusoidal_embedding_kernel(float t, float* __restrict__ out_emb, int dim = 256) {
    int tid = threadIdx.x;
    if (tid >= dim / 2) return;

    int half = dim / 2;
    float freq = expf(-logf(10000.0f) * (float)tid / (float)half);
    float arg = t * freq * 1000.0f;

    out_emb[tid] = sinf(arg);
    out_emb[half + tid] = cosf(arg);
}

// -------------------------------------------------------------
// 10. Transpose Packed QKV to Head-Major
// -------------------------------------------------------------
__global__ void transpose_qkv_to_heads_kernel(
    const float* __restrict__ qkv,
    float* __restrict__ q_head,
    float* __restrict__ k_head,
    float* __restrict__ v_head,
    int N, int H, int D
) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    int total = N * H * D;
    if (idx >= total) return;
    int d = idx % D;
    int h = (idx / D) % H;
    int n = idx / (H * D);

    int row_stride = 3 * H * D;
    int hd = H * D;

    int q_in = n * row_stride + h * D + d;
    int k_in = n * row_stride + hd + h * D + d;
    int v_in = n * row_stride + 2 * hd + h * D + d;

    int out_idx = h * (N * D) + n * D + d;
    q_head[out_idx] = qkv[q_in];
    k_head[out_idx] = qkv[k_in];
    v_head[out_idx] = qkv[v_in];
}

// -------------------------------------------------------------
// 11. Clamp Kernel
// -------------------------------------------------------------
__global__ void clamp_kernel(float* data, float min_val, float max_val, int N) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    if (idx < N) {
        data[idx] = fminf(max_val, fmaxf(min_val, data[idx]));
    }
}