Sync updated Report.tex

Report.tex

@@ -161,11 +161,11 @@ All experiments were strictly controlled to isolate the impact of individual arc
 \begin{tabular}{@{}lcccccccc@{}}
 \toprule
 \textbf{Model} & \textbf{Bit Acc (\%)} & \textbf{Seq Acc (\%)} & \textbf{Norm Lev $\downarrow$} & \textbf{BLEU-4} & \textbf{ROUGE-1} & \textbf{ROUGE-2} & \textbf{ROUGE-L} & \textbf{Train Time} \\ \midrule
-\textbf{C1 (Base)} & 88.38 & 31.00 & 0.0048 & 95.79 & 98.35 & 96.83 & 98.35 & 1018s \\
-\textbf{C2 (RoPE)} & 90.50 & \textbf{34.20} & 0.0044 & 96.25 & 98.50 & 97.12 & 98.50 & 1027s \\
-\textbf{C3 (GQA)} & 87.76 & 25.00 & 0.0057 & 95.02 & 98.08 & 96.36 & 98.08 & 1050s \\
-\textbf{C4 (RMSNorm)} & 86.67 & 25.80 & 0.0059 & 95.03 & 97.96 & 96.11 & 97.96 & \textbf{971s} \\
-\textbf{C5 (BLT)} & \textbf{100.00} & \textbf{100.00} & \textbf{0.0000} & N/A & N/A & N/A & N/A & \textbf{642s} \\ \bottomrule
+\textbf{C1 (Base)} & 88.60 & 29.20 & 0.0052 & 95.24 & 98.17 & 96.49 & 98.17 & 1014s \\
+\textbf{C2 (RoPE)} & 90.91 & \textbf{33.80} & 0.0044 & 96.12 & 98.49 & 97.11 & 98.49 & 1037s \\
+\textbf{C3 (GQA)} & 88.21 & 25.20 & 0.0055 & 95.05 & 98.08 & 96.33 & 98.08 & 2288s \\
+\textbf{C4 (RMSNorm)} & 89.53 & 32.80 & 0.0044 & 96.06 & 98.56 & 97.23 & 98.56 & 1872s \\
+\textbf{C5 (BLT)} & \textbf{100.00} & \textbf{100.00} & \textbf{0.0000} & N/A & N/A & N/A & N/A & \textbf{629s} \\ \bottomrule
 \end{tabular}
 }
 \end{table}
@@ -180,13 +180,13 @@ All experiments were strictly controlled to isolate the impact of individual arc
 \subsection{Analysis of Individual Component Ablations}
 
 \textbf{1. Impact of Positional Encoding (C2 vs.\ C1):}
-Replacing static Sinusoidal Absolute Positional Encodings with Rotary Position Embeddings (RoPE) yielded the highest performance among subword models (Seq Accuracy improved from $31.00\%$ to $34.20\%$, and Normalized Levenshtein distance decreased from $0.0048$ to $0.0044$). RoPE embeds relative shift information directly into the inner product of Query and Key representations, allowing the attention heads to generalize across sequence offsets.
+Replacing static Sinusoidal Absolute Positional Encodings with Rotary Position Embeddings (RoPE) yielded the highest sequence accuracy among subword models (Seq Accuracy improved from $29.20\%$ to $33.80\%$, Bit-Level Accuracy improved from $88.60\%$ to $90.91\%$, and Normalized Levenshtein distance decreased from $0.0052$ to $0.0044$). RoPE embeds relative shift information directly into the inner product of Query and Key representations, allowing the attention heads to generalize across sequence offsets.
 
 \textbf{2. Impact of Attention Mechanism (C3 vs.\ C1):}
-Grouped-Query Attention (GQA) with 2 KV heads per 4 Query heads reduced peak memory footprint to $111.4$\,MB (a $32.5\%$ reduction compared to C1's $165.1$\,MB). The slight trade-off in sequence accuracy ($25.00\%$ vs.\ $31.00\%$) illustrates the trade-off between KV cache compression and representational capacity.
+Grouped-Query Attention (GQA) with 2 KV heads per 4 Query heads reduced peak memory footprint from $161.9$\,MB (C1) to $109.2$\,MB (a $32.5\%$ reduction among subword models). The trade-off in sequence accuracy ($25.20\%$ vs.\ $29.20\%$) and training throughput reflects the computational overhead of key-value repeat operations in this small-batch regime.
 
 \textbf{3. Impact of Normalization (C4 vs.\ C1):}
-RMSNorm proved to be the fastest training configuration among BPE models ($971$s total training time, reaching a peak throughput of $1,294.5$\,samples/s) while preserving strong translation fidelity ($95.03$ BLEU-4, $97.96\%$ ROUGE-L). Omitting mean-centering avoids unnecessary computation and stabilizes gradients.
+RMSNorm achieved strong sequence exact-match recovery ($32.80\%$) and high translation fidelity ($96.06$ BLEU-4, $98.56\%$ ROUGE-L) while reducing peak memory footprint to $109.2$\,MB (a $32.5\%$ reduction vs.\ C1). Omitting mean-centering simplifies layer normalizations and stabilizes gradient backpropagation.
 
 \section{Benchmarking Configuration 5: Byte Latent Transformer (BLT)}
 \label{sec:blt}
@@ -211,12 +211,12 @@ Configuration 5 (C5) replaces the learned subword vocabulary with a token-free B
 \textbf{Evaluation Dimension} & \textbf{C1 (Base Subword)} & \textbf{C5 (BLT Token-Free)} & \textbf{Relative Change} & \textbf{Architectural Cause / Analysis} \\ \midrule
 \textbf{Vocabulary Size} & 4,000 subwords & 260 byte tokens & $-93.5\%$ & Complete elimination of Out-of-Vocabulary (OOV) risk \\
 \textbf{Total Parameters} & 1,503,744 & 1,159,556 & $-22.9\%$ & Compact embedding table ($260 \times 128$ vs $4000 \times 128$) \\
-\textbf{Sequence Accuracy} & 31.00\% & \textbf{100.00\%} & $+69.00\%$ & Elimination of subword boundary segmentation errors \\
-\textbf{Bit-Level Accuracy} & 88.38\% & \textbf{100.00\%} & $+11.62\%$ & Exact deterministic bit-level byte mapping \\
-\textbf{BLEU-4 / ROUGE-L} & 95.79 / 98.35 & N/A & --- & Token-level metrics omitted for token-free byte model \\
-\textbf{Levenshtein Distance} & 2.782 chars & \textbf{0.000 chars} & $-100.0\%$ & Zero character substitution, insertion, or deletion errors \\
-\textbf{Peak Memory Footprint} & 165.13 MB & 91.43 MB & $-44.6\%$ & Lower embedding and KV activation requirements \\
-\textbf{Training Throughput} & 1,231.7 samples/s & 1,917.3 samples/s & $+55.7\%$ & $4\times$ sequence length reduction via 4-byte patching \\
+\textbf{Sequence Accuracy} & 29.20\% & \textbf{100.00\%} & $+70.80\%$ & Elimination of subword boundary segmentation errors \\
+\textbf{Bit-Level Accuracy} & 88.60\% & \textbf{100.00\%} & $+11.40\%$ & Exact deterministic bit-level byte mapping \\
+\textbf{BLEU-4 / ROUGE-L} & 95.24 / 98.17 & N/A & --- & Token-level metrics omitted for token-free byte model \\
+\textbf{Levenshtein Distance} & 3.038 chars & \textbf{0.000 chars} & $-100.0\%$ & Zero character substitution, insertion, or deletion errors \\
+\textbf{Peak Memory Footprint} & 161.88 MB & 236.38 MB & $+46.0\%$ & Higher local activation volume during 1D conv and unpooling \\
+\textbf{Training Throughput} & 1,236.7 samples/s & 1,957.8 samples/s & $+58.3\%$ & $4\times$ sequence length reduction via 4-byte patching \\
 \bottomrule
 \end{tabular}
 }
@@ -224,12 +224,12 @@ Configuration 5 (C5) replaces the learned subword vocabulary with a token-free B
 
 \subsection{Training Speed and Computational Overhead}
 \textbf{Patch-Level Compression vs.\ Local Overhead:} In C5, grouping $P=4$ bytes into a single patch shortens the global sequence length from $L_{\text{byte}}=64$ to $L_{\text{patch}}=16$. Because self-attention complexity scales quadratically with sequence length ($\mathcal{O}(L^2)$), the global transformer attention computation is reduced by $P^2 = 16\times$. 
-Although the local byte encoder (1D convolution + intra-patch positioning + refinement MLP) and local byte decoder (linear unpooling expansion + local MLP + byte projection head) introduce local operations, the $4\times$ sequence length reduction dominates, delivering a substantial $+55.7\%$ increase in training throughput ($1,917.3$\,samples/s for C5 vs.\ $1,231.7$\,samples/s for C1) and reducing training time from $1018$s to $642$s.
+Although the local byte encoder (1D convolution + intra-patch positioning + refinement MLP) and local byte decoder (linear unpooling expansion + local MLP + byte projection head) introduce local operations, the $4\times$ sequence length reduction dominates, delivering a substantial $+58.3\%$ increase in training throughput ($1,957.8$\,samples/s for C5 vs.\ $1,236.7$\,samples/s for C1) and reducing training time from $1014$s to $629$s.
 
 \subsection{Peak GPU / MPS Memory Usage During Training}
 \textbf{Activation Footprint vs.\ Embedding Memory:} 
 C5 achieves a $22.9\%$ parameter reduction ($1.16$\,M vs.\ $1.50$\,M params) due to replacing the large $4,000$-entry BPE embedding and projection layers with a compact 260-entry byte embedding table. 
-Peak memory during training is also significantly lower for C5 ($91.43$\,MB vs.\ $165.13$\,MB for C1, a $-44.6\%$ reduction) because shorter latent sequence lengths dramatically reduce the memory required for storing attention maps and KV projections during the backward pass.
+Conversely, peak memory during training is higher for C5 ($236.38$\,MB vs.\ $161.88$\,MB for C1, a $+46.0\%$ increase). This increase is attributable to intermediate activation tensors stored during backpropagation across the dense local byte expansion layers ($16 \text{ patches} \times 4 \text{ bytes} \times 128 \text{ dim}$) and unpooled projection heads prior to cross-entropy loss.
 
 \subsection{Overall Sequence Reconstruction Performance on the Test Set}
 \textbf{Cipher Periodicity and Architectural Alignment as the Root Cause of C5's Success:}
@@ -238,7 +238,7 @@ C5 achieves a flawless \textbf{$100.00\%$ Sequence Exact Match Accuracy}, \textb
 Crucially, this perfect recovery is primarily driven by the mathematical alignment between the byte-latent architecture and the underlying data transformation rather than generic model superiority:
 \begin{itemize}[leftmargin=*,noitemsep,topsep=2pt]
 \item \textbf{Deterministic Period-8 Byte Substitution:} Statistical analysis reveals that the dataset encryption operates as an 8-byte periodic XOR substitution keystream. Because C5 aggregates raw bytes into fixed 4-byte patches ($P=4$), each 8-byte cipher cycle is divided into two symmetrical 4-byte sub-blocks (even vs.\ odd patch parity). With intra-patch and global positional encodings providing exact parity awareness, the local encoder and decoder can represent and invert this periodic byte-level mapping deterministically with zero reconstruction error.
-\item \textbf{Failure Mode of Subword Tokenization (C1--C4):} In contrast, the cipher-side BPE tokenizer learns merges from raw byte-pair frequency statistics within fixed 16-byte chunks of ciphertext---not from word-level segmentation. Because merge selection is content-driven rather than position-driven, the resulting variable-length subword tokens begin at content-dependent byte offsets rather than fixed multiples of the 8-byte key period. This breaks the clean patch-parity property that lets BLT recover the periodic mapping exactly, fragmenting the underlying periodic structure across differently-tokenized contexts. During autoregressive decoding, these compounding subword prediction errors limit exact-match sequence accuracy to $25.0\% - 34.2\%$.
+\item \textbf{Failure Mode of Subword Tokenization (C1--C4):} In contrast, the cipher-side BPE tokenizer learns merges from raw byte-pair frequency statistics within fixed 16-byte chunks of ciphertext---not from word-level segmentation. Because merge selection is content-driven rather than position-driven, the resulting variable-length subword tokens begin at content-dependent byte offsets rather than fixed multiples of the 8-byte key period. This breaks the clean patch-parity property that lets BLT recover the periodic mapping exactly, fragmenting the underlying periodic structure across differently-tokenized contexts. During autoregressive decoding, these compounding subword prediction errors limit exact-match sequence accuracy to $25.2\% - 33.8\%$.
 \end{itemize}
 
 \section{Conclusion}