lm-quant-toolkit/data-vis/plot-quant-cfg-llm.R

#!/usr/bin/env Rscript

library(tidyverse)
library(readr)
library(ggthemes)
library(ggplot2)
library(patchwork)
library(optparse)


weight_grid <- function(
    df_wdist, df_kurtosis, mod, show_legend = FALSE, show_cfg = TRUE) {
  df_mod_wdist <- df_wdist |> filter(module == mod)
  df_mod_kurt <- df_kurtosis |> filter(module == mod)
  # Line plot (on top)
  line_plot <- ggplot(df_mod_kurt, aes(x = layer, y = kurtosis)) +
    geom_line(color = "blue") +
    theme_gray(base_size = 14) +
    theme_minimal() +
    theme(
      axis.title.x = element_blank(),
      axis.text.x = element_blank()
    )

  # Bar plot (on bottom)
  module_disp <- df_mod_wdist$mod_disp[1]
  bar_plot <- ggplot(
    df_mod_wdist, aes(x = layer, y = abs_val, fill = nth_percentile)
  ) +
    geom_bar(stat = "identity", color = "gray50") +
    theme_gray(base_size = 14) +
    labs(
      x = module_disp, y = "Absolute Value", fill = "nth percentile"
    )
  if (show_cfg) {
    bar_plot <- bar_plot +
      geom_text(
        data = subset(df_mod_wdist, nth_percentile == 100),
        aes(x = layer, label = quant_cfg),
        angle = 90,
        vjust = 0.20,
        position = position_stack(vjust = 0.5),
        colour = "white",
        size = 2
      )
  }

  if (show_legend) {
    bar_plot <- bar_plot +
      theme(
        legend.position = "bottom",
        legend.text = element_text(size = 16),
        legend.title = element_text(size = 16)
      ) +
      # coord_flip() +
      scale_color_solarized()
  } else {
    bar_plot <- bar_plot +
      theme(legend.position = "none") +
      scale_color_solarized()
  }

  # Combine the line and bar plot vertically
  combined_plot <- line_plot / bar_plot + plot_layout(heights = c(1, 3))
  return(combined_plot)
}

weight_grid_only <- function(
    df_wdist, df_kurtosis, mod, show_legend = FALSE) {
  return(weight_grid(df_wdist, df_kurtosis, mod, show_legend, show_cfg = FALSE))
}

parser <- OptionParser()
parser <- add_option(
  parser, c("-w", "--weight_dist"),
  action = "store_true",
  default = FALSE,
  type = "logical",
  help = "Generate weight column diagram without cfg",
  metavar = "logical"
)
parser <- add_option(
  parser, c("-m", "--model"),
  type = "character",
  help = "The model for which the cfg column diagram is generated",
  metavar = "character"
)
args <- parse_args(parser)


if (is.null(args$model)) {
  model_id <- "Llama-2-7b-hf"
} else {
  model_id <- args$model
}

is_plot_weight <- args$weight_dist

df_cfgs <- read_csv("data/llama-mxq-cfgs.csv")
df_all <- read_csv(paste0("data/wdist/wdist-", model_id, ".csv"))
k_cols <- c("module", "layer", "kurtosis")
df_kurtosis <- df_all |>
  select(all_of(k_cols))

percentiles <- c("0", "99", "99.9", "99.99", "100")
module_param_count <- df_all |>
  select(
    module, param_count
  ) |>
  group_by(module) |>
  summarise(
    param_count = sum(param_count)
  ) |>
  mutate(
    mod_disp = paste0(module, "(", formatC(param_count, big.mark = ","), ")")
  )

budgets <- list(3.51, 3.25, 3.13, 4.51, 4.25, 4.13)

for (budget in budgets) {
  df_cfg_1 <- df_cfgs |>
    filter(bit_budget == budget & model == model_id) |>
    mutate(
      quant_cfg = paste0("b", b1, "g", g1)
    ) |>
    select(-c("b1", "g1", "b2", "g2", "bit_budget"))

  all_cols <- c("module", "layer", percentiles)
  df_wdist <- df_all |>
    mutate(
      `0` = percentile_0,
      `99` = percentile_99 - percentile_0,
      `99.9` = percentile_999 - percentile_99,
      `99.99` = percentile_9999 - percentile_999,
      `100` = percentile_100 - percentile_9999,
    ) |>
    select(all_of(all_cols)) |>
    pivot_longer(
      cols = percentiles,
      names_to = "nth_percentile",
      names_transform = list(nth_percentile = as.numeric),
      values_to = "abs_val"
    ) |>
    mutate(
      nth_percentile = factor(nth_percentile, levels = rev(percentiles))
    ) |>
    # filter(!grepl("_layernorm", module)) |>
    left_join(module_param_count, by = c("module")) |>
    left_join(df_cfg_1, by = c("module", "layer"))

  if (is_plot_weight) {
    p1 <- weight_grid_only(df_wdist, df_kurtosis, "input_layernorm")
    p2 <- weight_grid_only(df_wdist, df_kurtosis, "mlp.down_proj")
    p3 <- weight_grid_only(df_wdist, df_kurtosis, "mlp.gate_proj")
    p4 <- weight_grid_only(df_wdist, df_kurtosis, "mlp.up_proj")
    p5 <- weight_grid_only(df_wdist, df_kurtosis, "post_attention_layernorm")
    p6 <- weight_grid_only(df_wdist, df_kurtosis, "self_attn.k_proj")
    p7 <- weight_grid_only(df_wdist, df_kurtosis, "self_attn.o_proj")
    p8 <- weight_grid_only(df_wdist, df_kurtosis, "self_attn.q_proj", TRUE)
    p9 <- weight_grid_only(df_wdist, df_kurtosis, "self_attn.v_proj")

    # Create a 3x3 grid of combined plots
    final_plot1 <- (p1 | p2 | p3) / (p4 | p5 | p6) / (p7 | p8 | p9)
    ggsave(
      paste0("pdfs/", model_id, "-", budget, "-wdist-kurtosis.pdf"),
      plot = final_plot1,
      width = 16, height = 9
    )
  } else {
    p2 <- weight_grid(df_wdist, df_kurtosis, "mlp.down_proj")
    p3 <- weight_grid(df_wdist, df_kurtosis, "mlp.gate_proj")
    p4 <- weight_grid(df_wdist, df_kurtosis, "mlp.up_proj")
    p6 <- weight_grid(df_wdist, df_kurtosis, "self_attn.k_proj")
    p7 <- weight_grid(df_wdist, df_kurtosis, "self_attn.o_proj")
    p8 <- weight_grid(df_wdist, df_kurtosis, "self_attn.q_proj", TRUE)
    p9 <- weight_grid(df_wdist, df_kurtosis, "self_attn.v_proj")

    final_plot2 <- (p2 | p3 | p4) / (p6 | p7) / (p8 | p9)
    ggsave(
      paste0("pdfs/", model_id, "-", budget, "-mxq-cfgs-from-model.pdf"),
      plot = final_plot2,
      width = 12, height = 9
    )
  }
}