lm-quant-toolkit/data-vis/plot-ppl-mem-inc.R

#!/usr/bin/env Rscript

library(tidyverse)
library(dplyr)
library(readr)
library(openxlsx)
library(ggmagnify)
library(optparse)
library(this.path)

# make reference to library function portable
source(file.path(here("functions"), "utils.R"))
source(file.path(here("functions"), "allocation.R"))

plot_sensi_vs_ablation <- function(
    model_id,
    the_dataset,
    df_ppl_mem_inc,
    df_hqq,
    mag_from,
    mag_to,
    low_bound_scale = 0.99) {
  df_disp <- df_ppl_mem_inc |>
    filter(model == model_id & dataset == the_dataset) |>
    filter(method == "SensiBoost")
  plt <- plot_meta(
    aes(x = increment, y = ppl, shape = ablation, color = ablation),
    "sbab", model_id, the_dataset, df_disp, df_hqq, mag_from, mag_to,
    low_bound_scale
  )
  return(plt)
}

plot_kurt_vs_ablation <- function(
    model_id,
    the_dataset,
    df_ppl_mem_inc,
    df_hqq,
    mag_from,
    mag_to,
    low_bound_scale = 0.99) {
  df_disp <- df_ppl_mem_inc |>
    filter(model == model_id & dataset == the_dataset) |>
    filter(method == "KurtBoost")
  plt <- plot_meta(
    aes(x = increment, y = ppl, shape = ablation, color = ablation),
    "kbab", model_id, the_dataset, df_disp, df_hqq, mag_from, mag_to,
    low_bound_scale
  )
  return(plt)
}

plot_sensi_vs_kurt <- function(
    model_id,
    the_dataset,
    df_ppl_mem_inc,
    df_hqq,
    mag_from,
    mag_to,
    low_bound_scale = 0.99) {
  df_disp <- df_ppl_mem_inc |>
    filter(model == model_id & dataset == the_dataset) |>
    filter(ablation == FALSE)

  plt <- plot_meta(
    aes(x = increment, y = ppl, shape = method, color = method),
    "sk", model_id, the_dataset, df_disp, df_hqq, mag_from, mag_to,
    low_bound_scale
  )
  return(plt)
}

plot_meta <- function(
    ass,
    type,
    model_id,
    the_dataset,
    df_disp,
    df_hqq,
    mag_from,
    mag_to,
    low_bound_scale = 0.99) {
  df_hqq_mark <- df_hqq |> filter(model == model_id & bpp < 8)
  line_hqq_451 <- df_hqq_mark |> filter(bpp == 4.51)
  line_hqq_425 <- df_hqq_mark |> filter(bpp == 4.25)
  line_hqq_413 <- df_hqq_mark |> filter(bpp == 4.13)
  line_hqq_351 <- df_hqq_mark |> filter(bpp == 3.51)
  line_hqq_325 <- df_hqq_mark |> filter(bpp == 3.25)
  line_hqq_313 <- df_hqq_mark |> filter(bpp == 3.13)
  baseline_451 <- ifelse(
    the_dataset == "WikiText2", line_hqq_451$ppl_wikitext, line_hqq_451$ppl_c4
  )
  baseline_425 <- ifelse(
    the_dataset == "WikiText2", line_hqq_425$ppl_wikitext, line_hqq_425$ppl_c4
  )
  baseline_413 <- ifelse(
    the_dataset == "WikiText2", line_hqq_413$ppl_wikitext, line_hqq_413$ppl_c4
  )
  baseline_351 <- ifelse(
    the_dataset == "WikiText2", line_hqq_351$ppl_wikitext, line_hqq_351$ppl_c4
  )
  baseline_325 <- ifelse(
    the_dataset == "WikiText2", line_hqq_325$ppl_wikitext, line_hqq_325$ppl_c4
  )
  baseline_313 <- ifelse(
    the_dataset == "WikiText2", line_hqq_313$ppl_wikitext, line_hqq_313$ppl_c4
  )

  min_ppl <- min(df_disp$ppl) * low_bound_scale
  max_ppl <- max(df_disp$ppl)
  max_hqq_ppl <- ifelse(
    the_dataset == "WikiText2",
    max(df_hqq_mark$ppl_wikitext),
    max(df_hqq_mark$ppl_c4)
  )
  max_ppl <- ifelse(max_ppl > max_hqq_ppl, max_ppl, max_hqq_ppl)
  min_ppl <- floor(min_ppl * 10) / 10
  max_ppl <- ceiling(max_ppl * 10) / 10
  step <- round((max_ppl - min_ppl) / 15, digits = 2)
  legend_title <- "Method:"
  if (type == "sbab") {
    legend_title <- "SensiBoost Ablation:"
  } else if (type == "kbab") {
    legend_title <- "KurtBoost Ablation:"
  }

  # aes(x = increment, y = ppl, shape = method, color = method)
  plt <- ggplot(df_disp, ass) +
    scale_x_continuous(
      limits = c(0, 8.0),
      breaks = seq(0, 8.0, 0.50)
    ) +
    scale_y_continuous(
      limits = c(min_ppl, max_ppl),
      breaks = seq(min_ppl, max_ppl, step)
    ) +
    geom_point(size = 2) +
    geom_hline(
      yintercept = baseline_451,
      linetype = "dashed",
      linewidth = 0.1
    ) +
    annotate(
      "text",
      x = 7.5, y = baseline_451 - 0.01, size = 3, label = "HQQ b4g32"
    ) +
    geom_hline(
      yintercept = baseline_425,
      linetype = "dashed",
      linewidth = 0.1
    ) +
    annotate(
      "text",
      x = 7.5, y = baseline_425 + 0.01, size = 3, label = "HQQ b4g64"
    ) +
    geom_hline(
      yintercept = baseline_413,
      linetype = "dashed",
      linewidth = 0.1
    ) +
    annotate(
      "text",
      x = 7.5, y = baseline_413 + 0.01, size = 3, label = "HQQ b4g128"
    ) +
    geom_hline(
      yintercept = baseline_351,
      linetype = "dashed",
      linewidth = 0.1
    ) +
    annotate(
      "text",
      x = 7.5, y = baseline_351 + 0.01, size = 3, label = "HQQ b3g32"
    ) +
    geom_hline(
      yintercept = baseline_325,
      linetype = "dashed",
      linewidth = 0.1
    ) +
    annotate(
      "text",
      x = 7.5, y = baseline_325 + 0.01, size = 3, label = "HQQ b3g64"
    ) +
    geom_hline(
      yintercept = baseline_313,
      linetype = "dashed",
      linewidth = 0.1
    ) +
    annotate(
      "text",
      x = 7.5, y = baseline_313 - 0.01, size = 3, label = "HQQ b3g128"
    ) +
    geom_magnify(from = mag_from, to = mag_to, axes = "xy") +
    labs(x = "% Memory Increment", y = "Perplexity") +
    theme(
      strip.background = element_rect(
        color = "darkgray", fill = "white", linewidth = 1.0, linetype = "solid"
      ),
      strip.text.x = element_text(face = "bold", size = 12),
      strip.text.y = element_text(face = "bold", size = 12),
      axis.text.x = element_text(size = 12),
      axis.text.y = element_text(size = 12),
      axis.title.x = element_text(size = 14),
      axis.title.y = element_text(size = 14),
      legend.position = "bottom"
    ) +
    guides(
      shape = guide_legend(title = legend_title),
      color = guide_legend(title = legend_title)
    ) +
    facet_grid(dataset ~ model, scales = "free")

  ggsave(
    paste0("pdfs/ppl-", type, "-", model_id, "-", the_dataset, ".pdf"),
    plot = plt,
    width = 8,
    height = 6,
    dpi = 600
  )
  return(plt)
}

parser <- OptionParser()
parser <- add_option(
  parser, c("-d", "--combined_csv_file"),
  type = "character",
  help = "The combined csv file",
  metavar = "character"
)
parser <- add_option(
  parser, c("-a", "--allot_csv_file"),
  type = "character",
  help = "Allocation CSV file",
  metavar = "character"
)
parser <- add_option(
  parser, c("-t", "--type"),
  type = "character",
  help = "Type diagram",
  metavar = "character"
)

args <- parse_args(parser)
if (is.null(args$csv_file)) {
  combined_csv_fp <- "data/combined.csv"
} else {
  combined_csv_fp <- args$combined_csv_file
}
if (is.null(args$csv_file)) {
  allot_cfg_csv_fp <- "data/quant-cfg-allocation.csv"
} else {
  allot_cfg_csv_fp <- args$allot_csv_file
}
if (is.null(args$type)) {
  type <- "sensi-vs-kurt"
} else {
  type <- args$type
}



if (type == "sensi-vs-kurt") {
  plot_func <- plot_sensi_vs_kurt
} else if (type == "sensi-vs-ablation") {
  plot_func <- plot_sensi_vs_ablation
} else if (type == "kurt-vs-ablation") {
  plot_func <- plot_kurt_vs_ablation
}

tup <- load_ppl_mem_inc(allot_cfg_csv_fp, combined_csv_fp)
df_ppl_mem_inc <- tup[[1]]
df_hqq <- tup[[2]]

write.xlsx(df_ppl_mem_inc, "df_ppl_mem_inc.xlsx", asTable = TRUE)
models <- unique(df_ppl_mem_inc$model)
# models <- c("Llama-2-13B")
for (model_id in models) {
  if (model_id == "Llama-2-7B") {
    from_wk <- c(2.5, 3.2, 5.25, 5.38)
    to_wk <- c(5, 7, 5.5, 6.0)
    from_c4 <- c(2.5, 3.2, 7.04, 7.18)
    to_c4 <- c(4, 6, 7.4, 8.08)
    low_bound_scale <- 1.0
  } else if (model_id == "Llama-2-13B") {
    from_wk <- c(1.9, 2.5, 4.66, 4.75)
    to_wk <- c(4.5, 6.5, 4.85, 5.15)
    from_c4 <- c(1.9, 2.5, 6.5, 6.6)
    to_c4 <- c(4, 6.0, 6.7, 7.00)
    low_bound_scale <- 0.99
  } else if (model_id == "Llama-3-8B") {
    from_wk <- c(2.5, 3.2, 6.02, 6.42)
    to_wk <- c(4.0, 6.0, 6.80, 8.65)
    from_c4 <- c(2.5, 3.2, 9.31, 10.04)
    to_c4 <- c(4, 6, 11, 14)
    low_bound_scale <- 0.98
  }
  plot_func(
    model_id,
    "WikiText2",
    df_ppl_mem_inc,
    df_hqq,
    from_wk,
    to_wk,
    low_bound_scale
  )
  plot_func(
    model_id,
    "C4",
    df_ppl_mem_inc,
    df_hqq,
    from_c4,
    to_c4,
    low_bound_scale
  )
}