File size: 3,476 Bytes
998922f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 | library(tidyverse)
library(ggthemes)
library(readr)
zs_all_cols <- c(
"model", "algo", "config",
"bpp", "zeroshot_mem_allot", "acc1_zeroshot_cls",
"acc5_zeroshot_cls", "recall_zeroshot_cls", "duration_zeroshot_cls"
)
lp_all_cols <- c(
"model", "algo", "config",
"bpp", "linear_probe_mem_allot", "acc1_linear_probe",
"acc5_linear_probe", "recall_linear_probe", "duration_linear_probe"
)
df_all <- read_csv("data/vit/combined.csv")
df_zeroshot <- df_all |>
select(all_of(zs_all_cols)) |>
rename(
mem = zeroshot_mem_allot,
acc1 = acc1_zeroshot_cls,
acc5 = acc5_zeroshot_cls,
recall = recall_zeroshot_cls,
duration = duration_zeroshot_cls
)
df_linear_probe <- df_all |>
select(all_of(lp_all_cols)) |>
rename(
mem = linear_probe_mem_allot,
acc1 = acc1_linear_probe,
acc5 = acc5_linear_probe,
recall = recall_linear_probe,
duration = duration_linear_probe
)
zeroshot_h <- df_zeroshot |>
filter(
grepl("ViT-H-14", model)
)
max_acc1 <- max(zeroshot_h$acc1)
max_acc5 <- max(zeroshot_h$acc5)
ggplot(zeroshot_h, aes(x = bpp, y = acc1)) +
geom_point(size = 2.5, aes(shape = algo, color = algo, y = acc1)) +
ylim(max_acc1 * 0.95, max_acc5 * 1.01) +
geom_hline(yintercept = max_acc1 * 0.99, linetype = "dotted", color = "blue") +
geom_hline(yintercept = max_acc1, linetype = "dotted", color = "blue") +
annotate("text", x = 14, y = max_acc1 * 1.01, label = "0-shot Top-1 FP16") +
geom_point(size = 2.5, aes(shape = algo, color = algo, y = acc5)) +
geom_hline(yintercept = max_acc5 * 0.99, linetype = "dotted", color = "blue") +
geom_hline(yintercept = max_acc5, linetype = "dotted", color = "blue") +
annotate("text", x = 14, y = max_acc5 * 1.01, label = "0-shot Top-5 FP16") +
scale_x_continuous(trans = "log2") +
labs(x = "Bit Budget", y = "Zero-shot Classification Accuracy") +
theme_gray(base_size = 16) +
theme(
legend.position = "bottom",
legend.text = element_text(size = 16),
legend.title = element_text(size = 16)
) +
# facet_wrap(~model, scales = "free") +
scale_color_solarized()
ggsave("pdfs/mxq-vit-h-14-zs.pdf", width = 8, height = 6)
linear_probe_h <- df_linear_probe |>
filter(
grepl("ViT-H-14", model)
)
bmax_acc1 <- max(linear_probe_h$acc1)
bmax_acc5 <- max(linear_probe_h$acc5)
ggplot(linear_probe_h, aes(x = bpp, y = acc1)) +
geom_point(size = 2.5, aes(shape = algo, color = algo, y = acc1)) +
ylim(bmax_acc1 * 0.95, bmax_acc5 * 1.01) +
geom_hline(yintercept = bmax_acc1 * 0.99, linetype = "dotted", color = "blue") +
geom_hline(yintercept = bmax_acc1, linetype = "dotted", color = "blue") +
annotate("text", x = 14, y = bmax_acc1 * 1.01, label = "Linear Top-1 FP16") +
geom_point(size = 2.5, aes(shape = algo, color = algo, y = acc5)) +
geom_hline(yintercept = bmax_acc5 * 0.99, linetype = "dotted", color = "blue") +
geom_hline(yintercept = bmax_acc5, linetype = "dotted", color = "blue") +
annotate("text", x = 14, y = bmax_acc5 * 1.01, label = "Linear Top-5 FP16") +
scale_x_continuous(trans = "log2") +
labs(x = "Bit Budget", y = "Linear Probe Accuracy") +
theme_gray(base_size = 16) +
theme(
legend.position = "bottom",
legend.text = element_text(size = 16),
legend.title = element_text(size = 16)
) +
# facet_wrap(~model, scales = "free") +
scale_color_solarized()
ggsave("pdfs/mxq-vit-h-14-lp.pdf", width = 8, height = 6)
# geom_point(aes(size = mem, color = algo)) +
# geom_smooth() +
|