| import sys, torch | |
| sys.path.insert(0, "D:/SVD_Project/Codes") | |
| from transformers import AutoModelForSequenceClassification, AutoTokenizer | |
| from datasets import load_from_disk | |
| from framework.core.attention_utils import inject_attention | |
| data_dir = "D:/SVD_Project/Codes/framework/data/glue_data/sst2" | |
| ds = load_from_disk(data_dir) | |
| val = ds["validation"] | |
| tokenizer = AutoTokenizer.from_pretrained("D:/SVD_Project/models/roberta-base") | |
| x = tokenizer(val["sentence"][:16], padding="max_length", max_length=128, return_tensors="pt").to("cuda") | |
| # 1. Base model with fine-tuned checkpoint | |
| model = AutoModelForSequenceClassification.from_pretrained("D:/SVD_Project/models/roberta-base", num_labels=2).to("cuda") | |
| ckpt_path = "D:/SVD_Project/results/roberta_sst2_lora_baseline/finetuned_model.pt" | |
| ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False) | |
| st = ckpt.get("state_dict", ckpt) | |
| # Load checkpoint state | |
| from framework.run_quality_sweep import main | |
| # Replicate _safe_load_state | |
| model_st = model.state_dict() | |
| filtered = {} | |
| for k, v in st.items(): | |
| if k in model_st and v.shape == model_st[k].shape: | |
| filtered[k] = v | |
| elif k not in filtered: | |
| base_k = k.replace(".weight", ".base_layer.weight").replace(".bias", ".base_layer.bias") | |
| if base_k in st: | |
| filtered[k] = st[base_k] | |
| model.load_state_dict(filtered, strict=False) | |
| model.eval() | |
| with torch.no_grad(): | |
| out_orig = model(**x) | |
| preds_orig = out_orig.logits.argmax(-1) | |
| print("Original fine-tuned RoBERTa accuracy on 16 samples:", (preds_orig == torch.tensor(val["label"][:16], device="cuda")).float().mean().item()) | |
| # 2. Inject adaptive_span | |
| inject_attention(model, "adaptive_span", rank=16, copy_weights=True) | |
| model.eval() | |
| with torch.no_grad(): | |
| out_injected = model(**x) | |
| preds_injected = out_injected.logits.argmax(-1) | |
| print("Injected RoBERTa accuracy on 16 samples:", (preds_injected == torch.tensor(val["label"][:16], device="cuda")).float().mean().item()) | |
| print("Cosine similarity of logits:", torch.cosine_similarity(out_orig.logits, out_injected.logits, dim=-1).mean().item()) | |
Xet Storage Details
- Size:
- 2.1 kB
- Xet hash:
- eae21a10d1dcc495747dfb17bcb100ff9c1cdf017f5a4d9cbd8a9a80b91f35a1
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.