samwjya commited on
Commit Β·
a32e9c3
1
Parent(s): 81131cf
add paper link
Browse files- __pycache__/evaluate.cpython-313.pyc +0 -0
- __pycache__/reranker.cpython-313.pyc +0 -0
- __pycache__/retriever.cpython-313.pyc +0 -0
- evaluate.py +12 -0
- explanation.txt +416 -0
- main.py +1 -0
- retriever.py +5 -1
__pycache__/evaluate.cpython-313.pyc
ADDED
|
Binary file (8.65 kB). View file
|
|
|
__pycache__/reranker.cpython-313.pyc
ADDED
|
Binary file (4.97 kB). View file
|
|
|
__pycache__/retriever.cpython-313.pyc
ADDED
|
Binary file (7.42 kB). View file
|
|
|
evaluate.py
CHANGED
|
@@ -118,6 +118,18 @@ def print_results(query: str, bm25_papers: list[dict], reranked_papers: list[dic
|
|
| 118 |
print(f"{'MRR':<20} {metrics['bm25_mrr']:>10.4f} {metrics['ce_mrr']:>14.4f}")
|
| 119 |
print("-" * 40)
|
| 120 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 121 |
|
| 122 |
if __name__ == "__main__":
|
| 123 |
import argparse
|
|
|
|
| 118 |
print(f"{'MRR':<20} {metrics['bm25_mrr']:>10.4f} {metrics['ce_mrr']:>14.4f}")
|
| 119 |
print("-" * 40)
|
| 120 |
|
| 121 |
+
print()
|
| 122 |
+
print("BM25 Links:")
|
| 123 |
+
for i, p in enumerate(bm25_papers[:k], start=1):
|
| 124 |
+
url = p.get("url") or "N/A"
|
| 125 |
+
print(f" {i:>2}. {p.get('title', '')[:70]} β {url}")
|
| 126 |
+
|
| 127 |
+
print()
|
| 128 |
+
print("Reranked Links:")
|
| 129 |
+
for i, p in enumerate(reranked_papers[:k], start=1):
|
| 130 |
+
url = p.get("url") or "N/A"
|
| 131 |
+
print(f" {i:>2}. {p.get('title', '')[:70]} β {url}")
|
| 132 |
+
|
| 133 |
|
| 134 |
if __name__ == "__main__":
|
| 135 |
import argparse
|
explanation.txt
ADDED
|
@@ -0,0 +1,416 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
================================================================================
|
| 2 |
+
PaperRank β End-to-End Walkthrough
|
| 3 |
+
Input: python main.py "kubernetes autoscaling" --limit 10 --k 5
|
| 4 |
+
================================================================================
|
| 5 |
+
|
| 6 |
+
This document traces every function call the system makes, what data looks like
|
| 7 |
+
at each step, and how it produces the final table and metrics shown in the output.
|
| 8 |
+
|
| 9 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 10 |
+
STEP 0 β CLI Argument Parsing (main.py β parse_args)
|
| 11 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 12 |
+
|
| 13 |
+
main.py receives the command-line string and parse_args() maps it into:
|
| 14 |
+
|
| 15 |
+
args.query = "kubernetes autoscaling"
|
| 16 |
+
args.limit = 10 (fetch at most 10 candidates from OpenAlex)
|
| 17 |
+
args.k = 5 (evaluate/display top 5)
|
| 18 |
+
args.model = "BAAI/bge-reranker-v2-m3" (default)
|
| 19 |
+
args.relevant_ids = None (no manual judgments β use auto-relevance)
|
| 20 |
+
args.no_rerank = False (run the full pipeline)
|
| 21 |
+
|
| 22 |
+
main() then calls the three pipeline stages in order.
|
| 23 |
+
|
| 24 |
+
|
| 25 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 26 |
+
STEP 1 β Fetch Papers from OpenAlex (retriever.py β fetch_papers)
|
| 27 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 28 |
+
|
| 29 |
+
retrieve("kubernetes autoscaling", limit=10) is called, which calls fetch_papers.
|
| 30 |
+
|
| 31 |
+
fetch_papers does three things:
|
| 32 |
+
|
| 33 |
+
1a. Load the API key
|
| 34 |
+
_load_api_key() opens .env, finds the line "OPEN_ALEX_API=f0K6qBo0xkkQAWM6SrgrDP",
|
| 35 |
+
and returns the key string.
|
| 36 |
+
|
| 37 |
+
1b. Call the OpenAlex API
|
| 38 |
+
GET https://api.openalex.org/works
|
| 39 |
+
?search.semantic=kubernetes+autoscaling
|
| 40 |
+
&per_page=10
|
| 41 |
+
&api_key=f0K6qBo0xkkQAWM6SrgrDP
|
| 42 |
+
|
| 43 |
+
OpenAlex returns a JSON object. The relevant part is:
|
| 44 |
+
{
|
| 45 |
+
"results": [
|
| 46 |
+
{
|
| 47 |
+
"id": "https://openalex.org/W3123456789",
|
| 48 |
+
"title": "COPA: A Combined Autoscaling Method for Kubernetes",
|
| 49 |
+
"abstract_inverted_index": {
|
| 50 |
+
"Kubernetes": [0, 14],
|
| 51 |
+
"autoscaling": [1, 20],
|
| 52 |
+
"horizontal": [2],
|
| 53 |
+
...
|
| 54 |
+
},
|
| 55 |
+
"publication_year": 2022,
|
| 56 |
+
"cited_by_count": 41,
|
| 57 |
+
"authorships": [
|
| 58 |
+
{ "author": { "display_name": "Jane Doe" } },
|
| 59 |
+
...
|
| 60 |
+
]
|
| 61 |
+
},
|
| 62 |
+
... (up to 10 entries)
|
| 63 |
+
]
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
1c. Reconstruct each abstract
|
| 67 |
+
OpenAlex does not return plain text abstracts. It stores them as an
|
| 68 |
+
"inverted index": a dict mapping each word to the list of positions it
|
| 69 |
+
occupies in the original sentence.
|
| 70 |
+
|
| 71 |
+
_reconstruct_abstract({"Kubernetes": [0,14], "autoscaling": [1,20], ...})
|
| 72 |
+
works like this:
|
| 73 |
+
- Find the highest position across all words β max_pos = 20
|
| 74 |
+
- Allocate a list of 21 empty slots: words = ["", "", ..., ""]
|
| 75 |
+
- For every word and every position it appears at, place the word there:
|
| 76 |
+
words[0] = "Kubernetes"
|
| 77 |
+
words[1] = "autoscaling"
|
| 78 |
+
words[2] = "horizontal"
|
| 79 |
+
words[14] = "Kubernetes"
|
| 80 |
+
words[20] = "autoscaling"
|
| 81 |
+
...
|
| 82 |
+
- Join with spaces β "Kubernetes autoscaling horizontal ... Kubernetes ... autoscaling"
|
| 83 |
+
|
| 84 |
+
Papers with an empty or missing abstract_inverted_index are dropped here
|
| 85 |
+
because BM25 and the CrossEncoder both need text to work with. This is
|
| 86 |
+
why the output says "Retrieved 8 papers" even though --limit 10 was given:
|
| 87 |
+
2 of the 10 OpenAlex results had no abstract.
|
| 88 |
+
|
| 89 |
+
After fetch_papers, each paper is a plain dict with these keys:
|
| 90 |
+
{
|
| 91 |
+
"paperId": "https://openalex.org/W3123456789",
|
| 92 |
+
"title": "COPA: A Combined Autoscaling Method for Kubernetes",
|
| 93 |
+
"abstract": "Kubernetes autoscaling horizontal ...", β reconstructed
|
| 94 |
+
"year": 2022,
|
| 95 |
+
"citationCount": 41,
|
| 96 |
+
"authors": [{"name": "Jane Doe"}, ...]
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
At this point we have an unordered list of 8 such dicts.
|
| 100 |
+
|
| 101 |
+
Output printed to terminal:
|
| 102 |
+
[1/3] Fetching up to 10 papers from OpenAlex ...
|
| 103 |
+
Retrieved 8 papers with abstracts.
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 107 |
+
STEP 2 β BM25 Baseline Ranking (retriever.py β bm25_rank + tokenize)
|
| 108 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 109 |
+
|
| 110 |
+
Still inside retrieve(), bm25_rank("kubernetes autoscaling", papers) runs next.
|
| 111 |
+
|
| 112 |
+
2a. Build the corpus
|
| 113 |
+
For every paper, tokenize(title + " " + abstract) is called.
|
| 114 |
+
tokenize just does: text.lower().split()
|
| 115 |
+
|
| 116 |
+
Example for the COPA paper:
|
| 117 |
+
input : "COPA: A Combined Autoscaling Method for Kubernetes. Kubernetes autoscaling ..."
|
| 118 |
+
output: ["copa:", "a", "combined", "autoscaling", "method", "for", "kubernetes",
|
| 119 |
+
"kubernetes", "autoscaling", ...]
|
| 120 |
+
|
| 121 |
+
This produces a list of 8 token-lists (the corpus).
|
| 122 |
+
|
| 123 |
+
2b. Build BM25 index
|
| 124 |
+
BM25Okapi(corpus) builds an inverted index over all 8 documents, computing
|
| 125 |
+
term frequencies and inverse document frequencies for every token.
|
| 126 |
+
|
| 127 |
+
2c. Score the query
|
| 128 |
+
tokenize("kubernetes autoscaling") β ["kubernetes", "autoscaling"]
|
| 129 |
+
bm25.get_scores(["kubernetes", "autoscaling"]) returns one float per document,
|
| 130 |
+
measuring how well each paper matches the query tokens relative to all others.
|
| 131 |
+
|
| 132 |
+
Example scores (illustrative):
|
| 133 |
+
COPA paper β 1.1509
|
| 134 |
+
Survey paper β 1.1054
|
| 135 |
+
HPA paper β 1.0840
|
| 136 |
+
Load prediction β 1.0556
|
| 137 |
+
Proactive paper β 1.0003
|
| 138 |
+
ML-Based scaling β 0.9100
|
| 139 |
+
Edge computing β 0.8750
|
| 140 |
+
Resource mgmt β 0.6200
|
| 141 |
+
|
| 142 |
+
2d. Sort and annotate
|
| 143 |
+
Papers are sorted by BM25 score descending. Each dict gets two new fields:
|
| 144 |
+
paper["bm25_score"] = 1.1509 (rounded to 4 dp)
|
| 145 |
+
paper["bm25_rank"] = 1
|
| 146 |
+
|
| 147 |
+
The COPA paper is BM25 rank 1, Survey is rank 2, and so on.
|
| 148 |
+
|
| 149 |
+
retrieve() returns this list of 8 dicts, sorted by bm25_rank.
|
| 150 |
+
This list is stored as bm25_papers in main().
|
| 151 |
+
|
| 152 |
+
|
| 153 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 154 |
+
STEP 3 β Load the CrossEncoder Model (reranker.py β load_model)
|
| 155 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 156 |
+
|
| 157 |
+
load_model("BAAI/bge-reranker-v2-m3") calls:
|
| 158 |
+
CrossEncoder("BAAI/bge-reranker-v2-m3", max_length=512)
|
| 159 |
+
|
| 160 |
+
This downloads the model weights from HuggingFace on first run (cached locally
|
| 161 |
+
after that). The model is a BERT-style transformer fine-tuned specifically for
|
| 162 |
+
relevance scoring: given a (query, document) pair it outputs a single float
|
| 163 |
+
indicating how relevant the document is to the query. Higher = more relevant.
|
| 164 |
+
|
| 165 |
+
max_length=512 means each (query + document) pair is truncated to 512 tokens.
|
| 166 |
+
Abstracts longer than ~400 tokens will be cut off; this is acceptable because
|
| 167 |
+
the key relevance signals are usually front-loaded in academic abstracts.
|
| 168 |
+
|
| 169 |
+
Output printed to terminal:
|
| 170 |
+
[2/3] Loading CrossEncoder model: BAAI/bge-reranker-v2-m3
|
| 171 |
+
Loading reranker model: BAAI/bge-reranker-v2-m3 ...
|
| 172 |
+
Model loaded.
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 176 |
+
STEP 4 β CrossEncoder Reranking (reranker.py β build_pairs + rerank)
|
| 177 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 178 |
+
|
| 179 |
+
rerank("kubernetes autoscaling", bm25_papers, model) runs in two sub-steps.
|
| 180 |
+
|
| 181 |
+
4a. Build (query, document) pairs β build_pairs()
|
| 182 |
+
For each of the 8 papers, concatenate title + ". " + abstract:
|
| 183 |
+
|
| 184 |
+
pair 1: ("kubernetes autoscaling",
|
| 185 |
+
"COPA: A Combined Autoscaling Method for Kubernetes. Kubernetes ...")
|
| 186 |
+
pair 2: ("kubernetes autoscaling",
|
| 187 |
+
"A Survey of Autoscaling in Kubernetes. This survey covers ...")
|
| 188 |
+
... (8 pairs total)
|
| 189 |
+
|
| 190 |
+
The document uses both title and abstract because:
|
| 191 |
+
- The title gives a dense topical signal (the CrossEncoder reads it first)
|
| 192 |
+
- The abstract provides the detailed evidence for relevance
|
| 193 |
+
|
| 194 |
+
4b. Score all pairs β model.predict()
|
| 195 |
+
model.predict(pairs, batch_size=16) sends all 8 pairs through the transformer
|
| 196 |
+
in a single batch. For each pair the model outputs one raw logit (unbounded float).
|
| 197 |
+
sentence-transformers applies a sigmoid to map it to [0, 1].
|
| 198 |
+
|
| 199 |
+
Example CE scores:
|
| 200 |
+
Survey paper β 0.9995 β CrossEncoder prefers this over COPA
|
| 201 |
+
COPA paper β 0.9988
|
| 202 |
+
ML-Based scaling β 0.9982
|
| 203 |
+
HPA paper β 0.9980
|
| 204 |
+
Load prediction β 0.9957
|
| 205 |
+
Proactive paper β 0.9900
|
| 206 |
+
Edge computing β 0.9750
|
| 207 |
+
Resource mgmt β 0.9500
|
| 208 |
+
|
| 209 |
+
Notice the Survey paper jumped from BM25 rank 2 to CE rank 1. BM25 only
|
| 210 |
+
matched keyword frequency; the CrossEncoder read the full content and
|
| 211 |
+
decided the survey is a better overall answer to "kubernetes autoscaling".
|
| 212 |
+
|
| 213 |
+
4c. Sort and annotate
|
| 214 |
+
Papers sorted by CE score descending. Each dict gets two new fields:
|
| 215 |
+
paper["ce_score"] = 0.9995
|
| 216 |
+
paper["ce_rank"] = 1
|
| 217 |
+
|
| 218 |
+
Each paper dict now has ALL six ranking fields:
|
| 219 |
+
paperId, title, abstract, year, citationCount, authors,
|
| 220 |
+
bm25_score, bm25_rank, ce_score, ce_rank
|
| 221 |
+
|
| 222 |
+
Output printed to terminal:
|
| 223 |
+
Scoring 8 (query, paper) pairs ...
|
| 224 |
+
Batches: 100%|ββββββββββββ| 1/1 [00:35<00:00, 35.17s/it]
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 228 |
+
STEP 5 β Evaluation (evaluate.py β evaluate + auto_relevance + assign_relevance
|
| 229 |
+
β ndcg_at_k + dcg_at_k + mrr)
|
| 230 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 231 |
+
|
| 232 |
+
evaluate(bm25_papers, reranked_papers, relevant_ids=None, k=5) is called.
|
| 233 |
+
|
| 234 |
+
Because relevant_ids is None (no --relevant-ids flag was given), the system
|
| 235 |
+
cannot know the ground truth. It uses pseudo-relevance instead.
|
| 236 |
+
|
| 237 |
+
5a. auto_relevance() β derive a relevance set automatically
|
| 238 |
+
Takes the bm25_papers list, sorts by bm25_rank, and treats the top-5
|
| 239 |
+
(= k) as "relevant". It returns their paperIds as a set:
|
| 240 |
+
|
| 241 |
+
relevant_ids = {
|
| 242 |
+
"https://openalex.org/W_copa",
|
| 243 |
+
"https://openalex.org/W_survey",
|
| 244 |
+
"https://openalex.org/W_hpa",
|
| 245 |
+
"https://openalex.org/W_loadpred",
|
| 246 |
+
"https://openalex.org/W_proactive"
|
| 247 |
+
}
|
| 248 |
+
|
| 249 |
+
Interpretation: we are asking "did the CrossEncoder surface the same top-5
|
| 250 |
+
papers that BM25 found?" This is a relative comparison, not an absolute
|
| 251 |
+
quality measure.
|
| 252 |
+
|
| 253 |
+
5b. assign_relevance() β convert paper lists to relevance score lists
|
| 254 |
+
For each list (BM25 top-5 and CE top-5), map each paper to 1.0 if its
|
| 255 |
+
paperId is in relevant_ids, else 0.0.
|
| 256 |
+
|
| 257 |
+
BM25 top-5 (already sorted by bm25_rank):
|
| 258 |
+
rank 1 COPA β paperId in relevant_ids β 1.0
|
| 259 |
+
rank 2 Survey β paperId in relevant_ids β 1.0
|
| 260 |
+
rank 3 HPA β paperId in relevant_ids β 1.0
|
| 261 |
+
rank 4 Load pred β paperId in relevant_ids β 1.0
|
| 262 |
+
rank 5 Proactive β paperId in relevant_ids β 1.0
|
| 263 |
+
bm25_rels = [1.0, 1.0, 1.0, 1.0, 1.0]
|
| 264 |
+
|
| 265 |
+
CE top-5 (sorted by ce_rank):
|
| 266 |
+
rank 1 Survey β paperId in relevant_ids β 1.0
|
| 267 |
+
rank 2 COPA β paperId in relevant_ids β 1.0
|
| 268 |
+
rank 3 ML-Based β paperId NOT in relevant_ids β 0.0
|
| 269 |
+
rank 4 HPA β paperId in relevant_ids β 1.0
|
| 270 |
+
rank 5 Load pred β paperId in relevant_ids β 1.0
|
| 271 |
+
ce_rels = [1.0, 1.0, 0.0, 1.0, 1.0]
|
| 272 |
+
|
| 273 |
+
The ML-Based paper (originally BM25 rank 6) was promoted to CE rank 3,
|
| 274 |
+
displacing the Proactive paper, which drops out of the top-5 window.
|
| 275 |
+
|
| 276 |
+
5c. ndcg_at_k() and dcg_at_k() β compute NDCG@5
|
| 277 |
+
|
| 278 |
+
DCG formula: sum of rel_i / log2(i+1) for i = 1..k
|
| 279 |
+
|
| 280 |
+
BM25 DCG@5:
|
| 281 |
+
1.0/log2(2) + 1.0/log2(3) + 1.0/log2(4) + 1.0/log2(5) + 1.0/log2(6)
|
| 282 |
+
= 1.000 + 0.631 + 0.500 + 0.431 + 0.387
|
| 283 |
+
= 2.948
|
| 284 |
+
|
| 285 |
+
Ideal DCG@5 (sorted relevances = [1,1,1,1,1], same as BM25 here):
|
| 286 |
+
= 2.948
|
| 287 |
+
|
| 288 |
+
BM25 NDCG@5 = 2.948 / 2.948 = 1.0000 (perfect, because BM25 defined relevance)
|
| 289 |
+
|
| 290 |
+
CE DCG@5 (rels = [1,1,0,1,1]):
|
| 291 |
+
1.0/log2(2) + 1.0/log2(3) + 0.0/log2(4) + 1.0/log2(5) + 1.0/log2(6)
|
| 292 |
+
= 1.000 + 0.631 + 0.000 + 0.431 + 0.387
|
| 293 |
+
= 2.448 (missing the rank-3 contribution)
|
| 294 |
+
|
| 295 |
+
CE NDCG@5 = 2.448 / 2.948 = 0.9558
|
| 296 |
+
|
| 297 |
+
5d. mrr() β compute MRR
|
| 298 |
+
|
| 299 |
+
MRR = 1 / (rank of first relevant document)
|
| 300 |
+
|
| 301 |
+
BM25: first relevant is at rank 1 β MRR = 1/1 = 1.0000
|
| 302 |
+
CE: first relevant is at rank 1 β MRR = 1/1 = 1.0000
|
| 303 |
+
(Both lists have a relevant paper at rank 1, so MRR is the same.)
|
| 304 |
+
|
| 305 |
+
evaluate() returns:
|
| 306 |
+
{
|
| 307 |
+
"bm25_ndcg": 1.0000,
|
| 308 |
+
"bm25_mrr": 1.0000,
|
| 309 |
+
"ce_ndcg": 0.9558,
|
| 310 |
+
"ce_mrr": 1.0000
|
| 311 |
+
}
|
| 312 |
+
|
| 313 |
+
|
| 314 |
+
βββββββββββββββββββββββββββββοΏ½οΏ½ββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 315 |
+
STEP 6 β Print Results (evaluate.py β print_results + build_comparison_table)
|
| 316 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 317 |
+
|
| 318 |
+
print_results("kubernetes autoscaling", bm25_papers, reranked_papers, metrics, k=5)
|
| 319 |
+
|
| 320 |
+
6a. build_comparison_table(bm25_papers, reranked_papers, top_k=5)
|
| 321 |
+
Iterates positions 1..5. At each position i, picks bm25_papers[i] and
|
| 322 |
+
reranked_papers[i], trims titles to 60 characters, and builds a DataFrame row:
|
| 323 |
+
|
| 324 |
+
i=0: BM25 rank 1 = COPA (1.1509) CE rank 1 = Survey (0.9995)
|
| 325 |
+
i=1: BM25 rank 2 = Survey (1.1054) CE rank 2 = COPA (0.9988)
|
| 326 |
+
i=2: BM25 rank 3 = HPA (1.0840) CE rank 3 = ML-Based (0.9982)
|
| 327 |
+
i=3: BM25 rank 4 = Load pred (1.0556) CE rank 4 = HPA (0.9980)
|
| 328 |
+
i=4: BM25 rank 5 = Proactive (1.0003) CE rank 5 = Load pred (0.9957)
|
| 329 |
+
|
| 330 |
+
pandas prints this as the side-by-side table.
|
| 331 |
+
|
| 332 |
+
6b. Print the metrics block beneath the table using the dict from step 5.
|
| 333 |
+
|
| 334 |
+
Final terminal output:
|
| 335 |
+
[3/3] Evaluating and displaying results ...
|
| 336 |
+
|
| 337 |
+
================================================================================
|
| 338 |
+
Query: 'kubernetes autoscaling'
|
| 339 |
+
================================================================================
|
| 340 |
+
BM25 Title BM25 Score CE Title CE Score
|
| 341 |
+
Rank
|
| 342 |
+
1 COPA: A Combined Autoscaling Method for Kubernetes 1.1509 A Survey of Autoscaling in Kubernetes 0.9995
|
| 343 |
+
2 A Survey of Autoscaling in Kubernetes 1.1054 COPA: A Combined Autoscaling Method for Kubernetes 0.9988
|
| 344 |
+
3 Horizontal Pod Autoscaling in Kubernetes for Elastic Contain 1.0840 Machine Learning-Based Scaling Management for Kubernetes Edg 0.9982
|
| 345 |
+
4 Toward Optimal Load Prediction and Customizable Autoscaling 1.0556 Horizontal Pod Autoscaling in Kubernetes for Elastic Contain 0.9980
|
| 346 |
+
5 Proactive autoscaling for edge computing systems with kubern 1.0003 Toward Optimal Load Prediction and Customizable Autoscaling 0.9957
|
| 347 |
+
|
| 348 |
+
----------------------------------------
|
| 349 |
+
Metric BM25 CrossEncoder
|
| 350 |
+
----------------------------------------
|
| 351 |
+
NDCG@5 1.0000 0.9558
|
| 352 |
+
MRR 1.0000 1.0000
|
| 353 |
+
----------------------------------------
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 357 |
+
SUMMARY β Data Flow Diagram
|
| 358 |
+
ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 359 |
+
|
| 360 |
+
"kubernetes autoscaling" + --limit 10 + --k 5
|
| 361 |
+
β
|
| 362 |
+
βΌ
|
| 363 |
+
[main.py parse_args]
|
| 364 |
+
args.query="kubernetes autoscaling", args.limit=10, args.k=5
|
| 365 |
+
β
|
| 366 |
+
βΌ
|
| 367 |
+
[retriever.fetch_papers]
|
| 368 |
+
GET OpenAlex API β 10 raw results β drop 2 (no abstract)
|
| 369 |
+
_reconstruct_abstract() on each β 8 paper dicts with plain-text abstracts
|
| 370 |
+
β
|
| 371 |
+
βΌ
|
| 372 |
+
[retriever.bm25_rank]
|
| 373 |
+
tokenize all titles+abstracts β BM25Okapi index
|
| 374 |
+
score ["kubernetes","autoscaling"] against each doc
|
| 375 |
+
sort descending β add bm25_score + bm25_rank to each dict
|
| 376 |
+
β
|
| 377 |
+
βΌ bm25_papers (8 dicts, sorted by bm25_rank)
|
| 378 |
+
β
|
| 379 |
+
βΌ
|
| 380 |
+
[reranker.load_model]
|
| 381 |
+
CrossEncoder("BAAI/bge-reranker-v2-m3") loaded into memory
|
| 382 |
+
β
|
| 383 |
+
βΌ
|
| 384 |
+
[reranker.build_pairs]
|
| 385 |
+
8 Γ (query, "title. abstract") tuples
|
| 386 |
+
β
|
| 387 |
+
βΌ
|
| 388 |
+
[reranker.model.predict]
|
| 389 |
+
transformer forward pass β 8 relevance scores in [0,1]
|
| 390 |
+
β
|
| 391 |
+
βΌ
|
| 392 |
+
[reranker.rerank β sort + annotate]
|
| 393 |
+
sort by CE score descending β add ce_score + ce_rank to each dict
|
| 394 |
+
β
|
| 395 |
+
βΌ reranked_papers (same 8 dicts, now sorted by ce_rank)
|
| 396 |
+
β
|
| 397 |
+
βΌ
|
| 398 |
+
[evaluate.auto_relevance]
|
| 399 |
+
top-5 by bm25_rank β relevant_ids set (5 paperIds)
|
| 400 |
+
β
|
| 401 |
+
βΌ
|
| 402 |
+
[evaluate.assign_relevance]
|
| 403 |
+
bm25_papers[:5] β [1,1,1,1,1]
|
| 404 |
+
reranked_papers[:5] β [1,1,0,1,1]
|
| 405 |
+
β
|
| 406 |
+
βΌ
|
| 407 |
+
[evaluate.ndcg_at_k + evaluate.mrr]
|
| 408 |
+
BM25 NDCG@5=1.0000 BM25 MRR=1.0000
|
| 409 |
+
CE NDCG@5=0.9558 CE MRR=1.0000
|
| 410 |
+
β
|
| 411 |
+
βΌ
|
| 412 |
+
[evaluate.build_comparison_table + evaluate.print_results]
|
| 413 |
+
pandas DataFrame (5 rows Γ 4 cols) + metrics block
|
| 414 |
+
β
|
| 415 |
+
βΌ
|
| 416 |
+
FINAL OUTPUT printed to terminal
|
main.py
CHANGED
|
@@ -57,6 +57,7 @@ def main() -> None:
|
|
| 57 |
print(f"[#{paper['bm25_rank']}] score={paper['bm25_score']}")
|
| 58 |
print(f" {paper['title']}")
|
| 59 |
print(f" {paper.get('year', 'N/A')} | {paper.get('citationCount', 'N/A')} citations | {authors}")
|
|
|
|
| 60 |
print()
|
| 61 |
return
|
| 62 |
|
|
|
|
| 57 |
print(f"[#{paper['bm25_rank']}] score={paper['bm25_score']}")
|
| 58 |
print(f" {paper['title']}")
|
| 59 |
print(f" {paper.get('year', 'N/A')} | {paper.get('citationCount', 'N/A')} citations | {authors}")
|
| 60 |
+
print(f" {paper.get('url') or 'N/A'}")
|
| 61 |
print()
|
| 62 |
return
|
| 63 |
|
retriever.py
CHANGED
|
@@ -43,12 +43,16 @@ def fetch_papers(query: str, limit: int = 50) -> list[dict]:
|
|
| 43 |
abstract = _reconstruct_abstract(p.get("abstract_inverted_index") or {})
|
| 44 |
if not abstract:
|
| 45 |
continue
|
|
|
|
|
|
|
|
|
|
| 46 |
papers.append({
|
| 47 |
-
"paperId":
|
| 48 |
"title": p.get("title") or "",
|
| 49 |
"abstract": abstract,
|
| 50 |
"year": p.get("publication_year"),
|
| 51 |
"citationCount": p.get("cited_by_count"),
|
|
|
|
| 52 |
"authors": [
|
| 53 |
{"name": a["author"]["display_name"]}
|
| 54 |
for a in p.get("authorships", [])
|
|
|
|
| 43 |
abstract = _reconstruct_abstract(p.get("abstract_inverted_index") or {})
|
| 44 |
if not abstract:
|
| 45 |
continue
|
| 46 |
+
doi = p.get("doi") or ""
|
| 47 |
+
openalex_id = p.get("id") or ""
|
| 48 |
+
url = doi if doi else openalex_id
|
| 49 |
papers.append({
|
| 50 |
+
"paperId": openalex_id,
|
| 51 |
"title": p.get("title") or "",
|
| 52 |
"abstract": abstract,
|
| 53 |
"year": p.get("publication_year"),
|
| 54 |
"citationCount": p.get("cited_by_count"),
|
| 55 |
+
"url": url,
|
| 56 |
"authors": [
|
| 57 |
{"name": a["author"]["display_name"]}
|
| 58 |
for a in p.get("authorships", [])
|