Spaces:
Sleeping
Sleeping
RemanenetSpy commited on
Commit ·
cdb2a83
1
Parent(s): fd13692
fix(kaggle): update dataset path to match user's custom kaggle dataset output
Browse files
benchmark/smriti_longmemeval_kaggle.ipynb
CHANGED
|
@@ -32,7 +32,7 @@
|
|
| 32 |
"id": "title",
|
| 33 |
"metadata": {},
|
| 34 |
"source": [
|
| 35 |
-
"# Smriti
|
| 36 |
"\n",
|
| 37 |
"**What this notebook does:**\n",
|
| 38 |
"- Evaluates Smriti's pure-code memory algorithm (Gap=0.08, MaxCutoff=0.52) against the official LongMemEval benchmark (ICLR 2025)\n",
|
|
@@ -45,8 +45,8 @@
|
|
| 45 |
"**Algorithm:** 100% pure code, zero LLM calls at query time.\n",
|
| 46 |
"```\n",
|
| 47 |
"Vector Search (threshold=0.85)\n",
|
| 48 |
-
"
|
| 49 |
-
"
|
| 50 |
"```"
|
| 51 |
]
|
| 52 |
},
|
|
@@ -57,7 +57,7 @@
|
|
| 57 |
"metadata": {},
|
| 58 |
"outputs": [],
|
| 59 |
"source": [
|
| 60 |
-
"#
|
| 61 |
"import subprocess, sys\n",
|
| 62 |
"\n",
|
| 63 |
"pkgs = [\n",
|
|
@@ -77,11 +77,11 @@
|
|
| 77 |
"metadata": {},
|
| 78 |
"outputs": [],
|
| 79 |
"source": [
|
| 80 |
-
"#
|
| 81 |
"import os\n",
|
| 82 |
"\n",
|
| 83 |
-
"# Dataset paths
|
| 84 |
-
"DATA_DIR = '/kaggle/input/longmemeval'\n",
|
| 85 |
"S_SPLIT_PATH = os.path.join(DATA_DIR, 'longmemeval_s_cleaned.json')\n",
|
| 86 |
"M_SPLIT_PATH = os.path.join(DATA_DIR, 'longmemeval_m_cleaned.json')\n",
|
| 87 |
"OUTPUT_DIR = '/kaggle/working'\n",
|
|
@@ -104,7 +104,7 @@
|
|
| 104 |
"metadata": {},
|
| 105 |
"outputs": [],
|
| 106 |
"source": [
|
| 107 |
-
"#
|
| 108 |
"import uuid\n",
|
| 109 |
"from dataclasses import dataclass, field\n",
|
| 110 |
"from typing import List, Optional, Dict, Any\n",
|
|
@@ -176,7 +176,7 @@
|
|
| 176 |
"metadata": {},
|
| 177 |
"outputs": [],
|
| 178 |
"source": [
|
| 179 |
-
"#
|
| 180 |
"import re\n",
|
| 181 |
"import threading\n",
|
| 182 |
"import time\n",
|
|
@@ -306,7 +306,7 @@
|
|
| 306 |
"metadata": {},
|
| 307 |
"outputs": [],
|
| 308 |
"source": [
|
| 309 |
-
"#
|
| 310 |
"import json\n",
|
| 311 |
"import math\n",
|
| 312 |
"import time as time_mod\n",
|
|
@@ -333,7 +333,7 @@
|
|
| 333 |
"\n",
|
| 334 |
"def run_eval(cases, label='S'):\n",
|
| 335 |
" print(f'\\n{\"=\"*60}')\n",
|
| 336 |
-
" print(f' Running LongMemEval-{label}
|
| 337 |
" print(f'{\"=\"*60}')\n",
|
| 338 |
"\n",
|
| 339 |
" results = []\n",
|
|
@@ -408,7 +408,7 @@
|
|
| 408 |
" running_f1 = sum(r['f1'] for r in results) / len(results)\n",
|
| 409 |
" print(f' [{i+1:4d}/{len(cases)}] Sub={running_sub:.3f} F1={running_f1:.3f} lat={lat_ms:.0f}ms')\n",
|
| 410 |
"\n",
|
| 411 |
-
" #
|
| 412 |
" n = len(results)\n",
|
| 413 |
" total_sub = sum(r['substring_match'] for r in results) / n\n",
|
| 414 |
" total_f1 = sum(r['f1'] for r in results) / n\n",
|
|
@@ -461,7 +461,7 @@
|
|
| 461 |
"metadata": {},
|
| 462 |
"outputs": [],
|
| 463 |
"source": [
|
| 464 |
-
"#
|
| 465 |
"def load_split(path):\n",
|
| 466 |
" with open(path, 'r', encoding='utf-8') as f:\n",
|
| 467 |
" data = json.load(f)\n",
|
|
@@ -479,7 +479,7 @@
|
|
| 479 |
"metadata": {},
|
| 480 |
"outputs": [],
|
| 481 |
"source": [
|
| 482 |
-
"#
|
| 483 |
"summary_s = run_eval(cases_s, label='S')\n",
|
| 484 |
"\n",
|
| 485 |
"# Save results\n",
|
|
@@ -501,7 +501,7 @@
|
|
| 501 |
"metadata": {},
|
| 502 |
"outputs": [],
|
| 503 |
"source": [
|
| 504 |
-
"#
|
| 505 |
"# NOTE: This is the massive split (500 sessions per question, ~1.5M tokens each)\n",
|
| 506 |
"# It will take 2-4 hours. Kaggle allows up to 12 hours.\n",
|
| 507 |
"if cases_m:\n",
|
|
@@ -517,7 +517,7 @@
|
|
| 517 |
"\n",
|
| 518 |
" print('M-split done and saved.')\n",
|
| 519 |
"else:\n",
|
| 520 |
-
" print('M-split file not found
|
| 521 |
" summary_m = None"
|
| 522 |
]
|
| 523 |
},
|
|
@@ -528,7 +528,7 @@
|
|
| 528 |
"metadata": {},
|
| 529 |
"outputs": [],
|
| 530 |
"source": [
|
| 531 |
-
"#
|
| 532 |
"from datetime import datetime\n",
|
| 533 |
"\n",
|
| 534 |
"def make_benchmarks_md(summary_s, summary_m=None):\n",
|
|
@@ -545,7 +545,7 @@
|
|
| 545 |
" '## LongMemEval (ICLR 2025)',\n",
|
| 546 |
" '',\n",
|
| 547 |
" '> **Dataset:** `xiaowu0162/longmemeval-cleaned` ',\n",
|
| 548 |
-
" '> **Metric:** Substring Match (Recall)
|
| 549 |
" '',\n",
|
| 550 |
" ]\n",
|
| 551 |
"\n",
|
|
@@ -601,4 +601,4 @@
|
|
| 601 |
]
|
| 602 |
}
|
| 603 |
]
|
| 604 |
-
}
|
|
|
|
| 32 |
"id": "title",
|
| 33 |
"metadata": {},
|
| 34 |
"source": [
|
| 35 |
+
"# Smriti \u00d7 LongMemEval \u2014 Official Benchmark Run\n",
|
| 36 |
"\n",
|
| 37 |
"**What this notebook does:**\n",
|
| 38 |
"- Evaluates Smriti's pure-code memory algorithm (Gap=0.08, MaxCutoff=0.52) against the official LongMemEval benchmark (ICLR 2025)\n",
|
|
|
|
| 45 |
"**Algorithm:** 100% pure code, zero LLM calls at query time.\n",
|
| 46 |
"```\n",
|
| 47 |
"Vector Search (threshold=0.85)\n",
|
| 48 |
+
" \u2192 Entity/SVO Filter (zero-overlap candidates dropped)\n",
|
| 49 |
+
" \u2192 Bayesian Gap Cutoff (gap=0.08, max_cutoff=0.52)\n",
|
| 50 |
"```"
|
| 51 |
]
|
| 52 |
},
|
|
|
|
| 57 |
"metadata": {},
|
| 58 |
"outputs": [],
|
| 59 |
"source": [
|
| 60 |
+
"# \u2500\u2500 Install dependencies \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 61 |
"import subprocess, sys\n",
|
| 62 |
"\n",
|
| 63 |
"pkgs = [\n",
|
|
|
|
| 77 |
"metadata": {},
|
| 78 |
"outputs": [],
|
| 79 |
"source": [
|
| 80 |
+
"# \u2500\u2500 Configuration \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 81 |
"import os\n",
|
| 82 |
"\n",
|
| 83 |
+
"# Dataset paths \u2014 update if your Kaggle dataset has a different name\n",
|
| 84 |
+
" DATA_DIR = '/kaggle/input/longmemeval-clean/data/longmemeval'\\n",
|
| 85 |
"S_SPLIT_PATH = os.path.join(DATA_DIR, 'longmemeval_s_cleaned.json')\n",
|
| 86 |
"M_SPLIT_PATH = os.path.join(DATA_DIR, 'longmemeval_m_cleaned.json')\n",
|
| 87 |
"OUTPUT_DIR = '/kaggle/working'\n",
|
|
|
|
| 104 |
"metadata": {},
|
| 105 |
"outputs": [],
|
| 106 |
"source": [
|
| 107 |
+
"# \u2500\u2500 In-memory stores (self-contained, no smriti_core import needed) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 108 |
"import uuid\n",
|
| 109 |
"from dataclasses import dataclass, field\n",
|
| 110 |
"from typing import List, Optional, Dict, Any\n",
|
|
|
|
| 176 |
"metadata": {},
|
| 177 |
"outputs": [],
|
| 178 |
"source": [
|
| 179 |
+
"# \u2500\u2500 Smriti Mock Server (pure-code algorithm) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 180 |
"import re\n",
|
| 181 |
"import threading\n",
|
| 182 |
"import time\n",
|
|
|
|
| 306 |
"metadata": {},
|
| 307 |
"outputs": [],
|
| 308 |
"source": [
|
| 309 |
+
"# \u2500\u2500 Evaluation Engine \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 310 |
"import json\n",
|
| 311 |
"import math\n",
|
| 312 |
"import time as time_mod\n",
|
|
|
|
| 333 |
"\n",
|
| 334 |
"def run_eval(cases, label='S'):\n",
|
| 335 |
" print(f'\\n{\"=\"*60}')\n",
|
| 336 |
+
" print(f' Running LongMemEval-{label} \u2014 {len(cases)} cases')\n",
|
| 337 |
" print(f'{\"=\"*60}')\n",
|
| 338 |
"\n",
|
| 339 |
" results = []\n",
|
|
|
|
| 408 |
" running_f1 = sum(r['f1'] for r in results) / len(results)\n",
|
| 409 |
" print(f' [{i+1:4d}/{len(cases)}] Sub={running_sub:.3f} F1={running_f1:.3f} lat={lat_ms:.0f}ms')\n",
|
| 410 |
"\n",
|
| 411 |
+
" # \u2500\u2500 Summary \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 412 |
" n = len(results)\n",
|
| 413 |
" total_sub = sum(r['substring_match'] for r in results) / n\n",
|
| 414 |
" total_f1 = sum(r['f1'] for r in results) / n\n",
|
|
|
|
| 461 |
"metadata": {},
|
| 462 |
"outputs": [],
|
| 463 |
"source": [
|
| 464 |
+
"# \u2500\u2500 Load datasets \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 465 |
"def load_split(path):\n",
|
| 466 |
" with open(path, 'r', encoding='utf-8') as f:\n",
|
| 467 |
" data = json.load(f)\n",
|
|
|
|
| 479 |
"metadata": {},
|
| 480 |
"outputs": [],
|
| 481 |
"source": [
|
| 482 |
+
"# \u2500\u2500 Run LongMemEval-S (500 cases, ~30 mins on GPU) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 483 |
"summary_s = run_eval(cases_s, label='S')\n",
|
| 484 |
"\n",
|
| 485 |
"# Save results\n",
|
|
|
|
| 501 |
"metadata": {},
|
| 502 |
"outputs": [],
|
| 503 |
"source": [
|
| 504 |
+
"# \u2500\u2500 Run LongMemEval-M (500 cases, ~3 hrs on GPU) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 505 |
"# NOTE: This is the massive split (500 sessions per question, ~1.5M tokens each)\n",
|
| 506 |
"# It will take 2-4 hours. Kaggle allows up to 12 hours.\n",
|
| 507 |
"if cases_m:\n",
|
|
|
|
| 517 |
"\n",
|
| 518 |
" print('M-split done and saved.')\n",
|
| 519 |
"else:\n",
|
| 520 |
+
" print('M-split file not found \u2014 skipping.')\n",
|
| 521 |
" summary_m = None"
|
| 522 |
]
|
| 523 |
},
|
|
|
|
| 528 |
"metadata": {},
|
| 529 |
"outputs": [],
|
| 530 |
"source": [
|
| 531 |
+
"# \u2500\u2500 Generate BENCHMARKS.md \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
|
| 532 |
"from datetime import datetime\n",
|
| 533 |
"\n",
|
| 534 |
"def make_benchmarks_md(summary_s, summary_m=None):\n",
|
|
|
|
| 545 |
" '## LongMemEval (ICLR 2025)',\n",
|
| 546 |
" '',\n",
|
| 547 |
" '> **Dataset:** `xiaowu0162/longmemeval-cleaned` ',\n",
|
| 548 |
+
" '> **Metric:** Substring Match (Recall) \u2014 does the retrieved context contain the gold answer?',\n",
|
| 549 |
" '',\n",
|
| 550 |
" ]\n",
|
| 551 |
"\n",
|
|
|
|
| 601 |
]
|
| 602 |
}
|
| 603 |
]
|
| 604 |
+
}
|