RemanenetSpy commited on
Commit
cdb2a83
·
1 Parent(s): fd13692

fix(kaggle): update dataset path to match user's custom kaggle dataset output

Browse files
benchmark/smriti_longmemeval_kaggle.ipynb CHANGED
@@ -32,7 +32,7 @@
32
  "id": "title",
33
  "metadata": {},
34
  "source": [
35
- "# Smriti × LongMemEval — Official Benchmark Run\n",
36
  "\n",
37
  "**What this notebook does:**\n",
38
  "- Evaluates Smriti's pure-code memory algorithm (Gap=0.08, MaxCutoff=0.52) against the official LongMemEval benchmark (ICLR 2025)\n",
@@ -45,8 +45,8 @@
45
  "**Algorithm:** 100% pure code, zero LLM calls at query time.\n",
46
  "```\n",
47
  "Vector Search (threshold=0.85)\n",
48
- " → Entity/SVO Filter (zero-overlap candidates dropped)\n",
49
- " → Bayesian Gap Cutoff (gap=0.08, max_cutoff=0.52)\n",
50
  "```"
51
  ]
52
  },
@@ -57,7 +57,7 @@
57
  "metadata": {},
58
  "outputs": [],
59
  "source": [
60
- "# ── Install dependencies ──────────────────────────────────────────────────────\n",
61
  "import subprocess, sys\n",
62
  "\n",
63
  "pkgs = [\n",
@@ -77,11 +77,11 @@
77
  "metadata": {},
78
  "outputs": [],
79
  "source": [
80
- "# ── Configuration ─────────────────────────────────────────────────────────────\n",
81
  "import os\n",
82
  "\n",
83
- "# Dataset paths — update if your Kaggle dataset has a different name\n",
84
- "DATA_DIR = '/kaggle/input/longmemeval'\n",
85
  "S_SPLIT_PATH = os.path.join(DATA_DIR, 'longmemeval_s_cleaned.json')\n",
86
  "M_SPLIT_PATH = os.path.join(DATA_DIR, 'longmemeval_m_cleaned.json')\n",
87
  "OUTPUT_DIR = '/kaggle/working'\n",
@@ -104,7 +104,7 @@
104
  "metadata": {},
105
  "outputs": [],
106
  "source": [
107
- "# ── In-memory stores (self-contained, no smriti_core import needed) ───────────\n",
108
  "import uuid\n",
109
  "from dataclasses import dataclass, field\n",
110
  "from typing import List, Optional, Dict, Any\n",
@@ -176,7 +176,7 @@
176
  "metadata": {},
177
  "outputs": [],
178
  "source": [
179
- "# ── Smriti Mock Server (pure-code algorithm) ──────────────────────────────────\n",
180
  "import re\n",
181
  "import threading\n",
182
  "import time\n",
@@ -306,7 +306,7 @@
306
  "metadata": {},
307
  "outputs": [],
308
  "source": [
309
- "# ── Evaluation Engine ─────────────────────────────────────────────────────────\n",
310
  "import json\n",
311
  "import math\n",
312
  "import time as time_mod\n",
@@ -333,7 +333,7 @@
333
  "\n",
334
  "def run_eval(cases, label='S'):\n",
335
  " print(f'\\n{\"=\"*60}')\n",
336
- " print(f' Running LongMemEval-{label} — {len(cases)} cases')\n",
337
  " print(f'{\"=\"*60}')\n",
338
  "\n",
339
  " results = []\n",
@@ -408,7 +408,7 @@
408
  " running_f1 = sum(r['f1'] for r in results) / len(results)\n",
409
  " print(f' [{i+1:4d}/{len(cases)}] Sub={running_sub:.3f} F1={running_f1:.3f} lat={lat_ms:.0f}ms')\n",
410
  "\n",
411
- " # ── Summary ────────────────────────────────────────────────────────────────\n",
412
  " n = len(results)\n",
413
  " total_sub = sum(r['substring_match'] for r in results) / n\n",
414
  " total_f1 = sum(r['f1'] for r in results) / n\n",
@@ -461,7 +461,7 @@
461
  "metadata": {},
462
  "outputs": [],
463
  "source": [
464
- "# ── Load datasets ─────────────────────────────────────────────────────────────\n",
465
  "def load_split(path):\n",
466
  " with open(path, 'r', encoding='utf-8') as f:\n",
467
  " data = json.load(f)\n",
@@ -479,7 +479,7 @@
479
  "metadata": {},
480
  "outputs": [],
481
  "source": [
482
- "# ── Run LongMemEval-S (500 cases, ~30 mins on GPU) ───────────────────────────\n",
483
  "summary_s = run_eval(cases_s, label='S')\n",
484
  "\n",
485
  "# Save results\n",
@@ -501,7 +501,7 @@
501
  "metadata": {},
502
  "outputs": [],
503
  "source": [
504
- "# ── Run LongMemEval-M (500 cases, ~3 hrs on GPU) ─────────────────────────────\n",
505
  "# NOTE: This is the massive split (500 sessions per question, ~1.5M tokens each)\n",
506
  "# It will take 2-4 hours. Kaggle allows up to 12 hours.\n",
507
  "if cases_m:\n",
@@ -517,7 +517,7 @@
517
  "\n",
518
  " print('M-split done and saved.')\n",
519
  "else:\n",
520
- " print('M-split file not found — skipping.')\n",
521
  " summary_m = None"
522
  ]
523
  },
@@ -528,7 +528,7 @@
528
  "metadata": {},
529
  "outputs": [],
530
  "source": [
531
- "# ── Generate BENCHMARKS.md ────────────────────────────────────────────────────\n",
532
  "from datetime import datetime\n",
533
  "\n",
534
  "def make_benchmarks_md(summary_s, summary_m=None):\n",
@@ -545,7 +545,7 @@
545
  " '## LongMemEval (ICLR 2025)',\n",
546
  " '',\n",
547
  " '> **Dataset:** `xiaowu0162/longmemeval-cleaned` ',\n",
548
- " '> **Metric:** Substring Match (Recall) — does the retrieved context contain the gold answer?',\n",
549
  " '',\n",
550
  " ]\n",
551
  "\n",
@@ -601,4 +601,4 @@
601
  ]
602
  }
603
  ]
604
- }
 
32
  "id": "title",
33
  "metadata": {},
34
  "source": [
35
+ "# Smriti \u00d7 LongMemEval \u2014 Official Benchmark Run\n",
36
  "\n",
37
  "**What this notebook does:**\n",
38
  "- Evaluates Smriti's pure-code memory algorithm (Gap=0.08, MaxCutoff=0.52) against the official LongMemEval benchmark (ICLR 2025)\n",
 
45
  "**Algorithm:** 100% pure code, zero LLM calls at query time.\n",
46
  "```\n",
47
  "Vector Search (threshold=0.85)\n",
48
+ " \u2192 Entity/SVO Filter (zero-overlap candidates dropped)\n",
49
+ " \u2192 Bayesian Gap Cutoff (gap=0.08, max_cutoff=0.52)\n",
50
  "```"
51
  ]
52
  },
 
57
  "metadata": {},
58
  "outputs": [],
59
  "source": [
60
+ "# \u2500\u2500 Install dependencies \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
61
  "import subprocess, sys\n",
62
  "\n",
63
  "pkgs = [\n",
 
77
  "metadata": {},
78
  "outputs": [],
79
  "source": [
80
+ "# \u2500\u2500 Configuration \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
81
  "import os\n",
82
  "\n",
83
+ "# Dataset paths \u2014 update if your Kaggle dataset has a different name\n",
84
+ " DATA_DIR = '/kaggle/input/longmemeval-clean/data/longmemeval'\\n",
85
  "S_SPLIT_PATH = os.path.join(DATA_DIR, 'longmemeval_s_cleaned.json')\n",
86
  "M_SPLIT_PATH = os.path.join(DATA_DIR, 'longmemeval_m_cleaned.json')\n",
87
  "OUTPUT_DIR = '/kaggle/working'\n",
 
104
  "metadata": {},
105
  "outputs": [],
106
  "source": [
107
+ "# \u2500\u2500 In-memory stores (self-contained, no smriti_core import needed) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
108
  "import uuid\n",
109
  "from dataclasses import dataclass, field\n",
110
  "from typing import List, Optional, Dict, Any\n",
 
176
  "metadata": {},
177
  "outputs": [],
178
  "source": [
179
+ "# \u2500\u2500 Smriti Mock Server (pure-code algorithm) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
180
  "import re\n",
181
  "import threading\n",
182
  "import time\n",
 
306
  "metadata": {},
307
  "outputs": [],
308
  "source": [
309
+ "# \u2500\u2500 Evaluation Engine \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
310
  "import json\n",
311
  "import math\n",
312
  "import time as time_mod\n",
 
333
  "\n",
334
  "def run_eval(cases, label='S'):\n",
335
  " print(f'\\n{\"=\"*60}')\n",
336
+ " print(f' Running LongMemEval-{label} \u2014 {len(cases)} cases')\n",
337
  " print(f'{\"=\"*60}')\n",
338
  "\n",
339
  " results = []\n",
 
408
  " running_f1 = sum(r['f1'] for r in results) / len(results)\n",
409
  " print(f' [{i+1:4d}/{len(cases)}] Sub={running_sub:.3f} F1={running_f1:.3f} lat={lat_ms:.0f}ms')\n",
410
  "\n",
411
+ " # \u2500\u2500 Summary \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
412
  " n = len(results)\n",
413
  " total_sub = sum(r['substring_match'] for r in results) / n\n",
414
  " total_f1 = sum(r['f1'] for r in results) / n\n",
 
461
  "metadata": {},
462
  "outputs": [],
463
  "source": [
464
+ "# \u2500\u2500 Load datasets \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
465
  "def load_split(path):\n",
466
  " with open(path, 'r', encoding='utf-8') as f:\n",
467
  " data = json.load(f)\n",
 
479
  "metadata": {},
480
  "outputs": [],
481
  "source": [
482
+ "# \u2500\u2500 Run LongMemEval-S (500 cases, ~30 mins on GPU) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
483
  "summary_s = run_eval(cases_s, label='S')\n",
484
  "\n",
485
  "# Save results\n",
 
501
  "metadata": {},
502
  "outputs": [],
503
  "source": [
504
+ "# \u2500\u2500 Run LongMemEval-M (500 cases, ~3 hrs on GPU) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
505
  "# NOTE: This is the massive split (500 sessions per question, ~1.5M tokens each)\n",
506
  "# It will take 2-4 hours. Kaggle allows up to 12 hours.\n",
507
  "if cases_m:\n",
 
517
  "\n",
518
  " print('M-split done and saved.')\n",
519
  "else:\n",
520
+ " print('M-split file not found \u2014 skipping.')\n",
521
  " summary_m = None"
522
  ]
523
  },
 
528
  "metadata": {},
529
  "outputs": [],
530
  "source": [
531
+ "# \u2500\u2500 Generate BENCHMARKS.md \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\n",
532
  "from datetime import datetime\n",
533
  "\n",
534
  "def make_benchmarks_md(summary_s, summary_m=None):\n",
 
545
  " '## LongMemEval (ICLR 2025)',\n",
546
  " '',\n",
547
  " '> **Dataset:** `xiaowu0162/longmemeval-cleaned` ',\n",
548
+ " '> **Metric:** Substring Match (Recall) \u2014 does the retrieved context contain the gold answer?',\n",
549
  " '',\n",
550
  " ]\n",
551
  "\n",
 
601
  ]
602
  }
603
  ]
604
+ }