Eylon Caplan commited on
Commit ·
5a29ef3
1
Parent(s): 00c87be
mount fix1
Browse files
app.py
CHANGED
|
@@ -12,24 +12,47 @@ from core_logic import (
|
|
| 12 |
warmup_bm25
|
| 13 |
)
|
| 14 |
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 16 |
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 19 |
|
| 20 |
try:
|
| 21 |
if os.path.exists(TH3_DATA_PATH):
|
| 22 |
PSLP_DF = pd.read_json(TH3_DATA_PATH, lines=True)
|
| 23 |
else:
|
| 24 |
PSLP_DF = pd.DataFrame()
|
| 25 |
-
except Exception:
|
|
|
|
| 26 |
PSLP_DF = pd.DataFrame()
|
| 27 |
|
| 28 |
CREATIVE_LEXICON_PATH = "/homes/ecaplan/subspace/keyword_similarities.jsonl"
|
| 29 |
try:
|
| 30 |
# Load this larger dataset strictly for the "Auto-fill Creative Lexicon" button
|
| 31 |
CREATIVE_LEXICON_DF = pd.read_json(CREATIVE_LEXICON_PATH, lines=True)
|
| 32 |
-
except Exception:
|
|
|
|
| 33 |
CREATIVE_LEXICON_DF = PSLP_DF
|
| 34 |
|
| 35 |
|
|
@@ -51,21 +74,28 @@ def get_topics_for_demos(target, contrast):
|
|
| 51 |
dirs = [d for d in os.listdir(INDEX_DIR) if os.path.isdir(os.path.join(INDEX_DIR, d))]
|
| 52 |
import re
|
| 53 |
choices = []
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
pattern2 = f"_{contrast}-{target}"
|
| 57 |
|
| 58 |
for d in dirs:
|
| 59 |
if re.search(r"_[0-9]+-[0-9]+$", d):
|
| 60 |
continue
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 69 |
|
| 70 |
choices.sort(key=lambda x: x[0])
|
| 71 |
return choices if choices else [("No indices found for this pair", "")]
|
|
|
|
| 12 |
warmup_bm25
|
| 13 |
)
|
| 14 |
|
| 15 |
+
def resolve_existing_path(candidates):
|
| 16 |
+
for path in candidates:
|
| 17 |
+
if path and os.path.exists(path):
|
| 18 |
+
return path
|
| 19 |
+
return next((path for path in candidates if path), "")
|
| 20 |
|
| 21 |
+
|
| 22 |
+
INDEX_DIR = resolve_existing_path([
|
| 23 |
+
os.getenv('BM25_INDEX_DIR'),
|
| 24 |
+
'/data/bm25_indexes',
|
| 25 |
+
'/data/bm25_indexes/gt_reranking_sets',
|
| 26 |
+
'/data/ecaplan/splits-storage/bm25_indexes',
|
| 27 |
+
'/homes/ecaplan/gscratch1/bm25_indexes/gt_reranking_sets'
|
| 28 |
+
])
|
| 29 |
+
|
| 30 |
+
# Resolve PSLP data from Space bucket first, then local fallback.
|
| 31 |
+
TH3_DATA_PATH = resolve_existing_path([
|
| 32 |
+
os.getenv('PSLP_DATA_PATH'),
|
| 33 |
+
'/data/keyword_similarities_with_lifts_4.1.jsonl',
|
| 34 |
+
'/data/keyword_similarities_with_lifts_4.1_sample.jsonl',
|
| 35 |
+
'/homes/ecaplan/gscratch2/demo_dataset/keyword_similarities_with_lifts_4.1.jsonl'
|
| 36 |
+
])
|
| 37 |
+
|
| 38 |
+
print(f"Resolved INDEX_DIR: {INDEX_DIR}")
|
| 39 |
+
print(f"Resolved TH3_DATA_PATH: {TH3_DATA_PATH}")
|
| 40 |
|
| 41 |
try:
|
| 42 |
if os.path.exists(TH3_DATA_PATH):
|
| 43 |
PSLP_DF = pd.read_json(TH3_DATA_PATH, lines=True)
|
| 44 |
else:
|
| 45 |
PSLP_DF = pd.DataFrame()
|
| 46 |
+
except Exception as e:
|
| 47 |
+
print(f"Failed to load PSLP data from {TH3_DATA_PATH}: {e}")
|
| 48 |
PSLP_DF = pd.DataFrame()
|
| 49 |
|
| 50 |
CREATIVE_LEXICON_PATH = "/homes/ecaplan/subspace/keyword_similarities.jsonl"
|
| 51 |
try:
|
| 52 |
# Load this larger dataset strictly for the "Auto-fill Creative Lexicon" button
|
| 53 |
CREATIVE_LEXICON_DF = pd.read_json(CREATIVE_LEXICON_PATH, lines=True)
|
| 54 |
+
except Exception as e:
|
| 55 |
+
print(f"Failed to load creative lexicon data from {CREATIVE_LEXICON_PATH}: {e}")
|
| 56 |
CREATIVE_LEXICON_DF = PSLP_DF
|
| 57 |
|
| 58 |
|
|
|
|
| 74 |
dirs = [d for d in os.listdir(INDEX_DIR) if os.path.isdir(os.path.join(INDEX_DIR, d))]
|
| 75 |
import re
|
| 76 |
choices = []
|
| 77 |
+
|
| 78 |
+
pair_regex = re.compile(fr"(_({target}|{contrast})[-_]({target}|{contrast}))")
|
|
|
|
| 79 |
|
| 80 |
for d in dirs:
|
| 81 |
if re.search(r"_[0-9]+-[0-9]+$", d):
|
| 82 |
continue
|
| 83 |
+
|
| 84 |
+
match = pair_regex.search(d)
|
| 85 |
+
if not match:
|
| 86 |
+
continue
|
| 87 |
+
|
| 88 |
+
left_demo = match.group(2)
|
| 89 |
+
right_demo = match.group(3)
|
| 90 |
+
if left_demo == right_demo:
|
| 91 |
+
continue
|
| 92 |
+
if {left_demo, right_demo} != {target, contrast}:
|
| 93 |
+
continue
|
| 94 |
+
|
| 95 |
+
topic_part = d[:match.start()]
|
| 96 |
+
# Clean display name (hide backend index suffixes)
|
| 97 |
+
display_name = topic_part.replace('_', ' ').replace('-', '/')
|
| 98 |
+
choices.append((display_name, d))
|
| 99 |
|
| 100 |
choices.sort(key=lambda x: x[0])
|
| 101 |
return choices if choices else [("No indices found for this pair", "")]
|