som hf dataset
Browse files
app.py
CHANGED
|
@@ -206,26 +206,31 @@ def upload_embeddings_to_hub(local_path: str):
|
|
| 206 |
|
| 207 |
async def get_embeddings_from_hf_dataset(filename: str) -> Optional[List[np.ndarray]]:
|
| 208 |
""" (from app.py) """
|
| 209 |
-
|
| 210 |
-
print(f"\n📦 Fetching embeddings from HF dataset for: {
|
| 211 |
-
|
| 212 |
dataset = load_dataset("lyi029/malvec-embeddings-new", split="train", streaming=True)
|
| 213 |
embeddings = []
|
| 214 |
count = 0
|
| 215 |
for sample in dataset:
|
| 216 |
sample_filename = sample.get('source', '') or sample.get('filename', '')
|
| 217 |
-
|
|
|
|
|
|
|
|
|
|
| 218 |
emb = sample.get('embedding') or sample.get('emb')
|
| 219 |
if emb is not None:
|
| 220 |
embeddings.append(np.array(emb, dtype=np.float32))
|
| 221 |
count += 1
|
| 222 |
-
if count >= 1000: break
|
|
|
|
| 223 |
if embeddings:
|
| 224 |
-
print(f"✅ Found {len(embeddings)} embeddings for {
|
| 225 |
return embeddings
|
| 226 |
else:
|
| 227 |
-
print(f"⚠️ No embeddings found for {
|
| 228 |
return None
|
|
|
|
| 229 |
except Exception as e:
|
| 230 |
print(f"❌ Error fetching from HF dataset: {e}")
|
| 231 |
return None
|
|
|
|
| 206 |
|
| 207 |
async def get_embeddings_from_hf_dataset(filename: str) -> Optional[List[np.ndarray]]:
|
| 208 |
""" (from app.py) """
|
| 209 |
+
base_name = filename.rsplit('.', 1)[0].lower()
|
| 210 |
+
print(f"\n📦 Fetching embeddings from HF dataset for base name: {base_name}")
|
| 211 |
+
|
| 212 |
dataset = load_dataset("lyi029/malvec-embeddings-new", split="train", streaming=True)
|
| 213 |
embeddings = []
|
| 214 |
count = 0
|
| 215 |
for sample in dataset:
|
| 216 |
sample_filename = sample.get('source', '') or sample.get('filename', '')
|
| 217 |
+
|
| 218 |
+
# ✅ 2. 檢查 dataset source (例如 "unpacked_myfile_67.txt")
|
| 219 |
+
# 是否 *包含* 這個基礎檔名 (例如 "myfile_67")
|
| 220 |
+
if base_name in sample_filename.lower():
|
| 221 |
emb = sample.get('embedding') or sample.get('emb')
|
| 222 |
if emb is not None:
|
| 223 |
embeddings.append(np.array(emb, dtype=np.float32))
|
| 224 |
count += 1
|
| 225 |
+
if count >= 1000: break # 保留限制
|
| 226 |
+
|
| 227 |
if embeddings:
|
| 228 |
+
print(f"✅ Found {len(embeddings)} embeddings for base name {base_name}")
|
| 229 |
return embeddings
|
| 230 |
else:
|
| 231 |
+
print(f"⚠️ No embeddings found for base name {base_name} in dataset")
|
| 232 |
return None
|
| 233 |
+
|
| 234 |
except Exception as e:
|
| 235 |
print(f"❌ Error fetching from HF dataset: {e}")
|
| 236 |
return None
|