raxhel commited on
Commit
97c8ca6
·
verified ·
1 Parent(s): 0c40cb3

som hf dataset

Browse files
Files changed (1) hide show
  1. app.py +12 -7
app.py CHANGED
@@ -206,26 +206,31 @@ def upload_embeddings_to_hub(local_path: str):
206
 
207
  async def get_embeddings_from_hf_dataset(filename: str) -> Optional[List[np.ndarray]]:
208
  """ (from app.py) """
209
- try:
210
- print(f"\n📦 Fetching embeddings from HF dataset for: {filename}")
211
- # 注意: 這裡使用的是 app.py 中定義的 dataset (lyi029/malvec-embeddings-new)
212
  dataset = load_dataset("lyi029/malvec-embeddings-new", split="train", streaming=True)
213
  embeddings = []
214
  count = 0
215
  for sample in dataset:
216
  sample_filename = sample.get('source', '') or sample.get('filename', '')
217
- if filename.lower() in sample_filename.lower():
 
 
 
218
  emb = sample.get('embedding') or sample.get('emb')
219
  if emb is not None:
220
  embeddings.append(np.array(emb, dtype=np.float32))
221
  count += 1
222
- if count >= 1000: break
 
223
  if embeddings:
224
- print(f"✅ Found {len(embeddings)} embeddings for {filename}")
225
  return embeddings
226
  else:
227
- print(f"⚠️ No embeddings found for {filename} in dataset")
228
  return None
 
229
  except Exception as e:
230
  print(f"❌ Error fetching from HF dataset: {e}")
231
  return None
 
206
 
207
  async def get_embeddings_from_hf_dataset(filename: str) -> Optional[List[np.ndarray]]:
208
  """ (from app.py) """
209
+ base_name = filename.rsplit('.', 1)[0].lower()
210
+ print(f"\n📦 Fetching embeddings from HF dataset for base name: {base_name}")
211
+
212
  dataset = load_dataset("lyi029/malvec-embeddings-new", split="train", streaming=True)
213
  embeddings = []
214
  count = 0
215
  for sample in dataset:
216
  sample_filename = sample.get('source', '') or sample.get('filename', '')
217
+
218
+ # ✅ 2. 檢查 dataset source (例如 "unpacked_myfile_67.txt")
219
+ # 是否 *包含* 這個基礎檔名 (例如 "myfile_67")
220
+ if base_name in sample_filename.lower():
221
  emb = sample.get('embedding') or sample.get('emb')
222
  if emb is not None:
223
  embeddings.append(np.array(emb, dtype=np.float32))
224
  count += 1
225
+ if count >= 1000: break # 保留限制
226
+
227
  if embeddings:
228
+ print(f"✅ Found {len(embeddings)} embeddings for base name {base_name}")
229
  return embeddings
230
  else:
231
+ print(f"⚠️ No embeddings found for base name {base_name} in dataset")
232
  return None
233
+
234
  except Exception as e:
235
  print(f"❌ Error fetching from HF dataset: {e}")
236
  return None