ivykopal commited on
Commit
aefbb1e
·
1 Parent(s): 2ece21e

fix: don't stub missing snapshot files with empty placeholders

Browse files

download_snapshot wrote a 0-byte placeholder for any missing file,
including index.faiss. A 0-byte file is not a valid FAISS index and
crashes faiss.read_index instead of being treated as absent, breaking
the very first sync against a brand-new dataset repo. Missing files
are now left absent so load_index's existing FileNotFoundError
fallback (and sqlite3.connect's create-on-open behavior) apply as
intended.

sync/hf_dataset_store.py CHANGED
@@ -16,7 +16,6 @@ def download_snapshot(repo_id: str, local_dir: str, token: str) -> dict:
16
  try:
17
  hf_hub_download(repo_id=repo_id, repo_type="dataset", filename=filename, local_dir=local_dir, token=token)
18
  except (FileNotFoundError, EntryNotFoundError):
19
- open(os.path.join(local_dir, filename), "wb").close()
20
  continue
21
  if filename == "state.json":
22
  with open(os.path.join(local_dir, filename)) as f:
 
16
  try:
17
  hf_hub_download(repo_id=repo_id, repo_type="dataset", filename=filename, local_dir=local_dir, token=token)
18
  except (FileNotFoundError, EntryNotFoundError):
 
19
  continue
20
  if filename == "state.json":
21
  with open(os.path.join(local_dir, filename)) as f:
tests/sync/test_hf_dataset_store.py CHANGED
@@ -1,7 +1,10 @@
1
  import json
2
  import os
3
  import httpx
 
4
  from huggingface_hub.errors import RemoteEntryNotFoundError
 
 
5
  from sync.hf_dataset_store import download_snapshot, upload_snapshot
6
 
7
 
@@ -37,19 +40,28 @@ def test_download_snapshot_missing_state_returns_empty_state(tmp_path, monkeypat
37
  monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download)
38
  state = download_snapshot("org/repo", str(tmp_path), token="tok")
39
  assert state == {"papers": {}, "last_synced_at": None}
 
40
 
41
  def test_download_snapshot_handles_empty_remote_dataset_repo(tmp_path, monkeypatch):
42
  """Reproduces a real first-time sync against a brand-new, empty dataset repo:
43
  huggingface_hub raises RemoteEntryNotFoundError (an HTTP 404), not a plain
44
- FileNotFoundError, when a file doesn't exist yet on the Hub."""
 
 
45
  def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token):
46
  raise _remote_entry_not_found()
47
 
48
  monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download)
49
  state = download_snapshot("org/repo", str(tmp_path), token="tok")
50
  assert state == {"papers": {}, "last_synced_at": None}
51
- assert os.path.exists(tmp_path / "index.faiss")
52
- assert os.path.exists(tmp_path / "papers.db")
 
 
 
 
 
 
53
 
54
  def test_upload_snapshot_uploads_folder_as_single_atomic_commit(tmp_path, monkeypatch):
55
  for name in ["index.faiss", "papers.db", "state.json"]:
 
1
  import json
2
  import os
3
  import httpx
4
+ import pytest
5
  from huggingface_hub.errors import RemoteEntryNotFoundError
6
+ from common.vector_index import load_index
7
+ from common.db import init_db
8
  from sync.hf_dataset_store import download_snapshot, upload_snapshot
9
 
10
 
 
40
  monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download)
41
  state = download_snapshot("org/repo", str(tmp_path), token="tok")
42
  assert state == {"papers": {}, "last_synced_at": None}
43
+ assert not os.path.exists(tmp_path / "state.json")
44
 
45
  def test_download_snapshot_handles_empty_remote_dataset_repo(tmp_path, monkeypatch):
46
  """Reproduces a real first-time sync against a brand-new, empty dataset repo:
47
  huggingface_hub raises RemoteEntryNotFoundError (an HTTP 404), not a plain
48
+ FileNotFoundError, when a file doesn't exist yet on the Hub. Missing files
49
+ must be left absent (not stubbed with an empty placeholder), since a 0-byte
50
+ file is not a valid "missing" signal to faiss.read_index / sqlite3.connect."""
51
  def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token):
52
  raise _remote_entry_not_found()
53
 
54
  monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download)
55
  state = download_snapshot("org/repo", str(tmp_path), token="tok")
56
  assert state == {"papers": {}, "last_synced_at": None}
57
+ assert not os.path.exists(tmp_path / "index.faiss")
58
+ assert not os.path.exists(tmp_path / "papers.db")
59
+
60
+ # Downstream loaders must fall back cleanly, exactly as run_sync relies on.
61
+ with pytest.raises(FileNotFoundError):
62
+ load_index(str(tmp_path / "index.faiss"))
63
+ conn = init_db(str(tmp_path / "papers.db"))
64
+ assert conn.execute("SELECT COUNT(*) FROM papers").fetchone()[0] == 0
65
 
66
  def test_upload_snapshot_uploads_folder_as_single_atomic_commit(tmp_path, monkeypatch):
67
  for name in ["index.faiss", "papers.db", "state.json"]: