fix: don't stub missing snapshot files with empty placeholders
Browse filesdownload_snapshot wrote a 0-byte placeholder for any missing file,
including index.faiss. A 0-byte file is not a valid FAISS index and
crashes faiss.read_index instead of being treated as absent, breaking
the very first sync against a brand-new dataset repo. Missing files
are now left absent so load_index's existing FileNotFoundError
fallback (and sqlite3.connect's create-on-open behavior) apply as
intended.
sync/hf_dataset_store.py
CHANGED
|
@@ -16,7 +16,6 @@ def download_snapshot(repo_id: str, local_dir: str, token: str) -> dict:
|
|
| 16 |
try:
|
| 17 |
hf_hub_download(repo_id=repo_id, repo_type="dataset", filename=filename, local_dir=local_dir, token=token)
|
| 18 |
except (FileNotFoundError, EntryNotFoundError):
|
| 19 |
-
open(os.path.join(local_dir, filename), "wb").close()
|
| 20 |
continue
|
| 21 |
if filename == "state.json":
|
| 22 |
with open(os.path.join(local_dir, filename)) as f:
|
|
|
|
| 16 |
try:
|
| 17 |
hf_hub_download(repo_id=repo_id, repo_type="dataset", filename=filename, local_dir=local_dir, token=token)
|
| 18 |
except (FileNotFoundError, EntryNotFoundError):
|
|
|
|
| 19 |
continue
|
| 20 |
if filename == "state.json":
|
| 21 |
with open(os.path.join(local_dir, filename)) as f:
|
tests/sync/test_hf_dataset_store.py
CHANGED
|
@@ -1,7 +1,10 @@
|
|
| 1 |
import json
|
| 2 |
import os
|
| 3 |
import httpx
|
|
|
|
| 4 |
from huggingface_hub.errors import RemoteEntryNotFoundError
|
|
|
|
|
|
|
| 5 |
from sync.hf_dataset_store import download_snapshot, upload_snapshot
|
| 6 |
|
| 7 |
|
|
@@ -37,19 +40,28 @@ def test_download_snapshot_missing_state_returns_empty_state(tmp_path, monkeypat
|
|
| 37 |
monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download)
|
| 38 |
state = download_snapshot("org/repo", str(tmp_path), token="tok")
|
| 39 |
assert state == {"papers": {}, "last_synced_at": None}
|
|
|
|
| 40 |
|
| 41 |
def test_download_snapshot_handles_empty_remote_dataset_repo(tmp_path, monkeypatch):
|
| 42 |
"""Reproduces a real first-time sync against a brand-new, empty dataset repo:
|
| 43 |
huggingface_hub raises RemoteEntryNotFoundError (an HTTP 404), not a plain
|
| 44 |
-
FileNotFoundError, when a file doesn't exist yet on the Hub.
|
|
|
|
|
|
|
| 45 |
def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token):
|
| 46 |
raise _remote_entry_not_found()
|
| 47 |
|
| 48 |
monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download)
|
| 49 |
state = download_snapshot("org/repo", str(tmp_path), token="tok")
|
| 50 |
assert state == {"papers": {}, "last_synced_at": None}
|
| 51 |
-
assert os.path.exists(tmp_path / "index.faiss")
|
| 52 |
-
assert os.path.exists(tmp_path / "papers.db")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 53 |
|
| 54 |
def test_upload_snapshot_uploads_folder_as_single_atomic_commit(tmp_path, monkeypatch):
|
| 55 |
for name in ["index.faiss", "papers.db", "state.json"]:
|
|
|
|
| 1 |
import json
|
| 2 |
import os
|
| 3 |
import httpx
|
| 4 |
+
import pytest
|
| 5 |
from huggingface_hub.errors import RemoteEntryNotFoundError
|
| 6 |
+
from common.vector_index import load_index
|
| 7 |
+
from common.db import init_db
|
| 8 |
from sync.hf_dataset_store import download_snapshot, upload_snapshot
|
| 9 |
|
| 10 |
|
|
|
|
| 40 |
monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download)
|
| 41 |
state = download_snapshot("org/repo", str(tmp_path), token="tok")
|
| 42 |
assert state == {"papers": {}, "last_synced_at": None}
|
| 43 |
+
assert not os.path.exists(tmp_path / "state.json")
|
| 44 |
|
| 45 |
def test_download_snapshot_handles_empty_remote_dataset_repo(tmp_path, monkeypatch):
|
| 46 |
"""Reproduces a real first-time sync against a brand-new, empty dataset repo:
|
| 47 |
huggingface_hub raises RemoteEntryNotFoundError (an HTTP 404), not a plain
|
| 48 |
+
FileNotFoundError, when a file doesn't exist yet on the Hub. Missing files
|
| 49 |
+
must be left absent (not stubbed with an empty placeholder), since a 0-byte
|
| 50 |
+
file is not a valid "missing" signal to faiss.read_index / sqlite3.connect."""
|
| 51 |
def fake_hf_hub_download(repo_id, repo_type, filename, local_dir, token):
|
| 52 |
raise _remote_entry_not_found()
|
| 53 |
|
| 54 |
monkeypatch.setattr("sync.hf_dataset_store.hf_hub_download", fake_hf_hub_download)
|
| 55 |
state = download_snapshot("org/repo", str(tmp_path), token="tok")
|
| 56 |
assert state == {"papers": {}, "last_synced_at": None}
|
| 57 |
+
assert not os.path.exists(tmp_path / "index.faiss")
|
| 58 |
+
assert not os.path.exists(tmp_path / "papers.db")
|
| 59 |
+
|
| 60 |
+
# Downstream loaders must fall back cleanly, exactly as run_sync relies on.
|
| 61 |
+
with pytest.raises(FileNotFoundError):
|
| 62 |
+
load_index(str(tmp_path / "index.faiss"))
|
| 63 |
+
conn = init_db(str(tmp_path / "papers.db"))
|
| 64 |
+
assert conn.execute("SELECT COUNT(*) FROM papers").fetchone()[0] == 0
|
| 65 |
|
| 66 |
def test_upload_snapshot_uploads_folder_as_single_atomic_commit(tmp_path, monkeypatch):
|
| 67 |
for name in ["index.faiss", "papers.db", "state.json"]:
|