Spaces:
Running
Running
File size: 7,652 Bytes
3fcd223 02c176b 3fcd223 6c5a0cc 3fcd223 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 | from threading import Event
import pytest
from contrib import cache, collection
from contrib.aggregation import Accumulator
from contrib.domain import ActivityEvent, CollectionRequest, Update, timestamp
from contrib.config import SOURCES
from contrib.collectors.classifiers import classify
from contrib.collectors import activity
from contrib.hub import Cancelled, HubError, validate_username
def event(key="model.commit", identity="repo:commit", at="2024-02-29T12:00:00Z"):
return ActivityEvent(key, identity, timestamp(at))
def test_deduplication_keeps_creation_separate_from_commit_and_filters_year():
acc = Accumulator(CollectionRequest("someone", 2024))
acc.apply(Update("repositories", [event(), event(), event("model.created"),
event(at="2023-12-31T12:00:00Z")]))
assert len(acc.snapshot["bins"]["model.commit"]) == 366
assert sum(acc.snapshot["bins"]["model.commit"]) == 1
assert sum(acc.snapshot["bins"]["model.created"]) == 1
assert acc.snapshot["bins"]["model.commit"][59] == 1
def test_utc_date_boundary():
assert timestamp("2024-01-01T00:30:00+02:00").year == 2023
assert timestamp("invalid") is None
def test_partial_and_cancelled_are_never_complete():
acc = Accumulator(CollectionRequest("someone", 2024))
for source in SOURCES:
acc.apply(Update(source, status="complete"))
acc.finish()
assert acc.snapshot["phase"] == "complete"
acc.apply(Update("likes", status="partial", warnings=["Feed unavailable"]))
acc.finish()
assert acc.snapshot["phase"] == "partial"
acc.finish(cancelled=True)
assert acc.snapshot["phase"] == "cancelled"
def test_snapshots_do_not_mutate_previously_emitted_data():
acc = Accumulator(CollectionRequest("someone", 2024))
first = acc.copy()
acc.apply(Update("repositories", [event()]))
assert sum(first["bins"]["model.commit"]) == 0
def discussion(author, time="2024-03-01T12:00:00Z"):
return {"time": time, "eventId": "comment-1", "repoType": "model", "repoId": "other/model",
"discussionData": {"num": 2, "createdAt": "2024-02-01T12:00:00Z",
"author": {"name": author}, "isPullRequest": True}}
def test_own_pr_opening_and_later_followup_are_distinct():
records = classify("community", discussion("someone"), "someone")
assert [r.key for r in records] == ["pr.opened", "pr.followup"]
assert records[0].at.month == 2
assert records[1].at.month == 3
opening = classify("community", discussion("someone", "2024-02-01T12:00:03Z"), "someone")
assert [r.key for r in opening] == ["pr.opened"]
def test_external_pr_followup_and_like_subtype():
assert [r.key for r in classify("community", discussion("other"), "someone")] == ["pr.followup"]
records = classify("likes", {"time": "2024-01-02T00:00:00Z", "repoType": "space", "repoId": "other/app"}, "someone")
assert records[0].key == "like.space"
def test_collection_upvotes_match_current_hub_feed():
records = classify("upvotes", {"time": "2026-10-28T09:23:27Z", "targetType": "collection", "targetUrl": "/collections/org/name-id", "target": {"id": "id"}}, "someone")
assert records[0].key == "upvote.collection"
@pytest.mark.parametrize("payload", [[{"time": "x"}], {"recentActivity": [{"time": "x"}], "cursor": "next"}, {"items": [{"time": "x"}]}])
def test_benign_feed_envelopes(payload):
assert len(activity.normalize(payload)[0]) == 1
def test_unknown_feed_schema_is_not_zero_activity():
with pytest.raises(HubError):
activity.normalize({"unexpected": []})
def test_network_failure_preserves_first_batch(monkeypatch):
class FakeHub:
count = 0
def __init__(self, cancel): pass
def close(self): pass
def get(self, *args, **kwargs):
self.count += 1
if self.count > 1: raise HubError("Temporary failure")
return {"recentActivity": [{"time": "2024-03-01T00:00:00Z", "repoType": "space", "repoId": "other/app"}], "cursor": "next"}, None
monkeypatch.setattr(activity, "HubClient", FakeHub)
updates = []
activity.collect_activity(CollectionRequest("someone", 2024), "likes", Event(), updates.append)
assert updates[-1].status == "partial"
assert any(update.events for update in updates)
assert not any(update.status == "complete" for update in updates)
def test_repeated_feed_page_stops_with_partial_result(monkeypatch):
class FakeHub:
def __init__(self, cancel): pass
def close(self): pass
def get(self, *args, **kwargs):
return {"recentActivity": [{"time": "2024-03-01T00:00:00Z", "repoType": "space", "repoId": "other/app"}]}, None
monkeypatch.setattr(activity, "HubClient", FakeHub)
updates = []
activity.collect_activity(CollectionRequest("someone", 2024), "likes", Event(), updates.append)
assert updates[-1].status == "partial"
assert updates[-1].pages == 2
def test_cache_expiry_and_private_isolation(monkeypatch):
monkeypatch.setattr(cache, "current_year", lambda: 2024)
cache._items.clear()
now = [10.0]
monkeypatch.setattr(cache, "monotonic", lambda: now[0])
public = CollectionRequest("someone", 2024)
value = Accumulator(public).copy()
value["phase"] = "complete"
cache.put(public, value)
found = cache.get(public)
assert found["cached"]
found["username"] = "modified"
assert cache.get(public)["username"] == "someone"
private = CollectionRequest("someone", 2024, private=True, token="test-only")
assert cache.get(private) is None
cache.put(private, {**value, "username": "private-result"})
assert cache.get(public)["username"] == "someone"
now[0] += 601
assert cache.get(public) is None
def test_cancel_does_not_touch_other_sessions():
a, b = Event(), Event()
collection._active.update(test_a=a, test_b=b)
try:
collection.stop("test_a")
assert a.is_set() and not b.is_set()
finally:
collection._active.pop("test_a")
collection._active.pop("test_b")
def test_stream_cleans_up_after_generator_close(monkeypatch):
request = CollectionRequest("someone", 2024)
generator = collection.stream(request, "test-close", refresh=True)
assert next(generator)["phase"] == "queued"
assert "test-close" in collection._active
generator.close()
assert "test-close" not in collection._active
def test_closing_live_stream_joins_all_collection_workers(monkeypatch):
finished = []
class FakeHub:
def __init__(self, cancel): pass
def verify_profile(self, username): return {}
def close(self): pass
def worker(request, source, cancel, emit):
emit(Update(source, detail="Reading"))
cancel.wait(5)
finished.append(source)
monkeypatch.setattr(collection, "HubClient", FakeHub)
monkeypatch.setattr(collection, "collect_activity", worker)
monkeypatch.setattr(collection, "collect_repositories", lambda request, cancel, emit: worker(request, "repositories", cancel, emit))
generator = collection.stream(CollectionRequest("someone", 2024), "test-workers", refresh=True)
assert next(generator)["phase"] == "queued"
assert next(generator)["phase"] == "loading"
assert next(generator)["phase"] == "loading"
generator.close()
assert set(finished) == set(SOURCES)
assert "test-workers" not in collection._active
@pytest.mark.parametrize("value", ["https://huggingface.co/person", "@person", "<script>", ""])
def test_invalid_username_rejected(value):
with pytest.raises(HubError):
validate_username(value)
|