File size: 7,652 Bytes
3fcd223
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
02c176b
3fcd223
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6c5a0cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3fcd223
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
from threading import Event
import pytest
from contrib import cache, collection
from contrib.aggregation import Accumulator
from contrib.domain import ActivityEvent, CollectionRequest, Update, timestamp
from contrib.config import SOURCES
from contrib.collectors.classifiers import classify
from contrib.collectors import activity
from contrib.hub import Cancelled, HubError, validate_username


def event(key="model.commit", identity="repo:commit", at="2024-02-29T12:00:00Z"):
    return ActivityEvent(key, identity, timestamp(at))


def test_deduplication_keeps_creation_separate_from_commit_and_filters_year():
    acc = Accumulator(CollectionRequest("someone", 2024))
    acc.apply(Update("repositories", [event(), event(), event("model.created"),
                                     event(at="2023-12-31T12:00:00Z")]))
    assert len(acc.snapshot["bins"]["model.commit"]) == 366
    assert sum(acc.snapshot["bins"]["model.commit"]) == 1
    assert sum(acc.snapshot["bins"]["model.created"]) == 1
    assert acc.snapshot["bins"]["model.commit"][59] == 1


def test_utc_date_boundary():
    assert timestamp("2024-01-01T00:30:00+02:00").year == 2023
    assert timestamp("invalid") is None


def test_partial_and_cancelled_are_never_complete():
    acc = Accumulator(CollectionRequest("someone", 2024))
    for source in SOURCES:
        acc.apply(Update(source, status="complete"))
    acc.finish()
    assert acc.snapshot["phase"] == "complete"
    acc.apply(Update("likes", status="partial", warnings=["Feed unavailable"]))
    acc.finish()
    assert acc.snapshot["phase"] == "partial"
    acc.finish(cancelled=True)
    assert acc.snapshot["phase"] == "cancelled"


def test_snapshots_do_not_mutate_previously_emitted_data():
    acc = Accumulator(CollectionRequest("someone", 2024))
    first = acc.copy()
    acc.apply(Update("repositories", [event()]))
    assert sum(first["bins"]["model.commit"]) == 0


def discussion(author, time="2024-03-01T12:00:00Z"):
    return {"time": time, "eventId": "comment-1", "repoType": "model", "repoId": "other/model",
            "discussionData": {"num": 2, "createdAt": "2024-02-01T12:00:00Z",
                               "author": {"name": author}, "isPullRequest": True}}


def test_own_pr_opening_and_later_followup_are_distinct():
    records = classify("community", discussion("someone"), "someone")
    assert [r.key for r in records] == ["pr.opened", "pr.followup"]
    assert records[0].at.month == 2
    assert records[1].at.month == 3
    opening = classify("community", discussion("someone", "2024-02-01T12:00:03Z"), "someone")
    assert [r.key for r in opening] == ["pr.opened"]


def test_external_pr_followup_and_like_subtype():
    assert [r.key for r in classify("community", discussion("other"), "someone")] == ["pr.followup"]
    records = classify("likes", {"time": "2024-01-02T00:00:00Z", "repoType": "space", "repoId": "other/app"}, "someone")
    assert records[0].key == "like.space"


def test_collection_upvotes_match_current_hub_feed():
    records = classify("upvotes", {"time": "2026-10-28T09:23:27Z", "targetType": "collection", "targetUrl": "/collections/org/name-id", "target": {"id": "id"}}, "someone")
    assert records[0].key == "upvote.collection"


@pytest.mark.parametrize("payload", [[{"time": "x"}], {"recentActivity": [{"time": "x"}], "cursor": "next"}, {"items": [{"time": "x"}]}])
def test_benign_feed_envelopes(payload):
    assert len(activity.normalize(payload)[0]) == 1


def test_unknown_feed_schema_is_not_zero_activity():
    with pytest.raises(HubError):
        activity.normalize({"unexpected": []})


def test_network_failure_preserves_first_batch(monkeypatch):
    class FakeHub:
        count = 0
        def __init__(self, cancel): pass
        def close(self): pass
        def get(self, *args, **kwargs):
            self.count += 1
            if self.count > 1: raise HubError("Temporary failure")
            return {"recentActivity": [{"time": "2024-03-01T00:00:00Z", "repoType": "space", "repoId": "other/app"}], "cursor": "next"}, None
    monkeypatch.setattr(activity, "HubClient", FakeHub)
    updates = []
    activity.collect_activity(CollectionRequest("someone", 2024), "likes", Event(), updates.append)
    assert updates[-1].status == "partial"
    assert any(update.events for update in updates)
    assert not any(update.status == "complete" for update in updates)


def test_repeated_feed_page_stops_with_partial_result(monkeypatch):
    class FakeHub:
        def __init__(self, cancel): pass
        def close(self): pass
        def get(self, *args, **kwargs):
            return {"recentActivity": [{"time": "2024-03-01T00:00:00Z", "repoType": "space", "repoId": "other/app"}]}, None
    monkeypatch.setattr(activity, "HubClient", FakeHub)
    updates = []
    activity.collect_activity(CollectionRequest("someone", 2024), "likes", Event(), updates.append)
    assert updates[-1].status == "partial"
    assert updates[-1].pages == 2


def test_cache_expiry_and_private_isolation(monkeypatch):
    monkeypatch.setattr(cache, "current_year", lambda: 2024)
    cache._items.clear()
    now = [10.0]
    monkeypatch.setattr(cache, "monotonic", lambda: now[0])
    public = CollectionRequest("someone", 2024)
    value = Accumulator(public).copy()
    value["phase"] = "complete"
    cache.put(public, value)
    found = cache.get(public)
    assert found["cached"]
    found["username"] = "modified"
    assert cache.get(public)["username"] == "someone"
    private = CollectionRequest("someone", 2024, private=True, token="test-only")
    assert cache.get(private) is None
    cache.put(private, {**value, "username": "private-result"})
    assert cache.get(public)["username"] == "someone"
    now[0] += 601
    assert cache.get(public) is None


def test_cancel_does_not_touch_other_sessions():
    a, b = Event(), Event()
    collection._active.update(test_a=a, test_b=b)
    try:
        collection.stop("test_a")
        assert a.is_set() and not b.is_set()
    finally:
        collection._active.pop("test_a")
        collection._active.pop("test_b")


def test_stream_cleans_up_after_generator_close(monkeypatch):
    request = CollectionRequest("someone", 2024)
    generator = collection.stream(request, "test-close", refresh=True)
    assert next(generator)["phase"] == "queued"
    assert "test-close" in collection._active
    generator.close()
    assert "test-close" not in collection._active


def test_closing_live_stream_joins_all_collection_workers(monkeypatch):
    finished = []
    class FakeHub:
        def __init__(self, cancel): pass
        def verify_profile(self, username): return {}
        def close(self): pass
    def worker(request, source, cancel, emit):
        emit(Update(source, detail="Reading"))
        cancel.wait(5)
        finished.append(source)
    monkeypatch.setattr(collection, "HubClient", FakeHub)
    monkeypatch.setattr(collection, "collect_activity", worker)
    monkeypatch.setattr(collection, "collect_repositories", lambda request, cancel, emit: worker(request, "repositories", cancel, emit))
    generator = collection.stream(CollectionRequest("someone", 2024), "test-workers", refresh=True)
    assert next(generator)["phase"] == "queued"
    assert next(generator)["phase"] == "loading"
    assert next(generator)["phase"] == "loading"
    generator.close()
    assert set(finished) == set(SOURCES)
    assert "test-workers" not in collection._active


@pytest.mark.parametrize("value", ["https://huggingface.co/person", "@person", "<script>", ""])
def test_invalid_username_rejected(value):
    with pytest.raises(HubError):
        validate_username(value)