File size: 15,625 Bytes
7a0d33c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8f05c9e
 
 
7a0d33c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
"""ุตูุญุฉ ุงู„ู…ุฑุฌุนุŒ ูˆู‚ุงุฆู…ุฉ ุงู„ู…ุฑุงุฌุน โ€” ุชู‚ุฑูŠุฑ ู…ุนุงูŠูŠุฑ ุงู„ุชูˆุซูŠู‚.

ู‚ููŠุณุช ุนู„ู‰ ุชุดุบูŠู„ุงุช ุญู‚ูŠู‚ูŠุฉ ู‚ุจู„ ุงู„ุนู„ุงุฌ:

  ูจูข ุงุณุชุดู‡ุงุฏู‹ุง ู…ู† ูจูฃ ุนู„ู‰ ุณุช ุญู‚ุงุฆุจ ูŠุจุฏุฃ ู…ู† ุงู„ุตูุญุฉ ูกุŒ ู„ุฃู† ูƒู„ ู…ุตุฏุฑ ูƒุงู† ูŠูู‚ุฑุฃ
  ู…ู† ุฃูˆู„ ุญุฑู ููŠู‡ุŒ ูุงู„ุตูุญุฉ ุงู„ู…ุฐูƒูˆุฑุฉ ุตูุญุฉู ุบู„ุงู ุงู„ู…ุณุชู†ุฏ ู„ุง ุตูุญุฉู ุงู„ููƒุฑุฉ.

  ุญู‚ูŠุจุฉ ู…ู† ูŠูˆู…ูŠู† ุฎุฑุฌุช ุจู€ ูกูขูฅ ู…ุฑุฌุนู‹ุงุŒ ูกูขู  ู…ู†ู‡ุง ู„ู… ูŠูุณุชุดู‡ุฏ ุจู‡ ููŠ ุงู„ู…ุชู†ุŒ
  ูˆูงูกูช ู…ู†ู‡ุง ู…ูƒุชูˆุจ ููŠู‡ ยซ(ุฏ.ุช)ยปุŒ ูˆููŠู‡ุง ยซู…ุดุฑูˆุน ุงู„ูƒุงุฌูˆ ููŠ ู…ุตุฑยป ูˆยซุจูƒุงู„ูˆุฑูŠูˆุณ
  ุงู„ุฌูŠูˆู„ูˆุฌูŠุง ุงู„ุนุงู…ุฉยป ููŠ ุญู‚ูŠุจุฉ ุนู† ุงู„ุชุทูˆูŠุฑ ุงู„ุนู‚ุงุฑูŠ โ€” ู„ุฃู† ุงู„ู‚ุงุฆู…ุฉ ูƒุงู†ุช ุชูุจู†ู‰
  ู…ู† ูƒู„ ุฑุงุจุท ู†ุฒู‘ู„ู‡ ุงู„ุจุญุซ ู„ุง ู…ู† ุงู„ู…ุตุงุฏุฑ ุงู„ุชูŠ ุงุณุชุนู…ู„ู‡ุง ุงู„ู†ุต.
"""

from __future__ import annotations

import json

import pytest

from app.agents.arabic_bag.citations import citation_block, in_text, pages_for
from app.agents.arabic_bag.references import (
    _work_key,
    cited_urls,
    drop_stale,
    is_in_text_marker,
    orphan_citations,
)
from app.agents.arabic_bag.search import _best_window

# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• ุตูุญุฉ ุงู„ููƒุฑุฉ ู„ุง ุตูุญุฉ ุงู„ุบู„ุงู


def _pages(*spec):
    return [(number, text) for number, text in spec]


def test_the_window_is_the_pages_that_discuss_the_cover():
    pages = _pages(
        (1, "ุตูุญุฉ ุงู„ุบู„ุงู ูˆุงุณู… ุงู„ุฌุงู…ุนุฉ ูˆุงู„ู…ุดุฑู. " * 10),
        (2, "ูู‡ุฑุณ ุงู„ู…ุญุชูˆูŠุงุช ูˆู‚ุงุฆู…ุฉ ุงู„ุฌุฏุงูˆู„. " * 10),
        (18, "ุงู„ู…ุฎุงุทุฑุฉ ุงู„ู…ุญุณูˆุจุฉ ููŠ ุงู„ุชุทูˆูŠุฑ ุงู„ุนู‚ุงุฑูŠ. " * 10),
        (19, "ุฃุฏูˆุงุช ุฅุฏุงุฑุฉ ุงู„ู…ุฎุงุทุฑุฉ ุงู„ู…ุญุณูˆุจุฉ ุนู…ู„ูŠู‹ุง. " * 10),
    )
    excerpt, first, last = _best_window(pages, "ุงู„ู…ุฎุงุทุฑุฉ ุงู„ู…ุญุณูˆุจุฉ ุงู„ุชุทูˆูŠุฑ ุงู„ุนู‚ุงุฑูŠ")
    assert (first, last) == (18, 19)
    assert "ูู‡ุฑุณ ุงู„ู…ุญุชูˆูŠุงุช" not in excerpt


def test_a_narrower_window_wins_over_a_wider_one_of_the_same_score():
    """ู†ุงูุฐุฉ ุชุญู…ู„ ุงู„ุฏุฑุฌุฉ ู†ูุณู‡ุง ููŠ ุตูุญุชูŠู† ุฃุฏู‚ู‘ ููŠ ุงู„ุฅุญุงู„ุฉ ู…ู† ู†ุงูุฐุฉ ุชุจู„ุบู‡ุง ููŠ
    ุฎู…ุณุŒ ูˆุงุจุชู„ุงุน ุตูุญุงุช ู„ุง ุชุฎุตู‘ ุงู„ู…ูˆุถูˆุน ูŠุนูŠุฏ ุงู„ู…ุดูƒู„ุฉ ู†ูุณู‡ุง."""
    pages = _pages(
        (1, "ู…ู‚ุฏู…ุฉ ุนุงู…ุฉ ู„ุง ุชุฎุต ุงู„ู…ูˆุถูˆุน. " * 8),
        (2, "ุชู…ู‡ูŠุฏ ุนุงู… ุขุฎุฑ. " * 8),
        (3, "ุงู„ุชุณูˆูŠู‚ ุงู„ุนู‚ุงุฑูŠ ูˆู‚ู†ูˆุงุช ุงู„ุชุฑูˆูŠุฌ ุงู„ุฑู‚ู…ูŠุฉ. " * 8),
    )
    _, first, last = _best_window(pages, "ุงู„ุชุณูˆูŠู‚ ุงู„ุนู‚ุงุฑูŠ ู‚ู†ูˆุงุช ุงู„ุชุฑูˆูŠุฌ")
    assert (first, last) == (3, 3)


def test_a_document_that_never_mentions_the_cover_yields_its_opening():
    """ูˆู‡ูˆ ู…ุง ูƒุงู† ูŠูุนุงุฏ ู‚ุจู„ ู‡ุฐุงุŒ ูˆู‡ูˆ ุงู„ุตุฏู‚ ููŠ ูˆุตู ุงู„ู…ู‚ุชุทู."""
    pages = _pages((4, "ู†ุต ู„ุง ุนู„ุงู‚ุฉ ู„ู‡. " * 8), (5, "ูˆู„ุง ู‡ุฐุง. " * 8))
    excerpt, first, last = _best_window(pages, "ุงู„ุฐูƒุงุก ุงู„ุงุตุทู†ุงุนูŠ ุงู„ุชูˆู„ูŠุฏูŠ")
    assert first == 4 and excerpt
    assert last in (4, 5)


def test_the_document_s_own_page_number_is_used():
    """ุตูุญุฉูŒ ุฑู‚ู…ู‡ุง ูกูง ู„ุง ุชูุณู…ู‘ู‰ ุงู„ุฃูˆู„ู‰ ู„ุฃู† ู…ุง ู‚ุจู„ู‡ุง ู„ู… ูŠูุณุชุฎุฑุฌ."""
    pages = _pages((17, "ุงู„ุฃู…ู† ุงู„ููƒุฑูŠ ู„ุฏู‰ ุงู„ุทู„ุงุจ ุงู„ุฌุงู…ุนูŠูŠู†. " * 10))
    _, first, last = _best_window(pages, "ุงู„ุฃู…ู† ุงู„ููƒุฑูŠ ุงู„ุทู„ุงุจ")
    assert (first, last) == (17, 17)


def test_the_citation_reads_the_recorded_range_not_the_arithmetic():
    source = {"pages": [19, 21], "page_lengths": [1200, 1200, 1200, 1200]}
    assert pages_for(source, 2500) == (19, 21)


def test_a_source_read_before_the_window_existed_still_works():
    """ุชุดุบูŠู„ุฉ ู‚ุฏูŠู…ุฉ ู„ุง ุชุญู…ู„ `pages`ุŒ ููŠูุญุณุจ ุงู„ู…ุฏู‰ ู…ู† ุฃุทูˆุงู„ ุงู„ุตูุญุงุช ูƒู…ุง ูƒุงู†."""
    assert pages_for({"page_lengths": [1200, 1100, 900]}, 2500) == (1, 3)
    assert pages_for({"page_lengths": []}, 2500) is None


def test_the_page_marker_carries_its_dot():
    assert in_text("ุงู„ุนุชูŠุจูŠ", "2023", (19, 19)) == "(ุงู„ุนุชูŠุจูŠุŒ 2023ุŒ ุต. 19)"
    assert in_text("ุงู„ุนุชูŠุจูŠ", "2023", (19, 21)) == "(ุงู„ุนุชูŠุจูŠุŒ 2023ุŒ ุต ุต. 19-21)"
    assert in_text("Smith", "2023", (19, 21)) == "(Smith, 2023, pp. 19-21)"


def test_two_covers_of_one_document_can_cite_two_different_pages():
    """ยซูŠู…ู†ุน ุชูƒุฑุงุฑ ุฑู‚ู… ุงู„ุตูุญุฉ ู†ูุณู‡ ููŠ ุนุฏุฉ ู…ูˆุงุถุน ู…ุฎุชู„ูุฉ ุฏูˆู† ุชุญู‚ู‚ยป."""
    pages = _pages(
        (5, "ุงู„ุชู…ูˆูŠู„ ุงู„ุฐุงุชูŠ ูˆุงู„ุดุฑุงูƒุงุช ุงู„ุงุณุชุซู…ุงุฑูŠุฉ ููŠ ุงู„ู…ุดุฑูˆุนุงุช. " * 10),
        (31, "ุฅุฏุงุฑุฉ ุงู„ุฃุฒู…ุงุช ูˆุชู†ูˆูŠุน ุงู„ุงุณุชุซู…ุงุฑุงุช ู„ู…ูˆุงุฌู‡ุฉ ุงู„ู…ุฎุงุทุฑ. " * 10),
    )
    _, a, _ = _best_window(pages, "ุงู„ุชู…ูˆูŠู„ ุงู„ุฐุงุชูŠ ุงู„ุดุฑุงูƒุงุช ุงู„ุงุณุชุซู…ุงุฑูŠุฉ")
    _, b, _ = _best_window(pages, "ุฅุฏุงุฑุฉ ุงู„ุฃุฒู…ุงุช ุชู†ูˆูŠุน ุงู„ุงุณุชุซู…ุงุฑุงุช")
    assert a == 5 and b == 31


# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• ุงู„ู‚ุงุฆู…ุฉ = ู…ุง ุงุณุชุดู‡ุฏ ุจู‡ ุงู„ู…ุชู†


CREDITS = {
    "https://a.org/x.pdf": {"author": "ุงู„ุนู†ุฒูŠ", "year": "2023"},
    "https://b.org/y.pdf": {"author": "ุงู„ุบุงู…ุฏูŠ", "year": "2022"},
    "https://c.org/z.pdf": {"author": "ุงู„ู‚ุญุทุงู†ูŠ", "year": "2024"},
}
CONTENT = [{"cover": "ู…ุญูˆุฑ", "blocks": [
    {"block_type": "paragraph",
     "paragraphs": ["ู†ุต ูŠุณุชุดู‡ุฏ ุจู€ (ุงู„ุนู†ุฒูŠุŒ 2023ุŒ ุต. 19) ุซู… ูŠูˆุงุตู„.",
                    "ูˆู‡ู†ุง (ุงู„ุบุงู…ุฏูŠุŒ 2022ุŒ ุต ุต. 4-6) ููŠ ู…ูˆุถุน ุขุฎุฑ."]}]}]


def test_only_the_sources_the_text_cites_reach_the_list():
    used = cited_urls(CONTENT, CREDITS)
    assert used == {"https://a.org/x.pdf", "https://b.org/y.pdf"}
    assert "https://c.org/z.pdf" not in used


def test_a_source_is_matched_whatever_page_the_citation_names():
    """ุฑู‚ู… ุงู„ุตูุญุฉ ูŠุฎุชู„ู ู…ู† ู…ูˆุถุน ู„ุขุฎุฑุŒ ูุงู„ุงุณู… ูˆุงู„ุณู†ุฉ ู‡ู…ุง ู…ุง ูŠู…ูŠู‘ุฒ ุงู„ู…ุตุฏุฑ."""
    content = [{"blocks": [{"paragraphs": ["(ุงู„ุนู†ุฒูŠุŒ 2023)"]}]}]
    assert cited_urls(content, CREDITS) == {"https://a.org/x.pdf"}


def test_a_citation_with_no_source_in_this_run_is_named_not_hidden():
    content = [{"blocks": [{"paragraphs": ["ู†ุต ููŠู‡ (ุงู„ุญุฑุจูŠุŒ 2021ุŒ ุต. 3) ูู‚ุท."]}]}]
    assert orphan_citations(content, CREDITS) == ["ุงู„ุญุฑุจูŠ"]
    assert orphan_citations(CONTENT, CREDITS) == []


# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• ุดูƒู„ ุงู„ู…ุฏุฎู„


def test_an_in_text_marker_never_enters_the_list():
    """ยซู„ุง ูŠุฌูˆุฒ ุฃู† ุชู†ุชู‚ู„ ุตูŠุบุฉ ุงู„ุชูˆุซูŠู‚ ุฏุงุฎู„ ุงู„ู…ุชู† ุฅู„ู‰ ู‚ุงุฆู…ุฉ ุงู„ู…ุฑุงุฌุนยป."""
    assert is_in_text_marker("ุฃุญู…ุฏุŒ 2020ุŒ ุต. 3")
    assert is_in_text_marker("(ุฃุญู…ุฏุŒ 2020ุŒ ุต. 3)")
    assert is_in_text_marker("OECD, 2024, p. 55")
    assert is_in_text_marker("ุจูˆุจูƒุฑ ุฏูŠุงุจูŠุŒ 2012ุŒ ุต. 7")


def test_a_real_entry_is_not_mistaken_for_a_marker():
    assert not is_in_text_marker("ุฃุญู…ุฏุŒ ู…ุญู…ุฏ. (2020). ุนู†ูˆุงู† ุงู„ูƒุชุงุจ. ุงู„ู‚ุงู‡ุฑุฉ: ุฏุงุฑ ุงู„ู†ุดุฑ.")
    assert not is_in_text_marker("IBM. (n.d.). What is artificial intelligence? IBM.")
    assert not is_in_text_marker("ูˆุฒุงุฑุฉ ุงู„ุชุนู„ูŠู…. (ุฏ.ุช). ุงู„ุฏู„ูŠู„ ุงู„ุชู†ุธูŠู…ูŠ.")


def test_two_shapes_of_one_work_share_a_key():
    """ู†ูุณ ุงู„ูˆุฑู‚ุฉ ุจุนู†ูˆุงู†ูŠู†ุŒ ุฃุญุฏู‡ู…ุง ู…ุจุชูˆุฑ ูˆุฃุญุฏู‡ู…ุง ุจู„ุง ุณู†ุฉ โ€” ู…ุฏุฎู„ ูˆุงุญุฏ."""
    a = "ุฃุซุฑ ุงู„ู…ุฌุงู„ุงุช ุงู„ู…ุนุฑููŠุฉ ู„ุฅุฏุงุฑุฉ ุงู„ู…ุดุงุฑูŠุน ุนู„ู‰ ุฌูˆุฏุฉ ุงู„ู…ุฎุฑุฌุงุช. (2022). https://x/1"
    b = "ุฃุซุฑ ุงู„ู…ุฌุงู„ุงุช ุงู„ู…ุนุฑููŠุฉ ู„ุฅุฏุงุฑุฉ ุงู„ู…ุดุงุฑูŠุน ุนู„ู‰ ุฌูˆุฏุฉ ุงู„ู…ุฎุฑุฌุงุช. (ุฏ.ุช). https://x/2"
    assert _work_key(a) != _work_key(b)          # ุงู„ุณู†ุฉ ุชู…ูŠู‘ุฒ
    c = "ุฃุซุฑ ุงู„ู…ุฌุงู„ุงุช ุงู„ู…ุนุฑููŠุฉ ู„ุฅุฏุงุฑุฉ ุงู„ู…ุดุงุฑูŠุน ุนู„ู‰ ุฌูˆุฏุฉ ุงู„ู…ุฎุฑุฌุงุช. (2022). https://x/2"
    assert _work_key(a) == _work_key(c)          # ุงู„ุฑุงุจุท ูˆุญุฏู‡ ู„ุง ูŠู…ูŠู‘ุฒ


def test_an_entry_that_is_only_a_link_keeps_its_own_identity():
    """ุตูุญุฉ ู…ุคุณุณูŠุฉ ุจู„ุง ุนู†ูˆุงู† ุญู‚ูŠู‚ูŠ ุชูู…ูŠูŽู‘ุฒ ุจุฑุงุจุทู‡ุงุŒ ูˆุฅู„ุง ุงู†ุทูˆุช ูƒู„ู‡ุง ููŠ ู…ุฏุฎู„."""
    assert _work_key("ู…ุฑุฌุน https://x.org/0") != _work_key("ู…ุฑุฌุน https://x.org/1")


# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• ุงู„ุญุฏุงุซุฉ ู„ุง ุชูƒุณุฑ ุงู„ู…ุทุงุจู‚ุฉ


def _ref(text, language="ar"):
    return {"reference": text, "language": language, "kind": "ูƒุชุงุจ"}


def test_the_recency_floor_is_not_applied_to_a_cited_list():
    """ยซูƒู„ ู…ุตุฏุฑ ุชู… ุงู„ุงุณุชุดู‡ุงุฏ ุจู‡ ููŠ ุงู„ู…ุชู† ูŠุฌุจ ุฃู† ูŠุธู‡ุฑ ููŠ ู‚ุงุฆู…ุฉ ุงู„ู…ุฑุงุฌุนยป โ€”
    ูุญุฐู ุงู„ู‚ุฏูŠู… ู…ู† ุงู„ู‚ุงุฆู…ุฉ ูŠุตู†ุน ุจูŠุฏู‡ ุงู„ุฎู„ู„ ุงู„ุฐูŠ ูŠู…ู†ุนู‡."""
    import inspect

    from app.agents.arabic_bag import references

    # ุงู„ุตูŠุงุบุฉ ุจูˆูƒูŠู„ ุตุงุฑุช ุขุฎุฑ ู…ู„ุงุฐุŒ ูˆุงู„ุฃุฑุถูŠุฉ ุจู‚ูŠุช ู…ุนู‡ุง ูˆุญุฏู‡ุง.
    assert "drop_stale" in inspect.getsource(references._refs_from_agent)
    assert "drop_stale" not in inspect.getsource(references.run_references)


def test_the_floor_still_applies_when_nothing_was_cited():
    floor_refs = [_ref(f"ู…ุคู„ู{i}ุŒ ุฃ. (2024). ุนู†ูˆุงู†.") for i in range(9)]
    floor_refs.append(_ref("ู‚ุฏูŠู…ุŒ ุจ. (2009). ุนู†ูˆุงู†."))
    assert len(drop_stale(floor_refs)) == 9


# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• ุงู„ุฎุทูˆุฉ ู†ูุณู‡ุง


def test_the_references_step_can_no_longer_be_skipped():
    """ุญู‚ูŠุจุฉ ุดูุบูู‘ู„ุช ุจู„ุง ู‡ุฐู‡ ุงู„ุฎุทูˆุฉ ุฎุฑุฌุช ุจุนุดุฑุฉ ุงุณุชุดู‡ุงุฏุงุช ูˆุตูุญุฉ ู…ุฑุงุฌุน ูุงุฑุบุฉ."""
    from app.pipelines.registry import get, load_all

    load_all()
    step = next(s for s in get("arabic_training_bag").steps
                if s.key == "bag_ar.references")
    assert step.optional is False
    assert "bag_ar.content" in step.depends_on
    assert "bag_ar.citations" in step.depends_on


def test_the_references_step_runs_after_the_content_that_feeds_it():
    from app.pipelines.registry import get, load_all

    load_all()
    keys = [s.key for s in get("arabic_training_bag").steps]
    assert keys.index("bag_ar.content") < keys.index("bag_ar.references")


# โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ• ู…ุง ูŠูุนุฑุถ ุนู„ู‰ ุงู„ูƒุงุชุจ


def test_a_stale_source_is_still_not_offered_for_citation():
    """ุงู„ุญุฏุงุซุฉ ู…ูˆุถุนู‡ุง ุงู„ุงุฎุชูŠุงุฑุŒ ู‚ุจู„ ุงู„ุงุณุชุดู‡ุงุฏุŒ ู„ุง ุงู„ู‚ุงุฆู…ุฉ ุจุนุฏู‡."""
    scraped = [{"cover": "ู…", "sources": [
        {"title": "ุญุฏูŠุซ", "url": "https://a.org/x.pdf", "pages": [19, 20]},
        {"title": "ู‚ุฏูŠู…", "url": "https://b.org/y.pdf", "pages": [3, 4]}]}]
    credits = {"https://a.org/x.pdf": {"author": "ุงู„ุนู†ุฒูŠ", "year": "2024"},
               "https://b.org/y.pdf": {"author": "ุงู„ู‚ุฏูŠู…ูŠ", "year": "2011"}}
    block = citation_block("ู…", scraped, credits)
    assert "ุงู„ุนู†ุฒูŠ" in block and "ุงู„ู‚ุฏูŠู…ูŠ" not in block
    # ูˆุงู„ุตูุญุฉ ุงู„ู…ุนุฑูˆุถุฉ ู‡ูŠ ุตูุญุฉ ุงู„ู…ู‚ุทุน.
    assert "ุต ุต. 19-20" in block


@pytest.mark.parametrize("cover_pages,expected", [
    ([19, 19], "ุต. 19"),
    ([19, 21], "ุต ุต. 19-21"),
])
def test_the_offered_citation_names_the_excerpt_s_pages(cover_pages, expected):
    scraped = [{"cover": "ู…", "sources": [
        {"title": "ุช", "url": "https://a.org/x.pdf", "pages": cover_pages}]}]
    credits = {"https://a.org/x.pdf": {"author": "ุงู„ุนู†ุฒูŠ", "year": "2024"}}
    assert expected in citation_block("ู…", scraped, credits)


def test_the_head_of_the_document_reaches_the_extractor_not_the_excerpt():
    """ุงู„ู…ุคู„ู ูˆุงู„ุณู†ุฉ ุนู„ู‰ ุตูุญุฉ ุงู„ุนู†ูˆุงู†ุŒ ูˆุงู„ู…ู‚ุชุทู ุตุงุฑ ูŠุจุฏุฃ ู…ู† ูˆุณุท ุงู„ู…ู„ู."""
    import inspect

    from app.agents.arabic_bag import citations

    src = inspect.getsource(citations._extract)
    assert 's.get("head")' in src


def test_the_source_record_carries_the_window_and_the_head():
    import inspect

    from app.agents.arabic_bag import sources

    src = inspect.getsource(sources.run_source_finder)
    assert "focus=focus" in src
    assert '"pages": list(pages) if pages else []' in src
    assert '"head": head' in src


def test_nothing_in_the_new_path_invents_a_page():
    """ยซู„ุง ูŠุฌูˆุฒ ู„ู„ุฃุฏุงุฉ ุงุฎุชู„ุงู‚ ุฑู‚ู… ุตูุญุฉยป โ€” ูƒู„ ุฑู‚ู… ูŠูุทุจุน ุฌุงุก ู…ู† ุงู„ู…ุณุชู†ุฏ.

    ุตูุญุฉ ุงู„ูˆูŠุจ ู„ุง ุตูุญุงุช ู„ู‡ุงุŒ ูู„ุง ุฑู‚ู… ู„ู‡ุงุ› ูˆุงู„ู…ุณุชู†ุฏ ุงู„ุฐูŠ ู„ู… ูŠูุณุชุฎุฑุฌ ู…ู†ู‡ ุดูŠุก
    ู„ุง ู†ุงูุฐุฉ ู„ู‡.
    """
    assert pages_for({"pages": [], "page_lengths": []}, 2500) is None
    assert _best_window([], "ุฃูŠ ู…ูˆุถูˆุน") == ("", None, None)
    web = [{"cover": "ู…", "sources": [
        {"title": "ุตูุญุฉ", "url": "https://w.org/a", "pages": [], "page_lengths": []}]}]
    block = citation_block("ู…", web, {"https://w.org/a": {"author": "ุงู„ู…ุงู„ูƒูŠ",
                                                          "year": "2024"}})
    assert "โ†’ (ุงู„ู…ุงู„ูƒูŠุŒ 2024)" in block
    assert "ุต." not in block.rsplit("โ†’", 1)[-1]


def test_the_cited_set_survives_a_json_round_trip():
    """ุงู„ู…ุญุชูˆู‰ ูŠุตู„ ู…ู† ู…ุฎุฑูŽุฌ ุฎุทูˆุฉ ู…ุญููˆุธ ููŠ ู‚ุงุนุฏุฉ ุงู„ุจูŠุงู†ุงุชุŒ ู„ุง ู…ู† ุงู„ุฐุงูƒุฑุฉ."""
    revived = json.loads(json.dumps(CONTENT, ensure_ascii=False))
    assert cited_urls(revived, CREDITS) == {"https://a.org/x.pdf",
                                            "https://b.org/y.pdf"}


def test_a_cover_offered_sources_must_cite_at_least_one():
    """ยซุทุงู„ู…ุง ุงู„ู…ุตุฏุฑ ูˆูู‚ู‹ุง ู„ู„ู…ุนุงูŠูŠุฑ ูŠุจู‚ู‰ ู†ุณุชุดู‡ุฏ ุจู‡ ููŠ ุงู„ู…ุญูˆุฑ ุฏู‡ยป."""
    from app.agents.arabic_bag.writer import validate_written_cover

    plan = {"blocks": [{"block_type": "paragraph", "heading": "",
                        "needs_lead_in": False, "needs_closing_line": False}]}
    silent = {"blocks": [{"block_type": "paragraph", "heading": "",
                          "paragraphs": ["ูู‚ุฑุฉ ุจู„ุง ุฃูŠ ุชูˆุซูŠู‚ ููŠู‡ุง ุฅุทู„ุงู‚ู‹ุง."]}]}
    errs = validate_written_cover(silent, plan, sourced=True)
    assert any("ุจู„ุง ุชูˆุซูŠู‚ ูˆุงุญุฏ" in e for e in errs)

    voiced = {"blocks": [{"block_type": "paragraph", "heading": "",
                          "paragraphs": ["ูู‚ุฑุฉ ููŠู‡ุง (ุงู„ุนู†ุฒูŠุŒ 2023ุŒ ุต. 19)."]}]}
    assert not any("ุจู„ุง ุชูˆุซูŠู‚ ูˆุงุญุฏ" in e
                   for e in validate_written_cover(voiced, plan, sourced=True))


def test_a_cover_with_no_sources_is_not_asked_to_cite():
    from app.agents.arabic_bag.writer import validate_written_cover

    plan = {"blocks": [{"block_type": "paragraph", "heading": "",
                        "needs_lead_in": False, "needs_closing_line": False}]}
    silent = {"blocks": [{"block_type": "paragraph", "heading": "",
                          "paragraphs": ["ูู‚ุฑุฉ ุจู„ุง ุฃูŠ ุชูˆุซูŠู‚ ููŠู‡ุง ุฅุทู„ุงู‚ู‹ุง."]}]}
    assert not any("ุจู„ุง ุชูˆุซูŠู‚ ูˆุงุญุฏ" in e
                   for e in validate_written_cover(silent, plan, sourced=False))