R3.2.8.6-R1.3C-KK1-R6A-R3 Qualitative Appendix Citation Relevance and Quote Output Integrity Final Fix

#39
config.py CHANGED
@@ -196,7 +196,8 @@ def load_config():
196
  "reference", "references", "reference list", "参考文献",
197
  "acknowledgment", "acknowledgements", "致谢",
198
  "аннотация", "ключевые слова", "аңдатпа", "түйіндеме",
199
- "түйін сөздер", "түйінді сөздер", "кілт сөздер"
 
200
  ]
201
  },
202
 
 
196
  "reference", "references", "reference list", "参考文献",
197
  "acknowledgment", "acknowledgements", "致谢",
198
  "аннотация", "ключевые слова", "аңдатпа", "түйіндеме",
199
+ "түйін сөздер", "түйінді сөздер", "кілт сөздер",
200
+ "appendix", "附录", "приложение", "қосымша"
201
  ]
202
  },
203
 
core/generation_runner.py CHANGED
@@ -183,6 +183,7 @@ class GenerationRunner:
183
  self._translation_worker_collected = 0
184
  self._translation_worker_pending = set()
185
  self._translation_worker_context = {}
 
186
  if self.preview_token:
187
  preview_dir = Path(__file__).resolve().parents[1] / "static" / "previews"
188
  preview_dir.mkdir(parents=True, exist_ok=True)
@@ -345,7 +346,8 @@ class GenerationRunner:
345
  raise RuntimeError(f"Qualitative quotation already used before owning unit: {tag}")
346
 
347
  for tag in sorted(allowed):
348
- count = len(re.findall(rf"\b{re.escape(tag)}\b", value))
 
349
  if count > 1:
350
  raise RuntimeError(
351
  f"Qualitative quotation repeated within owning Findings unit: {tag} count={count}"
@@ -353,7 +355,7 @@ class GenerationRunner:
353
  if count == 1:
354
  quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
355
  value = re.sub(
356
- rf"\b{re.escape(tag)}\b",
357
  lambda _match, frozen_quote=quote: f'"{frozen_quote}"',
358
  value,
359
  count=1,
@@ -365,6 +367,84 @@ class GenerationRunner:
365
  raise RuntimeError(f"Unresolved qualitative quotation tags: {unresolved[:8]}")
366
  return value
367
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
368
  def _write_preview_payload(self, *, force: bool = False) -> None:
369
  with self._preview_lock:
370
  if self.preview_path is None:
@@ -642,6 +722,15 @@ class GenerationRunner:
642
  english_commit_text=english_commit_text,
643
  )
644
  updated_sections = dict(translation_sections or {})
 
 
 
 
 
 
 
 
 
645
 
646
  def publish_translation_stage(stage):
647
  value = str(stage or '').strip().casefold()
@@ -669,7 +758,20 @@ class GenerationRunner:
669
  supports_stage_callback = False
670
  if supports_stage_callback:
671
  call_kwargs['stage_callback'] = publish_translation_stage
672
- result = translate_method(source, target_lang, **call_kwargs)
 
 
 
 
 
 
 
 
 
 
 
 
 
673
  except PipelineCancelledError:
674
  raise
675
  except Exception as error:
@@ -923,6 +1025,10 @@ class GenerationRunner:
923
  )
924
  self._checkpoint("evidence_ready", state, 6)
925
 
 
 
 
 
926
  # ASQ planning never needs access to verbatim quotations. Keep the
927
  # research-logic/front-matter stage on a quote-free internal-evidence
928
  # summary; verbatim text is disclosed later only to its owning Findings
@@ -1400,8 +1506,20 @@ Output only final academic prose. Do not report the word count."""
1400
  if str(item.get("tag") or "") not in used_after
1401
  ]
1402
  needed = max(0, required_survival - len(survived))
 
 
 
 
 
 
 
 
 
 
 
 
1403
  supplemental_raw = agent.build_citation_recovery_paragraph(
1404
- missing_items[:needed], section_name=display_name
1405
  )
1406
  if supplemental_raw:
1407
  supplemental_resolved = agent.audit_and_resolve_citations(
@@ -1423,10 +1541,11 @@ Output only final academic prose. Do not report the word count."""
1423
  agent._extract_memory_from_text(en_raw, sec_title)
1424
  context_tail = " ".join(en_raw.split()[-200:])
1425
  actual_words = self._english_count_for_target(en_resolved)
1426
- accumulated_deviation += actual_words - adjusted_target
1427
- self._upsert_quality_diagnostic(
1428
- quality_diagnostics, self._section_word_diagnostic(display_name, actual_words, adjusted_target)
1429
- )
 
1430
  completed_sections.append(display_name)
1431
  section_key = str(sec.get('section_id') or sec_title)
1432
  if self._section_is_complete(adaptive_structure, completed_sections, section_key):
 
183
  self._translation_worker_collected = 0
184
  self._translation_worker_pending = set()
185
  self._translation_worker_context = {}
186
+ self._qualitative_quote_sources = {}
187
  if self.preview_token:
188
  preview_dir = Path(__file__).resolve().parents[1] / "static" / "previews"
189
  preview_dir.mkdir(parents=True, exist_ok=True)
 
346
  raise RuntimeError(f"Qualitative quotation already used before owning unit: {tag}")
347
 
348
  for tag in sorted(allowed):
349
+ tag_pattern = rf"(?<!\w)\[?{re.escape(tag)}\]?(?!\w)"
350
+ count = len(re.findall(tag_pattern, value))
351
  if count > 1:
352
  raise RuntimeError(
353
  f"Qualitative quotation repeated within owning Findings unit: {tag} count={count}"
 
355
  if count == 1:
356
  quote = str((evidence_by_tag.get(tag) or {}).get("quote") or "").strip()
357
  value = re.sub(
358
+ tag_pattern,
359
  lambda _match, frozen_quote=quote: f'"{frozen_quote}"',
360
  value,
361
  count=1,
 
367
  raise RuntimeError(f"Unresolved qualitative quotation tags: {unresolved[:8]}")
368
  return value
369
 
370
+ @staticmethod
371
+ def _qualitative_quote_source_map(data_metadata):
372
+ return {
373
+ str(item.get("tag") or "").strip(): str(item.get("quote") or "").strip()
374
+ for item in (data_metadata or {}).get("evidence", [])
375
+ if isinstance(item, dict)
376
+ and str(item.get("tag") or "").strip()
377
+ and str(item.get("quote") or "").strip()
378
+ }
379
+
380
+ @classmethod
381
+ def _prepare_qualitative_translation_source(cls, source, quote_sources, quote_ledger):
382
+ """Use one task-level target rendering for every frozen qualitative quote.
383
+
384
+ The first translated occurrence keeps the English quote in context and
385
+ appends its protected QUOTE tag so the target rendering can be captured.
386
+ Later occurrences (notably the Appendix) send only the protected tag and
387
+ are restored from the already QA-passed task-level quote ledger.
388
+ """
389
+ value = str(source or "")
390
+ pending = []
391
+ reused = []
392
+ for tag, quote in sorted((quote_sources or {}).items()):
393
+ wrapped = f'"{quote}"'
394
+ tagged = f'{wrapped} [{tag}]'
395
+ if tagged in value:
396
+ if tag in (quote_ledger or {}):
397
+ value = value.replace(tagged, f"[{tag}]")
398
+ reused.append(tag)
399
+ else:
400
+ pending.append(tag)
401
+ continue
402
+ if wrapped not in value:
403
+ continue
404
+ if tag in (quote_ledger or {}):
405
+ value = value.replace(wrapped, f"[{tag}]")
406
+ reused.append(tag)
407
+ else:
408
+ value = value.replace(wrapped, tagged)
409
+ pending.append(tag)
410
+ return value, pending, reused
411
+
412
+ @classmethod
413
+ def _finalize_qualitative_translation_quotes(cls, translated, pending, reused, quote_ledger):
414
+ value = str(translated or "")
415
+ ledger = dict(quote_ledger or {})
416
+ quote_pattern = r'(?P<quote>"[^"\n]{4,2500}"|“[^”\n]{4,2500}”|«[^»\n]{4,2500}»|„[^“\n]{4,2500}“)'
417
+ for tag in pending or []:
418
+ pattern = re.compile(
419
+ quote_pattern + rf"\s*(?<!\w)\[?{re.escape(tag)}\]?(?!\w)",
420
+ flags=re.UNICODE,
421
+ )
422
+ matches = list(pattern.finditer(value))
423
+ if len(matches) != 1:
424
+ raise RuntimeError(
425
+ f"Qualitative quote translation ledger capture failed: {tag} matches={len(matches)}"
426
+ )
427
+ rendered_quote = matches[0].group("quote")
428
+ ledger[tag] = rendered_quote
429
+ value = value[:matches[0].start()] + rendered_quote + value[matches[0].end():]
430
+
431
+ for tag in reused or []:
432
+ rendered_quote = str(ledger.get(tag) or "").strip()
433
+ if not rendered_quote:
434
+ raise RuntimeError(f"Qualitative quote translation ledger missing target text: {tag}")
435
+ pattern = rf"(?<!\w)\[?{re.escape(tag)}\]?(?!\w)"
436
+ count = len(re.findall(pattern, value))
437
+ if count != 1:
438
+ raise RuntimeError(
439
+ f"Qualitative quote translation ledger reuse failed: {tag} matches={count}"
440
+ )
441
+ value = re.sub(pattern, lambda _m, q=rendered_quote: q, value, count=1)
442
+
443
+ unresolved = sorted(set(re.findall(r"\bQUOTE-\d+\b", value)))
444
+ if unresolved:
445
+ raise RuntimeError(f"Unresolved target-language qualitative quote tags: {unresolved[:8]}")
446
+ return value, ledger
447
+
448
  def _write_preview_payload(self, *, force: bool = False) -> None:
449
  with self._preview_lock:
450
  if self.preview_path is None:
 
722
  english_commit_text=english_commit_text,
723
  )
724
  updated_sections = dict(translation_sections or {})
725
+ quote_ledger_record = dict(updated_sections.get("__quote_ledger__") or {})
726
+ quote_ledger = (
727
+ dict(quote_ledger_record.get("translations") or {})
728
+ if quote_ledger_record.get("target_lang") in {None, "", target_lang}
729
+ else {}
730
+ )
731
+ translation_source, pending_quote_tags, reused_quote_tags = self._prepare_qualitative_translation_source(
732
+ source, getattr(self, "_qualitative_quote_sources", {}), quote_ledger
733
+ )
734
 
735
  def publish_translation_stage(stage):
736
  value = str(stage or '').strip().casefold()
 
758
  supports_stage_callback = False
759
  if supports_stage_callback:
760
  call_kwargs['stage_callback'] = publish_translation_stage
761
+ result = translate_method(translation_source, target_lang, **call_kwargs)
762
+ translated_text = str(result.get('text') or '').strip()
763
+ if pending_quote_tags or reused_quote_tags:
764
+ translated_text, quote_ledger = self._finalize_qualitative_translation_quotes(
765
+ translated_text, pending_quote_tags, reused_quote_tags, quote_ledger
766
+ )
767
+ agent.validate_translation_alignment(source, translated_text)
768
+ result = dict(result)
769
+ result['text'] = translated_text
770
+ updated_sections['__quote_ledger__'] = {
771
+ 'target_lang': target_lang,
772
+ 'translations': dict(quote_ledger),
773
+ 'updated_at': datetime.now(timezone.utc).isoformat(),
774
+ }
775
  except PipelineCancelledError:
776
  raise
777
  except Exception as error:
 
1025
  )
1026
  self._checkpoint("evidence_ready", state, 6)
1027
 
1028
+ self._qualitative_quote_sources = (
1029
+ self._qualitative_quote_source_map(data_metadata) if mode == "asq" else {}
1030
+ )
1031
+
1032
  # ASQ planning never needs access to verbatim quotations. Keep the
1033
  # research-logic/front-matter stage on a quote-free internal-evidence
1034
  # summary; verbatim text is disclosed later only to its owning Findings
 
1506
  if str(item.get("tag") or "") not in used_after
1507
  ]
1508
  needed = max(0, required_survival - len(survived))
1509
+ recovery_items = agent.select_contextual_citation_recovery_items(
1510
+ missing_items, section_name=display_name, section_context=en_resolved,
1511
+ needed=needed, excluded_tags=used_after,
1512
+ )
1513
+ if len(recovery_items) < needed:
1514
+ recovery_items.extend(
1515
+ agent.fetch_contextual_jit_recovery_items(
1516
+ topic=topic, section_name=display_name, section_context=en_resolved,
1517
+ needed=needed - len(recovery_items),
1518
+ excluded_tags=used_after | {str(item.get("tag") or "") for item in recovery_items},
1519
+ )
1520
+ )
1521
  supplemental_raw = agent.build_citation_recovery_paragraph(
1522
+ recovery_items[:needed], section_name=display_name, section_context=en_resolved
1523
  )
1524
  if supplemental_raw:
1525
  supplemental_resolved = agent.audit_and_resolve_citations(
 
1541
  agent._extract_memory_from_text(en_raw, sec_title)
1542
  context_tail = " ".join(en_raw.split()[-200:])
1543
  actual_words = self._english_count_for_target(en_resolved)
1544
+ if not bool(sec.get("exclude_from_word_budget")):
1545
+ accumulated_deviation += actual_words - adjusted_target
1546
+ self._upsert_quality_diagnostic(
1547
+ quality_diagnostics, self._section_word_diagnostic(display_name, actual_words, adjusted_target)
1548
+ )
1549
  completed_sections.append(display_name)
1550
  section_key = str(sec.get('section_id') or sec_title)
1551
  if self._section_is_complete(adaptive_structure, completed_sections, section_key):
core/pipeline_agent.py CHANGED
@@ -2515,13 +2515,216 @@ INPUT:
2515
  lines.append(f"- {tag} SUPPORTED CLAIM: {claim}\n EVIDENCE: {evidence}")
2516
  return '\n'.join(lines)
2517
 
2518
- def build_citation_recovery_paragraph(self, items, section_name=''):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2519
  """Create one bounded literature paragraph from already planned evidence.
2520
 
2521
  This is invoked only when semantic validation removed too many planned
2522
  citations from a unit. It never invents new sources or study facts.
2523
  """
2524
- selected = [item for item in (items or []) if item.get('tag') and item.get('claim') and item.get('evidence')][:5]
 
 
 
 
2525
  if not selected:
2526
  return ''
2527
  plan_text = self.format_claim_evidence_plan(selected)
@@ -2788,13 +2991,14 @@ Return exactly:
2788
  for technical_attempt in range(technical_retries):
2789
  self._check_cancelled()
2790
  try:
 
2791
  response = self.call_deepseek(
2792
  system_prompt,
2793
  user_prompt,
2794
- max_tokens=8000,
2795
  temp=0.0,
2796
  model_role="reasoning",
2797
- thinking=True,
2798
  allow_fallback=False,
2799
  )
2800
  parsed = self._extract_json_from_text(response)
@@ -2991,7 +3195,7 @@ Return exactly:
2991
  coverage = overlap / max(1, len(claim_terms))
2992
  return coverage, overlap
2993
 
2994
- def _local_citation_recovery(self, text, section_name=''):
2995
  """Conservative deterministic recovery for technical audit failures.
2996
 
2997
  This routine never keeps the first citation merely because it appeared
@@ -3022,7 +3226,7 @@ Return exactly:
3022
  elif len(valid) == 1:
3023
  selected_tag = valid[0]
3024
  coverage, overlap = self._citation_support_score(sentence, selected_tag)
3025
- if overlap >= 2 and coverage >= 0.12:
3026
  self._append_citation_audit(
3027
  section_name,
3028
  'retain_lexically_supported_source',
@@ -3048,7 +3252,7 @@ Return exactly:
3048
  reverse=True,
3049
  )
3050
  coverage, overlap, _, selected_tag = scored[0]
3051
- if overlap >= 2 and coverage >= 0.12:
3052
  revised = re.sub(
3053
  r'\s*[\[(]?CITE-\d+(?:\s*[,;]\s*CITE-\d+)*[\])]?\s*',
3054
  ' ',
@@ -3187,7 +3391,7 @@ Return exactly:
3187
  error.code,
3188
  error.safe_detail,
3189
  )
3190
- value = self._local_citation_recovery(value, section_name)
3191
  except Exception as error:
3192
  if bool(self.citation_semantic_audit.get('fail_closed', False)):
3193
  raise
@@ -3196,7 +3400,7 @@ Return exactly:
3196
  section_name,
3197
  self._safe_source_error(error),
3198
  )
3199
- value = self._local_citation_recovery(value, section_name)
3200
  elif violations:
3201
  value = self._local_citation_recovery(value, section_name)
3202
  if self.find_multi_citation_violations(value):
 
2515
  lines.append(f"- {tag} SUPPORTED CLAIM: {claim}\n EVIDENCE: {evidence}")
2516
  return '\n'.join(lines)
2517
 
2518
+ _CITATION_RECOVERY_GENERIC_TERMS = frozenset({
2519
+ 'study','studies','research','data','analysis','analyses','model','models','learning',
2520
+ 'system','systems','method','methods','result','results','technology','technologies',
2521
+ 'education','educational','artificial','intelligence','application','applications',
2522
+ 'use','uses','used','using','user','users','approach','approaches','framework','frameworks',
2523
+ })
2524
+
2525
+ @classmethod
2526
+ def _citation_recovery_overlap(cls, section_name, section_context, claim, evidence):
2527
+ anchor_terms = cls._citation_terms(f"{section_name} {section_context}")
2528
+ candidate_terms = cls._citation_terms(f"{claim} {evidence}")
2529
+ overlap_terms = anchor_terms & candidate_terms
2530
+ distinctive = overlap_terms - cls._CITATION_RECOVERY_GENERIC_TERMS
2531
+ return len(overlap_terms), len(distinctive)
2532
+
2533
+ @classmethod
2534
+ def _citation_recovery_item_relevant(cls, item, section_name='', section_context=''):
2535
+ if not str(section_context or '').strip():
2536
+ return True
2537
+ claim = str((item or {}).get('claim') or '')
2538
+ evidence = str((item or {}).get('evidence') or '')
2539
+ overlap, distinctive = cls._citation_recovery_overlap(
2540
+ section_name, section_context, claim, evidence
2541
+ )
2542
+ return distinctive >= 2 or (distinctive >= 1 and overlap >= 3)
2543
+
2544
+ @classmethod
2545
+ def _best_contextual_evidence_claim(cls, evidence, section_name='', section_context=''):
2546
+ raw = re.sub(r'\s+', ' ', str(evidence or '')).strip()
2547
+ if not raw:
2548
+ return ''
2549
+ abstract = raw.split('Abstract evidence:', 1)[-1].strip() if 'Abstract evidence:' in raw else raw
2550
+ sentences = [part.strip() for part in re.split(r'(?<=[.!?])\s+', abstract) if 8 <= len(part.split()) <= 55]
2551
+ ranked = []
2552
+ for sentence in sentences:
2553
+ overlap, distinctive = cls._citation_recovery_overlap(
2554
+ section_name, section_context, sentence, raw
2555
+ )
2556
+ if distinctive >= 2 or (distinctive >= 1 and overlap >= 3):
2557
+ ranked.append((distinctive, overlap, -len(sentence.split()), sentence))
2558
+ if not ranked:
2559
+ return ''
2560
+ ranked.sort(reverse=True)
2561
+ return ranked[0][3]
2562
+
2563
+ def select_contextual_citation_recovery_items(
2564
+ self, items, *, section_name='', section_context='', needed=1, excluded_tags=None,
2565
+ ):
2566
+ """Select only recovery citations that are actually about the current unit.
2567
+
2568
+ First consider the unit's planned-but-unused items. If those are not
2569
+ contextually relevant, draw from the already-retrieved evidence pool
2570
+ using an abstract sentence copied verbatim as the narrow recovery claim.
2571
+ This prevents quota-driven paragraphs about unrelated domains.
2572
+ """
2573
+ needed = max(0, int(needed or 0))
2574
+ if needed <= 0:
2575
+ return []
2576
+ excluded = {str(tag or '').strip() for tag in (excluded_tags or set()) if str(tag or '').strip()}
2577
+ ranked = []
2578
+ seen = set()
2579
+ for item in items or []:
2580
+ if not isinstance(item, dict):
2581
+ continue
2582
+ tag = str(item.get('tag') or '').strip()
2583
+ if not tag or tag in excluded or tag in seen:
2584
+ continue
2585
+ if not self._citation_recovery_item_relevant(item, section_name, section_context):
2586
+ continue
2587
+ overlap, distinctive = self._citation_recovery_overlap(
2588
+ section_name, section_context, item.get('claim'), item.get('evidence')
2589
+ )
2590
+ ranked.append((distinctive, overlap, -self._citation_tag_number(tag), dict(item)))
2591
+ seen.add(tag)
2592
+ ranked.sort(reverse=True, key=lambda row: (row[0], row[1], row[2]))
2593
+ selected = [row[3] for row in ranked[:needed]]
2594
+ if len(selected) >= needed:
2595
+ return selected
2596
+
2597
+ used = excluded | {str(item.get('tag') or '') for item in selected}
2598
+ pool = []
2599
+ for tag, meta in self.citation_map.items():
2600
+ if tag in used:
2601
+ continue
2602
+ evidence = re.sub(r'\s+', ' ', str(meta.get('evidence') or '')).strip()
2603
+ claim = self._best_contextual_evidence_claim(
2604
+ evidence, section_name=section_name, section_context=section_context
2605
+ )
2606
+ if not claim:
2607
+ continue
2608
+ overlap, distinctive = self._citation_recovery_overlap(
2609
+ section_name, section_context, claim, evidence
2610
+ )
2611
+ pool.append((distinctive, overlap, -self._citation_tag_number(tag), {
2612
+ 'tag': tag, 'claim': claim, 'evidence': evidence[:900],
2613
+ }))
2614
+ pool.sort(reverse=True, key=lambda row: (row[0], row[1], row[2]))
2615
+ for _distinctive, _overlap, _order, item in pool:
2616
+ selected.append(item)
2617
+ if len(selected) >= needed:
2618
+ break
2619
+ return selected
2620
+
2621
+ def fetch_contextual_jit_recovery_items(
2622
+ self, *, topic, section_name, section_context, needed=1, excluded_tags=None,
2623
+ ):
2624
+ """Bounded last-resort JIT top-up for a genuine section-specific citation gap.
2625
+
2626
+ At most one query is issued to each literature backend. New references
2627
+ are admitted only when title/abstract evidence has distinctive lexical
2628
+ overlap with the current manuscript unit; otherwise the final >=50 gate
2629
+ is allowed to fail rather than padding the manuscript with irrelevant
2630
+ literature.
2631
+ """
2632
+ needed = max(0, int(needed or 0))
2633
+ if needed <= 0:
2634
+ return []
2635
+ excluded = {str(tag or '').strip() for tag in (excluded_tags or set()) if str(tag or '').strip()}
2636
+ context_words = [
2637
+ term for term in sorted(self._citation_terms(section_context))
2638
+ if term not in self._CITATION_RECOVERY_GENERIC_TERMS
2639
+ ][:12]
2640
+ prompt = (
2641
+ "Generate exactly one concise English academic search query of 3-5 terms. "
2642
+ "It must target the named manuscript unit and must not introduce a different application domain. "
2643
+ "Return only the query.\n"
2644
+ f"Topic: {str(topic or '')[:300]}\nSection: {section_name}\n"
2645
+ f"Context terms: {' '.join(context_words)}"
2646
+ )
2647
+ try:
2648
+ query = self.call_deepseek(
2649
+ 'You are a precise academic librarian.', prompt, max_tokens=50, temp=0.1,
2650
+ model_role='fast', thinking=False, allow_fallback=False,
2651
+ ).strip(' "\'')
2652
+ query = self._normalize_search_query(query)
2653
+ except Exception as exc:
2654
+ logging.warning('Contextual citation JIT query generation failed: section=%s error=%s', section_name, type(exc).__name__)
2655
+ return []
2656
+ if not query:
2657
+ return []
2658
+ logging.info('Contextual citation JIT: section=%s query=%r needed=%s', section_name, query, needed)
2659
+ results = []
2660
+ try:
2661
+ with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
2662
+ futures = [
2663
+ executor.submit(self._s2_worker, query, 0),
2664
+ executor.submit(self._oa_worker, query, 1, 'language:en,type:article,is_paratext:false'),
2665
+ ]
2666
+ for future in concurrent.futures.as_completed(futures):
2667
+ self._check_cancelled()
2668
+ results.extend(future.result())
2669
+ except Exception as exc:
2670
+ logging.warning('Contextual citation JIT retrieval failed: section=%s error=%s', section_name, type(exc).__name__)
2671
+ return []
2672
+
2673
+ existing_keys = {
2674
+ key for metadata in self.citation_map.values() for key in self._reference_keys(metadata)
2675
+ }
2676
+ candidates = []
2677
+ staged_keys = set()
2678
+ for res in results:
2679
+ keys = self._reference_keys(res)
2680
+ if keys and (any(key in existing_keys for key in keys) or any(key in staged_keys for key in keys)):
2681
+ continue
2682
+ evidence = re.sub(r'\s+', ' ', str(res.get('rag_text') or '')).strip()
2683
+ claim = self._best_contextual_evidence_claim(
2684
+ evidence, section_name=section_name, section_context=section_context
2685
+ )
2686
+ if not claim:
2687
+ continue
2688
+ overlap, distinctive = self._citation_recovery_overlap(
2689
+ section_name, section_context, claim, evidence
2690
+ )
2691
+ candidates.append((distinctive, overlap, res, claim, evidence, keys))
2692
+ staged_keys.update(keys)
2693
+ candidates.sort(reverse=True, key=lambda row: (row[0], row[1]))
2694
+ selected = []
2695
+ with self.lock:
2696
+ for _distinctive, _overlap, res, claim, evidence, keys in candidates:
2697
+ tag = f"[CITE-{len(self.citation_map)+1:02d}]"
2698
+ if tag in excluded:
2699
+ continue
2700
+ self.citation_map[tag] = {
2701
+ 'in_text': res.get('in_text', ''),
2702
+ 'ref': res.get('ref', ''),
2703
+ 'id': keys[0] if keys else ('title_year', res.get('title'), res.get('year')),
2704
+ 'doi': res.get('doi', ''),
2705
+ 'external_id': res.get('external_id', ''),
2706
+ 'title': res.get('title', ''),
2707
+ 'year': res.get('year', ''),
2708
+ 'evidence': evidence,
2709
+ 'sources': [res.get('source', 'jit')],
2710
+ }
2711
+ selected.append({'tag': tag, 'claim': claim, 'evidence': evidence[:900]})
2712
+ if len(selected) >= needed:
2713
+ break
2714
+ logging.info('Contextual citation JIT accepted: section=%s accepted=%s/%s', section_name, len(selected), needed)
2715
+ return selected
2716
+
2717
+ def build_citation_recovery_paragraph(self, items, section_name='', section_context=''):
2718
  """Create one bounded literature paragraph from already planned evidence.
2719
 
2720
  This is invoked only when semantic validation removed too many planned
2721
  citations from a unit. It never invents new sources or study facts.
2722
  """
2723
+ selected = [
2724
+ item for item in (items or [])
2725
+ if item.get('tag') and item.get('claim') and item.get('evidence')
2726
+ and self._citation_recovery_item_relevant(item, section_name, section_context)
2727
+ ][:5]
2728
  if not selected:
2729
  return ''
2730
  plan_text = self.format_claim_evidence_plan(selected)
 
2991
  for technical_attempt in range(technical_retries):
2992
  self._check_cancelled()
2993
  try:
2994
+ compact_retry = technical_attempt > 0
2995
  response = self.call_deepseek(
2996
  system_prompt,
2997
  user_prompt,
2998
+ max_tokens=3200 if compact_retry else 8000,
2999
  temp=0.0,
3000
  model_role="reasoning",
3001
+ thinking=not compact_retry,
3002
  allow_fallback=False,
3003
  )
3004
  parsed = self._extract_json_from_text(response)
 
3195
  coverage = overlap / max(1, len(claim_terms))
3196
  return coverage, overlap
3197
 
3198
+ def _local_citation_recovery(self, text, section_name='', retain_lexical=True):
3199
  """Conservative deterministic recovery for technical audit failures.
3200
 
3201
  This routine never keeps the first citation merely because it appeared
 
3226
  elif len(valid) == 1:
3227
  selected_tag = valid[0]
3228
  coverage, overlap = self._citation_support_score(sentence, selected_tag)
3229
+ if retain_lexical and overlap >= 2 and coverage >= 0.12:
3230
  self._append_citation_audit(
3231
  section_name,
3232
  'retain_lexically_supported_source',
 
3252
  reverse=True,
3253
  )
3254
  coverage, overlap, _, selected_tag = scored[0]
3255
+ if retain_lexical and overlap >= 2 and coverage >= 0.12:
3256
  revised = re.sub(
3257
  r'\s*[\[(]?CITE-\d+(?:\s*[,;]\s*CITE-\d+)*[\])]?\s*',
3258
  ' ',
 
3391
  error.code,
3392
  error.safe_detail,
3393
  )
3394
+ value = self._local_citation_recovery(value, section_name, retain_lexical=False)
3395
  except Exception as error:
3396
  if bool(self.citation_semantic_audit.get('fail_closed', False)):
3397
  raise
 
3400
  section_name,
3401
  self._safe_source_error(error),
3402
  )
3403
+ value = self._local_citation_recovery(value, section_name, retain_lexical=False)
3404
  elif violations:
3405
  value = self._local_citation_recovery(value, section_name)
3406
  if self.find_multi_citation_violations(value):
smartpaper_restore_bridge/frontend/index.html CHANGED
@@ -408,6 +408,13 @@
408
  "әдебиеттер","пайдаланылған әдебиеттер"
409
  ]);
410
 
 
 
 
 
 
 
 
411
  function cleanHeadingTitle(value) {
412
  return String(value || "").trim().replace(/^\d+(?:\.\d+){0,2}\s+/, "");
413
  }
@@ -420,11 +427,11 @@
420
  const match = line.match(/^(#{1,3})\s+(.+?)\s*$/);
421
  return match ? { index, level:match[1].length, title:cleanHeadingTitle(match[2]) } : null;
422
  }).filter(Boolean);
423
- const hasFrontMatter = headingRows.some(row => UNNUMBERED_HEADINGS.has(row.title.toLocaleLowerCase()));
424
  const firstH1 = headingRows.find(row => row.level === 1);
425
  for (const row of headingRows) {
426
  const key = row.title.toLocaleLowerCase();
427
- const isKnownUnnumbered = UNNUMBERED_HEADINGS.has(key);
428
  const isTitle = hasFrontMatter && firstH1 && row.index === firstH1.index && !isKnownUnnumbered;
429
  if (isKnownUnnumbered || isTitle) {
430
  lines[row.index] = row.title;
 
408
  "әдебиеттер","пайдаланылған әдебиеттер"
409
  ]);
410
 
411
+ const UNNUMBERED_HEADING_PREFIXES = ["appendix", "附录", "приложение", "қосымша"];
412
+
413
+ function isUnnumberedHeading(value) {
414
+ const key = String(value || "").trim().toLocaleLowerCase();
415
+ return UNNUMBERED_HEADINGS.has(key) || UNNUMBERED_HEADING_PREFIXES.some(prefix => key.startsWith(prefix));
416
+ }
417
+
418
  function cleanHeadingTitle(value) {
419
  return String(value || "").trim().replace(/^\d+(?:\.\d+){0,2}\s+/, "");
420
  }
 
427
  const match = line.match(/^(#{1,3})\s+(.+?)\s*$/);
428
  return match ? { index, level:match[1].length, title:cleanHeadingTitle(match[2]) } : null;
429
  }).filter(Boolean);
430
+ const hasFrontMatter = headingRows.some(row => isUnnumberedHeading(row.title));
431
  const firstH1 = headingRows.find(row => row.level === 1);
432
  for (const row of headingRows) {
433
  const key = row.title.toLocaleLowerCase();
434
+ const isKnownUnnumbered = isUnnumberedHeading(row.title);
435
  const isTitle = hasFrontMatter && firstH1 && row.index === firstH1.index && !isKnownUnnumbered;
436
  if (isKnownUnnumbered || isTitle) {
437
  lines[row.index] = row.title;
utils/docx_builder.py CHANGED
@@ -3,7 +3,11 @@ import io
3
  import mistune
4
  from bs4 import BeautifulSoup
5
  from docx import Document
6
- from docx.shared import Pt
 
 
 
 
7
  from config import APP_CONFIG
8
 
9
 
@@ -123,24 +127,75 @@ class DocxRenderer(mistune.HTMLRenderer):
123
  def table(self, text):
124
  soup = BeautifulSoup(text, 'html.parser')
125
  rows = soup.find_all('tr')
126
- if not rows:
127
  return ''
128
-
129
  max_cols = max(len(row.find_all(['td', 'th'])) for row in rows)
130
- if max_cols == 0:
131
  return ''
132
-
 
 
 
 
 
 
 
 
 
 
 
 
 
133
  table = self.doc.add_table(rows=len(rows), cols=max_cols)
134
- try:
135
  table.style = 'Table Grid'
136
- except:
137
  pass
138
-
 
 
 
 
 
 
 
139
  for i, row in enumerate(rows):
140
  for j, cell in enumerate(row.find_all(['td', 'th'])):
141
- if j < max_cols:
142
- table.cell(i, j).text = cell.get_text(strip=True)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
143
  self.doc.add_paragraph()
 
 
 
 
 
144
  return ''
145
 
146
  def thematic_break(self):
 
3
  import mistune
4
  from bs4 import BeautifulSoup
5
  from docx import Document
6
+ from docx.shared import Pt, Inches
7
+ from docx.enum.section import WD_SECTION, WD_ORIENT
8
+ from docx.enum.table import WD_CELL_VERTICAL_ALIGNMENT
9
+ from docx.oxml import OxmlElement
10
+ from docx.oxml.ns import qn
11
  from config import APP_CONFIG
12
 
13
 
 
127
  def table(self, text):
128
  soup = BeautifulSoup(text, 'html.parser')
129
  rows = soup.find_all('tr')
130
+ if not rows:
131
  return ''
132
+
133
  max_cols = max(len(row.find_all(['td', 'th'])) for row in rows)
134
+ if max_cols == 0:
135
  return ''
136
+
137
+ header_labels = [cell.get_text(strip=True) for cell in rows[0].find_all(['td', 'th'])]
138
+ qualitative_headers = [
139
+ 'Higher-order Theme', 'Empirical Concept', 'Process Stage',
140
+ 'Case/Actor', 'Evidence Role', 'Representative Quote',
141
+ ]
142
+ qualitative_evidence_table = header_labels == qualitative_headers
143
+
144
+ if qualitative_evidence_table:
145
+ previous = self.doc.sections[-1]
146
+ landscape = self.doc.add_section(WD_SECTION.NEW_PAGE)
147
+ landscape.orientation = WD_ORIENT.LANDSCAPE
148
+ landscape.page_width, landscape.page_height = previous.page_height, previous.page_width
149
+
150
  table = self.doc.add_table(rows=len(rows), cols=max_cols)
151
+ try:
152
  table.style = 'Table Grid'
153
+ except:
154
  pass
155
+
156
+ if qualitative_evidence_table:
157
+ table.autofit = False
158
+ widths = [1.15, 1.10, 0.85, 0.85, 0.75, 3.80] if qualitative_evidence_table else []
159
+ if qualitative_evidence_table:
160
+ for grid_col, width in zip(table._tbl.tblGrid.gridCol_lst, widths):
161
+ grid_col.set(qn('w:w'), str(int(width * 1440)))
162
+
163
  for i, row in enumerate(rows):
164
  for j, cell in enumerate(row.find_all(['td', 'th'])):
165
+ if j >= max_cols:
166
+ continue
167
+ target = table.cell(i, j)
168
+ target.text = cell.get_text(strip=True)
169
+ if qualitative_evidence_table:
170
+ target.vertical_alignment = WD_CELL_VERTICAL_ALIGNMENT.TOP
171
+ target.width = Inches(widths[j])
172
+ tc_pr = target._tc.get_or_add_tcPr()
173
+ tc_w = tc_pr.first_child_found_in('w:tcW')
174
+ if tc_w is not None:
175
+ tc_w.set(qn('w:type'), 'dxa')
176
+ tc_w.set(qn('w:w'), str(int(widths[j] * 1440)))
177
+ for paragraph in target.paragraphs:
178
+ paragraph.paragraph_format.space_before = Pt(0)
179
+ paragraph.paragraph_format.space_after = Pt(0)
180
+ for run in paragraph.runs:
181
+ run.font.size = Pt(9)
182
+
183
+ if qualitative_evidence_table and table.rows:
184
+ header_pr = table.rows[0]._tr.get_or_add_trPr()
185
+ header = OxmlElement('w:tblHeader')
186
+ header.set(qn('w:val'), 'true')
187
+ header_pr.append(header)
188
+ for row in table.rows:
189
+ tr_pr = row._tr.get_or_add_trPr()
190
+ cant_split = OxmlElement('w:cantSplit')
191
+ tr_pr.append(cant_split)
192
+
193
  self.doc.add_paragraph()
194
+ if qualitative_evidence_table:
195
+ current = self.doc.sections[-1]
196
+ portrait = self.doc.add_section(WD_SECTION.NEW_PAGE)
197
+ portrait.orientation = WD_ORIENT.PORTRAIT
198
+ portrait.page_width, portrait.page_height = current.page_height, current.page_width
199
  return ''
200
 
201
  def thematic_break(self):