S0-R1-C5.5-R6-R8 Deterministic Citation Identity Compiler + Author-Year Disambiguation

core/pipeline_agent.py

@@ -1240,12 +1240,16 @@ class SSciPipelineAgent:
                             "in_text": in_text,
                             "ref": apa_full,
                             "id": title.lower().strip()[:50],
+                            "title": title,
+                            "year": str(year),
+                            "sources": ["user_core"],
                             "evidence": (
                                 f"Title: {title}. Venue: {paper.get('venue', 'Unknown')}. "
                                 f"Core thesis: {core_thesis}. Abstract: {abstract}"
                             ),
                         }
                     rag_context.append(f"{cite_tag} [USER CORE PAPER] Title: {title}. Core Thesis: {core_thesis}. Abstract: {abstract}")
+                self.compile_citation_identities()
                 logging.info(f"✅ [本地解析] 成功提取 {len(papers)} 篇本地文献元数据。")
                 return "\n".join(rag_context)
             except Exception as e: 
@@ -1856,6 +1860,293 @@ If the immutable hypothesis contract is non-empty, preserve its H-identifiers an
             )
         return body, tail
 
+    @staticmethod
+    def _citation_alpha_suffix(index):
+        """Return a deterministic lowercase alphabetic suffix: 0->a, 25->z, 26->aa."""
+        value = int(index)
+        if value < 0:
+            raise DeepSeekRequestError('Citation disambiguation suffix index must be non-negative.')
+        chars = []
+        while True:
+            value, remainder = divmod(value, 26)
+            chars.append(chr(ord('a') + remainder))
+            if value == 0:
+                break
+            value -= 1
+        return ''.join(reversed(chars))
+
+    @classmethod
+    def _citation_source_identity(cls, metadata):
+        """Return a provider-independent deterministic source identity for citation disambiguation."""
+        row = dict(metadata or {})
+        cluster_id = str(row.get('_citation_source_cluster_id') or '').strip()
+        if cluster_id:
+            return ('cluster', cluster_id)
+        doi = cls._normalize_doi(row.get('doi'))
+        if doi:
+            return ('doi', doi)
+        title = re.sub(r'[^a-z0-9]+', ' ', str(row.get('title') or '').casefold()).strip()
+        year = str(row.get('year') or '').strip().casefold()
+        if title:
+            return ('title_year', title, year)
+        ref = re.sub(r'\s+', ' ', str(row.get('_citation_base_ref') or row.get('ref') or '')).strip().casefold()
+        if ref:
+            return ('ref', ref)
+        external_id = str(row.get('external_id') or '').strip().casefold()
+        if external_id:
+            return ('external', external_id)
+        return ('unknown', repr(sorted((str(k), str(v)) for k, v in row.items() if k != 'evidence')))
+
+    @staticmethod
+    def _citation_marker_with_year_suffix(marker, suffix):
+        value = str(marker or '').strip()
+        suffix = str(suffix or '').strip().lower()
+        if not value or not suffix:
+            return value
+        # SmartPaper's reader-facing citation grammar is author-year. Only the
+        # terminal publication year is changed; author text remains byte-stable.
+        updated, count = re.subn(
+            r'\b((?:19|20)\d{2})(?:[a-z]+)?\b(?=[^()]*\)$)',
+            lambda m: f"{m.group(1)}{suffix}",
+            value,
+            count=1,
+            flags=re.I,
+        )
+        if count != 1:
+            raise DeepSeekRequestError(
+                f'Citation identity compiler could not suffix author-year marker: {value}'
+            )
+        return updated
+
+    @staticmethod
+    def _citation_reference_with_year_suffix(reference, year, suffix):
+        value = str(reference or '').strip()
+        year = str(year or '').strip()
+        suffix = str(suffix or '').strip().lower()
+        if not value or not suffix:
+            return value
+        if re.fullmatch(r'(?:19|20)\d{2}', year):
+            pattern = re.compile(rf'\({re.escape(year)}(?:[a-z]+)?\)')
+        else:
+            # Fallback to the first conventional four-digit reference year.
+            pattern = re.compile(r'\(((?:19|20)\d{2})(?:[a-z]+)?\)')
+        match = pattern.search(value)
+        if not match:
+            raise DeepSeekRequestError(
+                f'Citation identity compiler could not suffix reference year: {value[:180]}'
+            )
+        base_year = year if re.fullmatch(r'(?:19|20)\d{2}', year) else match.group(1)
+        return value[:match.start()] + f'({base_year}{suffix})' + value[match.end():]
+
+    def compile_citation_identities(self):
+        """Make every reader-facing author-year marker map to exactly one source.
+
+        R6-R8 authority rule:
+        - exact duplicate source identities may share one marker;
+        - distinct sources that collapse to the same author-year marker receive
+          deterministic year suffixes (a/b/c/...) before citation resolution;
+        - already-resolved/used markers are immutable. A late JIT source that
+          collides with one receives the next free suffix instead of renaming
+          prose that has already been sealed.
+
+        This compiler is deterministic and model-free. ZH/RU/KK inherit the
+        frozen English marker as a protected citation anchor.
+        """
+        with self.lock:
+            citation_map = self.citation_map or {}
+            if not citation_map:
+                return {}
+
+            # Preserve unsuffixed authority once so repeated compilation and
+            # checkpoint resume never accumulate suffixes.
+            for metadata in citation_map.values():
+                if not isinstance(metadata, dict):
+                    continue
+                metadata.setdefault('_citation_base_in_text', str(metadata.get('in_text') or '').strip())
+                metadata.setdefault('_citation_base_ref', str(metadata.get('ref') or '').strip())
+
+            used = set(self.used_citations or set())
+
+            # Collapse provider/user-core aliases before author-year collision
+            # handling. Two records are the same source when any authoritative
+            # DOI/external/title-year key overlaps. This mirrors retrieval merge
+            # semantics and prevents duplicate bibliographic records from being
+            # misclassified as same-author/same-year distinct works.
+            tags_in_order = sorted(citation_map, key=self._citation_tag_number)
+            parent = {tag: tag for tag in tags_in_order}
+
+            def find(tag):
+                while parent[tag] != tag:
+                    parent[tag] = parent[parent[tag]]
+                    tag = parent[tag]
+                return tag
+
+            def union(left, right):
+                a, b = find(left), find(right)
+                if a != b:
+                    parent[b] = a if self._citation_tag_number(a) <= self._citation_tag_number(b) else b
+                    if parent[b] == b:
+                        parent[a] = b
+
+            source_keys = {}
+            for tag in tags_in_order:
+                metadata = citation_map.get(tag) or {}
+                keys = set(self._reference_keys(metadata))
+                if not keys:
+                    ref_key = re.sub(
+                        r'\s+', ' ', str(metadata.get('_citation_base_ref') or metadata.get('ref') or '')
+                    ).strip().casefold()
+                    if ref_key:
+                        keys.add(('ref', ref_key))
+                source_keys[tag] = keys
+            for index, left in enumerate(tags_in_order):
+                if not source_keys[left]:
+                    continue
+                for right in tags_in_order[index + 1:]:
+                    if source_keys[left] & source_keys[right]:
+                        union(left, right)
+
+            components = {}
+            for tag in tags_in_order:
+                components.setdefault(find(tag), []).append(tag)
+            for component_tags in components.values():
+                all_keys = sorted({key for tag in component_tags for key in source_keys.get(tag, set())}, key=repr)
+                if all_keys:
+                    preferred = sorted(
+                        all_keys,
+                        key=lambda key: (
+                            {'doi': 0, 'title_year': 1, 'external': 2, 'ref': 3}.get(str(key[0]), 9),
+                            repr(key),
+                        ),
+                    )[0]
+                    cluster_id = hashlib.sha256(repr(preferred).encode('utf-8')).hexdigest()[:20]
+                else:
+                    cluster_id = hashlib.sha256('|'.join(component_tags).encode('utf-8')).hexdigest()[:20]
+                for tag in component_tags:
+                    (citation_map.get(tag) or {})['_citation_source_cluster_id'] = cluster_id
+
+                if len(component_tags) <= 1:
+                    continue
+                used_aliases = [tag for tag in component_tags if tag in used]
+                authority_tag = min(used_aliases or component_tags, key=self._citation_tag_number)
+                authority = citation_map.get(authority_tag) or {}
+                authority_marker = str(authority.get('in_text') or '').strip()
+                authority_ref = str(authority.get('ref') or '').strip()
+                authority_base_marker = str(authority.get('_citation_base_in_text') or authority_marker).strip()
+                authority_base_ref = str(authority.get('_citation_base_ref') or authority_ref).strip()
+                # If multiple aliases have already been rendered with different
+                # reader-facing identities, there is no tag-level information in
+                # sealed prose to repair safely. Preserve them for the final
+                # fail-closed legacy gate rather than guessing.
+                used_markers = {
+                    str((citation_map.get(tag) or {}).get('in_text') or '').strip()
+                    for tag in used_aliases
+                }
+                if len(used_markers) > 1:
+                    continue
+                for tag in component_tags:
+                    if tag in used_aliases:
+                        continue
+                    metadata = citation_map.get(tag) or {}
+                    metadata['in_text'] = authority_marker
+                    metadata['ref'] = authority_ref
+                    metadata['_citation_base_in_text'] = authority_base_marker
+                    metadata['_citation_base_ref'] = authority_base_ref
+                    metadata['citation_source_alias_of'] = authority_tag
+
+            groups = {}
+            for tag, metadata in citation_map.items():
+                if not isinstance(metadata, dict):
+                    continue
+                base_marker = str(metadata.get('_citation_base_in_text') or metadata.get('in_text') or '').strip()
+                base_ref = str(metadata.get('_citation_base_ref') or metadata.get('ref') or '').strip()
+                if not base_marker or not base_ref:
+                    continue
+                groups.setdefault(base_marker, []).append(str(tag))
+
+            changed = {}
+            for base_marker, tags in groups.items():
+                # Cluster duplicate provider records by canonical source identity.
+                clusters = {}
+                for tag in tags:
+                    metadata = citation_map.get(tag) or {}
+                    identity = self._citation_source_identity(metadata)
+                    clusters.setdefault(identity, []).append(tag)
+                if len(clusters) <= 1:
+                    continue
+
+                ordered = sorted(
+                    clusters.items(),
+                    key=lambda item: (
+                        re.sub(r'\s+', ' ', str((citation_map.get(item[1][0]) or {}).get('title') or '')).casefold(),
+                        repr(item[0]),
+                        min(self._citation_tag_number(tag) for tag in item[1]),
+                    ),
+                )
+                used_clusters = {
+                    identity for identity, cluster_tags in ordered if any(tag in used for tag in cluster_tags)
+                }
+
+                reserved_markers = {
+                    str((citation_map.get(tag) or {}).get('in_text') or '').strip()
+                    for tag in used
+                    if tag in tags and str((citation_map.get(tag) or {}).get('in_text') or '').strip()
+                }
+                next_suffix_index = 0
+                for identity, cluster_tags in ordered:
+                    cluster_is_used = identity in used_clusters
+                    if cluster_is_used:
+                        # Never rename a marker that may already exist in sealed prose.
+                        markers = {
+                            str((citation_map.get(tag) or {}).get('in_text') or '').strip()
+                            for tag in cluster_tags if tag in used
+                        }
+                        if len(markers) > 1:
+                            raise DeepSeekRequestError(
+                                f'Citation identity compiler found one used source with inconsistent markers: {sorted(markers)}'
+                            )
+                        reserved_markers.update(markers)
+                        continue
+
+                    # Before any citation is used, assign conventional a/b/c to
+                    # the complete collision set. After prose has been sealed,
+                    # only new/unresolved sources receive a suffix, preserving
+                    # previously rendered text byte-for-byte.
+                    while True:
+                        suffix = self._citation_alpha_suffix(next_suffix_index)
+                        next_suffix_index += 1
+                        candidate_marker = self._citation_marker_with_year_suffix(base_marker, suffix)
+                        if candidate_marker not in reserved_markers:
+                            break
+                    reserved_markers.add(candidate_marker)
+                    for tag in cluster_tags:
+                        metadata = citation_map.get(tag) or {}
+                        base_ref = str(metadata.get('_citation_base_ref') or metadata.get('ref') or '').strip()
+                        year = str(metadata.get('year') or '').strip()
+                        candidate_ref = self._citation_reference_with_year_suffix(base_ref, year, suffix)
+                        old_marker = str(metadata.get('in_text') or '').strip()
+                        old_ref = str(metadata.get('ref') or '').strip()
+                        metadata['in_text'] = candidate_marker
+                        metadata['ref'] = candidate_ref
+                        metadata['citation_disambiguation_suffix'] = suffix
+                        metadata['citation_identity_compiler_version'] = 'R6-R8-v1'
+                        if candidate_marker != old_marker or candidate_ref != old_ref:
+                            changed[tag] = {'from': old_marker, 'to': candidate_marker, 'suffix': suffix}
+
+                # If no member of the collision group has been used yet, all
+                # clusters must be suffixed. The loop above does that because
+                # used_clusters is empty. If one or more clusters are already
+                # used, unchanged used markers remain authoritative while new
+                # markers are made unique against them.
+
+            if changed:
+                logging.info(
+                    'Citation identity compiler disambiguated markers: count=%s changes=%s',
+                    len(changed),
+                    {tag: row['to'] for tag, row in sorted(changed.items(), key=lambda item: self._citation_tag_number(item[0]))},
+                )
+            return changed
+
     @staticmethod
     def _resolved_parenthetical_citations(text):
         """Return resolved parenthetical author-year citation markers in prose.
@@ -1907,13 +2198,12 @@ If the immutable hypothesis contract is non-empty, preserve its H-identifiers an
             if len(tags) == 1:
                 ledger.add(tags[0])
                 continue
-            distinct_refs = {
-                str((self.citation_map.get(tag) or {}).get('ref') or '').strip()
+            distinct_sources = {
+                self._citation_source_identity(self.citation_map.get(tag) or {})
                 for tag in tags
-                if str((self.citation_map.get(tag) or {}).get('ref') or '').strip()
             }
             previously_used = [tag for tag in tags if tag in set(self.used_citations or set())]
-            if len(distinct_refs) == 1:
+            if len(distinct_sources) == 1:
                 ledger.add(sorted(tags, key=self._citation_tag_number)[0])
             elif len(previously_used) == 1:
                 ledger.add(previously_used[0])
@@ -2099,6 +2389,7 @@ If the immutable hypothesis contract is non-empty, preserve its H-identifiers an
                             "evidence_lanes": [lane],
                         }
                         jit_rag_texts.append(f"{cite_tag} {res.get('rag_text', '')}")
+        self.compile_citation_identities()
         logging.info("METHOD_AUTHORITY retrieval complete | accepted=%s", len(jit_rag_texts))
         return "\n".join(jit_rag_texts)
 
@@ -2627,6 +2918,7 @@ Return ONLY a valid JSON array of strings."""
                 else:
                     metric['status'] = 'no_results'
                     metric['last_safe_error'] = no_result_code
+        self.compile_citation_identities()
         logging.info(
             'Reference search completed: unique=%s metrics=%s',
             len(self.citation_map),
@@ -3251,6 +3543,7 @@ INPUT:
                 selected.append({'tag': tag, 'claim': claim, 'evidence': evidence[:900]})
                 if len(selected) >= needed:
                     break
+        self.compile_citation_identities()
         logging.info('Contextual citation JIT accepted: section=%s accepted=%s/%s', section_name, len(selected), needed)
         return selected
 
@@ -9749,6 +10042,10 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
         return violations
 
     def resolve_citations(self, text, quarantine=False):
+        # R6-R8 invariant: author-year identity is compiled immediately before
+        # any raw [CITE-XX] tag becomes reader-facing prose. This prevents a
+        # late final-ledger collision even if a new source was admitted by JIT.
+        self.compile_citation_identities()
         if quarantine:
             text = re.sub(r'\s*\(?[A-Z][a-z]+ (?:& [A-Z][a-z]+ |et al\., )\d{4}\)?\s*', '', text)
             text = re.sub(r'\[?CITE-\d+(?:,\s*CITE-\d+)*\]?', '', text, flags=re.IGNORECASE)