Spaces:
Running
Running
Step 4A-6-C-R3.2.8.5-R2 Source-Derived Numeric Equivalence + Claim-Local Fact Recovery
#25
by GuXiaopeng - opened
- .gitattributes +0 -1
- __pycache__/app.cpython-313.pyc +0 -0
- __pycache__/config.cpython-313.pyc +0 -0
- __pycache__/desktop.ini +0 -2
- __pycache__/i18n.cpython-313.pyc +0 -0
- __pycache__/managed_auth.cpython-313.pyc +0 -0
- __pycache__/task_executor.cpython-313.pyc +0 -0
- core/__pycache__/checkpoint_store.cpython-313.pyc +0 -0
- core/__pycache__/desktop.ini +0 -2
- core/__pycache__/generation_runner.cpython-313.pyc +0 -0
- core/__pycache__/meta_extractor.cpython-313.pyc +0 -0
- core/__pycache__/pipeline_agent.cpython-313.pyc +0 -3
- core/__pycache__/production_guard.cpython-313.pyc +0 -0
- core/__pycache__/qualitative_engine.cpython-313.pyc +0 -0
- core/__pycache__/quantitative_engine.cpython-313.pyc +0 -0
- core/__pycache__/rag_engine.cpython-313.pyc +0 -0
- core/generation_runner.py +1 -1
- core/pipeline_agent.py +65 -12
- smartpaper_restore_bridge/__pycache__/__init__.cpython-313.pyc +0 -0
- smartpaper_restore_bridge/__pycache__/desktop.ini +0 -2
- utils/__pycache__/desktop.ini +0 -2
- utils/__pycache__/docx_builder.cpython-313.pyc +0 -0
- utils/__pycache__/file_parser.cpython-313.pyc +0 -0
.gitattributes
CHANGED
|
@@ -33,4 +33,3 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
| 36 |
-
core/__pycache__/pipeline_agent.cpython-313.pyc filter=lfs diff=lfs merge=lfs -text
|
|
|
|
| 33 |
*.zip filter=lfs diff=lfs merge=lfs -text
|
| 34 |
*.zst filter=lfs diff=lfs merge=lfs -text
|
| 35 |
*tfevents* filter=lfs diff=lfs merge=lfs -text
|
|
|
__pycache__/app.cpython-313.pyc
DELETED
|
Binary file (61.3 kB)
|
|
|
__pycache__/config.cpython-313.pyc
DELETED
|
Binary file (31.9 kB)
|
|
|
__pycache__/desktop.ini
DELETED
|
@@ -1,2 +0,0 @@
|
|
| 1 |
-
[.ShellClassInfo]
|
| 2 |
-
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
|
|
|
|
|
|
|
|
__pycache__/i18n.cpython-313.pyc
DELETED
|
Binary file (76.6 kB)
|
|
|
__pycache__/managed_auth.cpython-313.pyc
DELETED
|
Binary file (31.1 kB)
|
|
|
__pycache__/task_executor.cpython-313.pyc
DELETED
|
Binary file (33.7 kB)
|
|
|
core/__pycache__/checkpoint_store.cpython-313.pyc
DELETED
|
Binary file (4.64 kB)
|
|
|
core/__pycache__/desktop.ini
DELETED
|
@@ -1,2 +0,0 @@
|
|
| 1 |
-
[.ShellClassInfo]
|
| 2 |
-
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
|
|
|
|
|
|
|
|
core/__pycache__/generation_runner.cpython-313.pyc
DELETED
|
Binary file (70.2 kB)
|
|
|
core/__pycache__/meta_extractor.cpython-313.pyc
DELETED
|
Binary file (3.44 kB)
|
|
|
core/__pycache__/pipeline_agent.cpython-313.pyc
DELETED
|
@@ -1,3 +0,0 @@
|
|
| 1 |
-
version https://git-lfs.github.com/spec/v1
|
| 2 |
-
oid sha256:5ff1e1c8ae14f93f19fb2d92015df442e858db9e6cf8deccff732c4e0c8b1973
|
| 3 |
-
size 273806
|
|
|
|
|
|
|
|
|
|
|
|
core/__pycache__/production_guard.cpython-313.pyc
DELETED
|
Binary file (13.9 kB)
|
|
|
core/__pycache__/qualitative_engine.cpython-313.pyc
DELETED
|
Binary file (15.3 kB)
|
|
|
core/__pycache__/quantitative_engine.cpython-313.pyc
DELETED
|
Binary file (24.4 kB)
|
|
|
core/__pycache__/rag_engine.cpython-313.pyc
DELETED
|
Binary file (4.37 kB)
|
|
|
core/generation_runner.py
CHANGED
|
@@ -33,7 +33,7 @@ ProgressCallback = Callable[..., None]
|
|
| 33 |
MIN_FINAL_REFERENCES = 50
|
| 34 |
LITERATURE_EVIDENCE_TARGET = 75
|
| 35 |
CITATION_PLAN_TARGET = 70
|
| 36 |
-
TRANSLATION_CONTRACT_VERSION = 'claim-bound-
|
| 37 |
|
| 38 |
|
| 39 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
|
|
|
| 33 |
MIN_FINAL_REFERENCES = 50
|
| 34 |
LITERATURE_EVIDENCE_TARGET = 75
|
| 35 |
CITATION_PLAN_TARGET = 70
|
| 36 |
+
TRANSLATION_CONTRACT_VERSION = 'claim-bound-v2'
|
| 37 |
|
| 38 |
|
| 39 |
def _clean_trailing_references(text, keywords=("reference", "references", "reference list", "参考文献")):
|
core/pipeline_agent.py
CHANGED
|
@@ -56,7 +56,7 @@ class CitationAuditError(DeepSeekRequestError):
|
|
| 56 |
|
| 57 |
|
| 58 |
class SSciPipelineAgent:
|
| 59 |
-
TRANSLATION_CONTRACT_VERSION = 'claim-bound-
|
| 60 |
|
| 61 |
def __init__(self, ds_key, s2_key=""):
|
| 62 |
self.ds_key = str(ds_key or "").strip()
|
|
@@ -3589,16 +3589,21 @@ Output only the repaired section."""
|
|
| 3589 |
|
| 3590 |
@classmethod
|
| 3591 |
def _translation_lexical_quantifier_candidates(cls, value):
|
| 3592 |
-
"""High-confidence source
|
| 3593 |
|
| 3594 |
-
|
| 3595 |
-
|
| 3596 |
-
|
|
|
|
|
|
|
| 3597 |
"""
|
| 3598 |
text = str(value or '')
|
| 3599 |
cardinal = {
|
| 3600 |
'zero':'0','one':'1','two':'2','three':'3','four':'4','five':'5','six':'6',
|
| 3601 |
'seven':'7','eight':'8','nine':'9','ten':'10','eleven':'11','twelve':'12',
|
|
|
|
|
|
|
|
|
|
| 3602 |
}
|
| 3603 |
ordinal = {
|
| 3604 |
'first':'1','second':'2','third':'3','fourth':'4','fifth':'5','sixth':'6',
|
|
@@ -3607,22 +3612,42 @@ Output only the repaired section."""
|
|
| 3607 |
measurable_nouns = (
|
| 3608 |
r'mechanisms?|groups?|stages?|steps?|waves?|models?|hypotheses?|dimensions?|factors?|items?|'
|
| 3609 |
r'rounds?|phases?|samples?|studies?|analyses?|conditions?|paths?|constructs?|measures?|components?|'
|
| 3610 |
-
r'categories?|levels?|questions?|options?|variables?|sections?|claims?|explanations?|participants?|teams?'
|
|
|
|
| 3611 |
)
|
| 3612 |
found = []
|
| 3613 |
cardinal_words = '|'.join(cardinal)
|
| 3614 |
-
|
|
|
|
|
|
|
| 3615 |
found.append(cardinal[match.group(1).casefold()])
|
| 3616 |
ordinal_words = '|'.join(ordinal)
|
| 3617 |
-
for match in re.finditer(rf'(?i)\b({ordinal_words})\s+({measurable_nouns})\b', text):
|
| 3618 |
found.append(ordinal[match.group(1).casefold()])
|
| 3619 |
for match in re.finditer(r'(?i)\b(twice|twofold|threefold|fourfold)\b', text):
|
| 3620 |
found.append({'twice':'2','twofold':'2','threefold':'3','fourfold':'4'}[match.group(1).casefold()])
|
| 3621 |
# one-to-one is a relational form with two explicit quantitative roles.
|
| 3622 |
if re.search(r'(?i)\bone[- ]to[- ]one\b', text):
|
| 3623 |
found.extend(['1', '1'])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3624 |
return Counter(found)
|
| 3625 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3626 |
@classmethod
|
| 3627 |
def _protect_translation_tokens(cls, value, claim_id='C000001'):
|
| 3628 |
"""Protect one claim while keeping factual values in a private server ledger."""
|
|
@@ -4322,10 +4347,37 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4322 |
temp=0.0, model_role='translation', thinking=False, allow_fallback=False,
|
| 4323 |
)
|
| 4324 |
parsed = self._parse_marked_translation(result, [block_id])
|
| 4325 |
-
|
| 4326 |
-
|
| 4327 |
-
|
| 4328 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 4329 |
clean = self._strip_translation_claim_markers(restored)
|
| 4330 |
self._validate_translation_block_protected(block_id, source_claim, clean, target_lang)
|
| 4331 |
_order, segments = self._parse_translation_claim_segments(restored, [claim_id])
|
|
@@ -4473,6 +4525,7 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
|
|
| 4473 |
message = str(error)
|
| 4474 |
if (
|
| 4475 |
message.startswith('Translation claim-local QA failed')
|
|
|
|
| 4476 |
or message.startswith('Translation alignment')
|
| 4477 |
or message.startswith('Translation changed ')
|
| 4478 |
or message.startswith('Translation protected ')
|
|
|
|
| 56 |
|
| 57 |
|
| 58 |
class SSciPipelineAgent:
|
| 59 |
+
TRANSLATION_CONTRACT_VERSION = 'claim-bound-v2'
|
| 60 |
|
| 61 |
def __init__(self, ds_key, s2_key=""):
|
| 62 |
self.ds_key = str(ds_key or "").strip()
|
|
|
|
| 3589 |
|
| 3590 |
@classmethod
|
| 3591 |
def _translation_lexical_quantifier_candidates(cls, value):
|
| 3592 |
+
"""High-confidence source numeric semantics that may render as Arabic digits.
|
| 3593 |
|
| 3594 |
+
R3.2.8.5-R2 keeps attribution claim-local and source-derived. It accepts
|
| 3595 |
+
ordinary and hyphenated academic quantity forms (``eleven-item``,
|
| 3596 |
+
``three-wave``, ``second-order``), relational forms, and English month
|
| 3597 |
+
names that are commonly rendered numerically in Chinese. These values
|
| 3598 |
+
are private validation evidence and are never exposed to the model.
|
| 3599 |
"""
|
| 3600 |
text = str(value or '')
|
| 3601 |
cardinal = {
|
| 3602 |
'zero':'0','one':'1','two':'2','three':'3','four':'4','five':'5','six':'6',
|
| 3603 |
'seven':'7','eight':'8','nine':'9','ten':'10','eleven':'11','twelve':'12',
|
| 3604 |
+
'thirteen':'13','fourteen':'14','fifteen':'15','sixteen':'16','seventeen':'17',
|
| 3605 |
+
'eighteen':'18','nineteen':'19','twenty':'20','thirty':'30','forty':'40',
|
| 3606 |
+
'fifty':'50','sixty':'60','seventy':'70','eighty':'80','ninety':'90',
|
| 3607 |
}
|
| 3608 |
ordinal = {
|
| 3609 |
'first':'1','second':'2','third':'3','fourth':'4','fifth':'5','sixth':'6',
|
|
|
|
| 3612 |
measurable_nouns = (
|
| 3613 |
r'mechanisms?|groups?|stages?|steps?|waves?|models?|hypotheses?|dimensions?|factors?|items?|'
|
| 3614 |
r'rounds?|phases?|samples?|studies?|analyses?|conditions?|paths?|constructs?|measures?|components?|'
|
| 3615 |
+
r'categories?|levels?|questions?|options?|variables?|sections?|claims?|explanations?|participants?|teams?|'
|
| 3616 |
+
r'orders?|points?|scales?|types?|classes?|periods?|years?|months?|weeks?|days?|trials?|arms?|effects?|generations?'
|
| 3617 |
)
|
| 3618 |
found = []
|
| 3619 |
cardinal_words = '|'.join(cardinal)
|
| 3620 |
+
# Academic attributive compounds are especially important because a
|
| 3621 |
+
# source such as ``eleven-item scale`` naturally becomes ``11项量表``.
|
| 3622 |
+
for match in re.finditer(rf'(?i)\b({cardinal_words})(?:\s+|-)({measurable_nouns})\b', text):
|
| 3623 |
found.append(cardinal[match.group(1).casefold()])
|
| 3624 |
ordinal_words = '|'.join(ordinal)
|
| 3625 |
+
for match in re.finditer(rf'(?i)\b({ordinal_words})(?:\s+|-)({measurable_nouns})\b', text):
|
| 3626 |
found.append(ordinal[match.group(1).casefold()])
|
| 3627 |
for match in re.finditer(r'(?i)\b(twice|twofold|threefold|fourfold)\b', text):
|
| 3628 |
found.append({'twice':'2','twofold':'2','threefold':'3','fourfold':'4'}[match.group(1).casefold()])
|
| 3629 |
# one-to-one is a relational form with two explicit quantitative roles.
|
| 3630 |
if re.search(r'(?i)\bone[- ]to[- ]one\b', text):
|
| 3631 |
found.extend(['1', '1'])
|
| 3632 |
+
# Month names can legitimately surface as Arabic month numbers in
|
| 3633 |
+
# Chinese even when the source contains no digit (e.g. November -> 11月).
|
| 3634 |
+
months = {
|
| 3635 |
+
'january':'1','february':'2','march':'3','april':'4','may':'5','june':'6',
|
| 3636 |
+
'july':'7','august':'8','september':'9','october':'10','november':'11','december':'12',
|
| 3637 |
+
}
|
| 3638 |
+
for month, number in months.items():
|
| 3639 |
+
if re.search(rf'(?i)\b{month}\b', text):
|
| 3640 |
+
found.append(number)
|
| 3641 |
return Counter(found)
|
| 3642 |
|
| 3643 |
+
@staticmethod
|
| 3644 |
+
def _is_translation_unowned_numeric_error(error):
|
| 3645 |
+
message = str(error or '')
|
| 3646 |
+
return (
|
| 3647 |
+
message.startswith('Translation introduced unowned factual content')
|
| 3648 |
+
and "{'numeric':" in message
|
| 3649 |
+
)
|
| 3650 |
+
|
| 3651 |
@classmethod
|
| 3652 |
def _protect_translation_tokens(cls, value, claim_id='C000001'):
|
| 3653 |
"""Protect one claim while keeping factual values in a private server ledger."""
|
|
|
|
| 4347 |
temp=0.0, model_role='translation', thinking=False, allow_fallback=False,
|
| 4348 |
)
|
| 4349 |
parsed = self._parse_marked_translation(result, [block_id])
|
| 4350 |
+
try:
|
| 4351 |
+
restored = self._restore_translation_tokens(
|
| 4352 |
+
parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
|
| 4353 |
+
claim_sources={claim_id: source_claim}, keep_claim_markers=True,
|
| 4354 |
+
)
|
| 4355 |
+
except DeepSeekRequestError as error:
|
| 4356 |
+
if not self._is_translation_unowned_numeric_error(error):
|
| 4357 |
+
raise
|
| 4358 |
+
logging.warning(
|
| 4359 |
+
'Translation claim-local numeric recovery: block=SP-P%s claim=%s error=%s',
|
| 4360 |
+
block_id, claim_id, self._safe_source_error(error),
|
| 4361 |
+
)
|
| 4362 |
+
numeric_prompt = system_prompt + (
|
| 4363 |
+
'\nNUMERIC FACT RECOVERY FOR ONE CLAIM. Repair only this claim. Preserve every SP-P, SP-C, and SP-T marker byte-for-byte. '
|
| 4364 |
+
'Do not introduce any Arabic digit, percentage, sample size, scale point, model number, date number, or other numeric surface unless it is represented by an SP-T marker or is directly and unambiguously expressed by an English number-word, ordinal, fold expression, relational quantity, or month name in this source claim. '
|
| 4365 |
+
'If a lexical quantity is not protected by SP-T and you are uncertain, express it lexically in the target language rather than inventing an Arabic digit. '
|
| 4366 |
+
'Do not add, remove, merge, or reinterpret claims. Return only the marked translated claim.'
|
| 4367 |
+
+ '\n<SEMANTIC_ANCHOR_MANIFEST>\n'
|
| 4368 |
+
+ json.dumps({block_id: safe_manifest}, ensure_ascii=False)
|
| 4369 |
+
+ '\n</SEMANTIC_ANCHOR_MANIFEST>'
|
| 4370 |
+
)
|
| 4371 |
+
retry_result = self.call_deepseek(
|
| 4372 |
+
numeric_prompt, marked_source,
|
| 4373 |
+
max_tokens=max(1200, min(2600, self._count_tokens(marked_source) * 2 + 900)),
|
| 4374 |
+
temp=0.0, model_role='translation', thinking=False, allow_fallback=False,
|
| 4375 |
+
)
|
| 4376 |
+
retry_parsed = self._parse_marked_translation(retry_result, [block_id])
|
| 4377 |
+
restored = self._restore_translation_tokens(
|
| 4378 |
+
retry_parsed[block_id], mapping, f'SP-P{block_id}', target_lang,
|
| 4379 |
+
claim_sources={claim_id: source_claim}, keep_claim_markers=True,
|
| 4380 |
+
)
|
| 4381 |
clean = self._strip_translation_claim_markers(restored)
|
| 4382 |
self._validate_translation_block_protected(block_id, source_claim, clean, target_lang)
|
| 4383 |
_order, segments = self._parse_translation_claim_segments(restored, [claim_id])
|
|
|
|
| 4525 |
message = str(error)
|
| 4526 |
if (
|
| 4527 |
message.startswith('Translation claim-local QA failed')
|
| 4528 |
+
or message.startswith('Translation introduced unowned factual content')
|
| 4529 |
or message.startswith('Translation alignment')
|
| 4530 |
or message.startswith('Translation changed ')
|
| 4531 |
or message.startswith('Translation protected ')
|
smartpaper_restore_bridge/__pycache__/__init__.cpython-313.pyc
DELETED
|
Binary file (3.14 kB)
|
|
|
smartpaper_restore_bridge/__pycache__/desktop.ini
DELETED
|
@@ -1,2 +0,0 @@
|
|
| 1 |
-
[.ShellClassInfo]
|
| 2 |
-
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
|
|
|
|
|
|
|
|
utils/__pycache__/desktop.ini
DELETED
|
@@ -1,2 +0,0 @@
|
|
| 1 |
-
[.ShellClassInfo]
|
| 2 |
-
IconResource=C:\Program Files\Google\Drive File Stream\129.0.1.0\GoogleDriveFS.exe,27
|
|
|
|
|
|
|
|
|
utils/__pycache__/docx_builder.cpython-313.pyc
DELETED
|
Binary file (9.4 kB)
|
|
|
utils/__pycache__/file_parser.cpython-313.pyc
DELETED
|
Binary file (8.03 kB)
|
|
|