S0-R1-C5.5-R6-R13 Claim-Bound Numeric Fact Recovery

#101
Files changed (1) hide show
  1. core/pipeline_agent.py +199 -37
core/pipeline_agent.py CHANGED
@@ -7077,16 +7077,27 @@ TEXT:
7077
  r'mechanisms?|groups?|stages?|steps?|waves?|models?|hypotheses?|dimensions?|factors?|items?|'
7078
  r'rounds?|phases?|samples?|studies?|analyses?|conditions?|paths?|constructs?|measures?|components?|'
7079
  r'categories?|levels?|questions?|options?|variables?|sections?|claims?|explanations?|participants?|teams?|'
7080
- r'orders?|points?|scales?|types?|classes?|periods?|years?|months?|weeks?|days?|trials?|arms?|effects?|generations?'
 
 
7081
  )
7082
  found = []
7083
  cardinal_words = '|'.join(cardinal)
7084
- # Academic attributive compounds are especially important because a
7085
- # source such as ``eleven-item scale`` naturally becomes ``11项量表``.
7086
- for match in re.finditer(rf'(?i)\b({cardinal_words})(?:\s+|-)({measurable_nouns})\b', text):
 
 
 
 
 
 
 
7087
  found.append(cardinal[match.group(1).casefold()])
7088
  ordinal_words = '|'.join(ordinal)
7089
- for match in re.finditer(rf'(?i)\b({ordinal_words})(?:\s+|-)({measurable_nouns})\b', text):
 
 
7090
  found.append(ordinal[match.group(1).casefold()])
7091
  for match in re.finditer(r'(?i)\b(twice|twofold|threefold|fourfold)\b', text):
7092
  found.append({'twice':'2','twofold':'2','threefold':'3','fourfold':'4'}[match.group(1).casefold()])
@@ -8944,6 +8955,45 @@ Translate headings and terminology naturally. Preserve standard variable abbrevi
8944
  )
8945
  restored[block_id] = translated_with_claims
8946
  continue
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
8947
  self._record_translation_batch_damage(target_lang, 'protected_or_claim_integrity')
8948
  logging.warning(
8949
  'Translation protected block recovery: block=SP-P%s error=%s; strict isolated retry.',
@@ -9222,17 +9272,23 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
9222
  return states
9223
 
9224
  @classmethod
9225
- def _restore_translation_claim_states(cls, source_blocks, claim_states, section_name=''):
9226
- """Reassemble marked blocks only when every frozen source claim has a hash-matched durable target."""
 
 
 
 
 
 
9227
  states = dict(claim_states or {})
9228
- if not states:
9229
- return {}, {}, False
9230
- translations = {}
9231
  pending = {}
 
9232
  for block_id, source_block in dict(source_blocks or {}).items():
9233
  source_claims = cls._translation_claim_units(source_block)
9234
  order = [item['claim_id'] for item in source_claims]
9235
  segments = {}
 
9236
  for item in source_claims:
9237
  claim_id = item['claim_id']
9238
  key = f'SP-P{block_id}-{claim_id}'
@@ -9240,26 +9296,90 @@ Each code must be one of: {sorted(allowed)}. Return one or more specific dimensi
9240
  expected_hash = hashlib.sha256(str(item['text']).encode('utf-8')).hexdigest()
9241
  expected_semantic_hash = cls._source_semantic_claim_frame(item['text'], section_name).get('frame_hash', '')
9242
  expected_object_hash = cls._translation_semantic_object_frame(item['text'], section_name).get('object_hash', '')
9243
- if not isinstance(row, dict) or row.get('source_hash') != expected_hash or not str(row.get('target_text') or '').strip():
9244
- return {}, {}, False
9245
- # R6-R9: durable target claims are semantic-object bound. Any
9246
- # pre-v1 object state is target-only stale and must be rebuilt;
9247
- # sealed English remains untouched.
9248
- if str(row.get('semantic_object_version') or '') != cls.TRANSLATION_SEMANTIC_OBJECT_VERSION:
9249
- return {}, {}, False
9250
- if str(row.get('semantic_object_hash') or '') != expected_object_hash:
9251
- return {}, {}, False
9252
- stored_semantic_hash = str(row.get('semantic_frame_hash') or '')
9253
- if stored_semantic_hash and stored_semantic_hash != expected_semantic_hash:
9254
- return {}, {}, False
 
 
9255
  segments[claim_id] = str(row.get('target_text') or '').strip()
 
9256
  if str(row.get('qa_status') or '') == 'repair_pending':
9257
- codes = [str(code or '').strip().upper() for code in (row.get('issue_codes') or []) if str(code or '').strip()]
 
 
 
9258
  if codes:
9259
  pending.setdefault(str(block_id), {})[str(claim_id)] = codes[:5]
9260
- translations[str(block_id)] = cls._compose_translation_claim_segments(order, segments)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9261
  return translations, pending, True
9262
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9263
  def _adjudicate_persistent_translation_issues(
9264
  self, sources, translations, issues, target_lang, glossary=None, section_name=''
9265
  ):
@@ -9738,20 +9858,62 @@ Decision must be CONFIRMED_BLOCKING or NOT_CONFIRMED. evidence_code must be one
9738
  sum(len(v or {}) for v in quality_issues.values()),
9739
  )
9740
  else:
9741
- pending = list(pending_blocks)
9742
- while pending:
9743
- batch, pending = self._take_adaptive_translation_batch(
9744
- pending, target_lang, token_budget
9745
- )
9746
- damage_before = self._translation_batch_damage_score(target_lang)
9747
- translations.update(self._translate_marked_batch_with_recovery(batch, system_prompt, target_lang))
9748
- if self._translation_batch_damage_score(target_lang) == damage_before and damage_before > 0:
9749
- self._relax_translation_batch_damage(target_lang)
9750
- emit_stage('qa')
9751
- quality_issues = self._audit_translation_blocks_with_retry(
9752
- source_blocks, translations, target_lang, frozen_glossary,
9753
- section_name=section_name or 'frozen section', max_attempts=2,
9754
  )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
9755
  self._last_translation_claim_states = self._snapshot_translation_claim_states(
9756
  source_blocks, translations, quality_issues, self._translation_claim_qa_evidence,
9757
  self._translation_claim_repair_meta, section_name=section_name or 'frozen section',
 
7077
  r'mechanisms?|groups?|stages?|steps?|waves?|models?|hypotheses?|dimensions?|factors?|items?|'
7078
  r'rounds?|phases?|samples?|studies?|analyses?|conditions?|paths?|constructs?|measures?|components?|'
7079
  r'categories?|levels?|questions?|options?|variables?|sections?|claims?|explanations?|participants?|teams?|'
7080
+ r'orders?|points?|scales?|types?|classes?|periods?|years?|months?|weeks?|days?|trials?|arms?|effects?|generations?|'
7081
+ r'associations?|coefficients?|estimates?|relationships?|correlations?|regressions?|predictors?|outcomes?|mediators?|'
7082
+ r'indicators?|loadings?|intervals?|tests?|diagnostics?'
7083
  )
7084
  found = []
7085
  cardinal_words = '|'.join(cardinal)
7086
+ # R6-R13: quantity ownership is source-semantic, not surface-adjacent.
7087
+ # Allow a bounded chain of ordinary academic modifiers between the
7088
+ # English number word and a countable/statistical semantic head, e.g.
7089
+ # ``two constituent associations`` or ``two statistically significant
7090
+ # direct paths``. Punctuation cannot be crossed and one source quantity
7091
+ # still authorizes only one matching target numeral occurrence.
7092
+ bounded_modifiers = r'(?:(?:[A-Za-z]+(?:-[A-Za-z]+)?)\s+){0,4}'
7093
+ for match in re.finditer(
7094
+ rf'(?i)\b({cardinal_words})(?:\s+|-){bounded_modifiers}({measurable_nouns})\b', text
7095
+ ):
7096
  found.append(cardinal[match.group(1).casefold()])
7097
  ordinal_words = '|'.join(ordinal)
7098
+ for match in re.finditer(
7099
+ rf'(?i)\b({ordinal_words})(?:\s+|-){bounded_modifiers}({measurable_nouns})\b', text
7100
+ ):
7101
  found.append(ordinal[match.group(1).casefold()])
7102
  for match in re.finditer(r'(?i)\b(twice|twofold|threefold|fourfold)\b', text):
7103
  found.append({'twice':'2','twofold':'2','threefold':'3','fourfold':'4'}[match.group(1).casefold()])
 
8955
  )
8956
  restored[block_id] = translated_with_claims
8957
  continue
8958
+ if self._is_translation_unowned_numeric_error(error):
8959
+ claim_match = re.search(r'claim=(C\d{6})', str(error))
8960
+ source_claims = dict(claim_sources_by_block.get(block_id) or {})
8961
+ claim_id = claim_match.group(1) if claim_match else ''
8962
+ if claim_id and claim_id in source_claims:
8963
+ logging.warning(
8964
+ 'Translation batch numeric claim-local recovery: block=SP-P%s claim=%s error=%s',
8965
+ block_id, claim_id, self._safe_source_error(error),
8966
+ )
8967
+ claim_order = list(source_claims)
8968
+ _order, segments = self._parse_translation_claim_segments(
8969
+ parsed[block_id], claim_order
8970
+ )
8971
+ sibling_snapshot = {
8972
+ key: str(value or '') for key, value in segments.items() if key != claim_id
8973
+ }
8974
+ segments[claim_id] = self._translate_single_claim_with_recovery(
8975
+ block_id, claim_id, source_claims[claim_id],
8976
+ system_prompt, target_lang, ['FACT_INTEGRITY_NUMERIC'],
8977
+ current_target_claim=segments.get(claim_id),
8978
+ )
8979
+ # The failed claim is the only model-owned repair surface.
8980
+ # Sibling claims must remain byte-stable.
8981
+ if any(str(segments.get(key) or '') != value for key, value in sibling_snapshot.items()):
8982
+ raise DeepSeekRequestError(
8983
+ f'Translation numeric claim-local recovery mutated sibling claim in SP-P{block_id}.'
8984
+ )
8985
+ repaired = self._compose_translation_claim_segments(claim_order, segments)
8986
+ translated_with_claims = self._restore_translation_tokens(
8987
+ repaired, protected_maps.get(block_id, {}), f'SP-P{block_id}', target_lang,
8988
+ claim_sources=source_claims, keep_claim_markers=True,
8989
+ glossary=getattr(self, '_active_translation_glossary', {}),
8990
+ )
8991
+ clean_translation = self._strip_translation_claim_markers(translated_with_claims)
8992
+ self._validate_translation_block_protected(
8993
+ block_id, source_by_id[block_id], clean_translation, target_lang
8994
+ )
8995
+ restored[block_id] = translated_with_claims
8996
+ continue
8997
  self._record_translation_batch_damage(target_lang, 'protected_or_claim_integrity')
8998
  logging.warning(
8999
  'Translation protected block recovery: block=SP-P%s error=%s; strict isolated retry.',
 
9272
  return states
9273
 
9274
  @classmethod
9275
+ def _restore_translation_claim_state_segments(cls, source_blocks, claim_states, section_name=''):
9276
+ """Return every hash-valid durable claim independently of missing siblings.
9277
+
9278
+ R6-R13 keeps passed claims reusable even when another claim/block is
9279
+ missing or stale. This helper never invents a target: it returns only
9280
+ source-hash + semantic-object matched rows and explicitly records the
9281
+ missing claim IDs that still require translation.
9282
+ """
9283
  states = dict(claim_states or {})
9284
+ blocks = {}
 
 
9285
  pending = {}
9286
+ reusable_claims = 0
9287
  for block_id, source_block in dict(source_blocks or {}).items():
9288
  source_claims = cls._translation_claim_units(source_block)
9289
  order = [item['claim_id'] for item in source_claims]
9290
  segments = {}
9291
+ missing = []
9292
  for item in source_claims:
9293
  claim_id = item['claim_id']
9294
  key = f'SP-P{block_id}-{claim_id}'
 
9296
  expected_hash = hashlib.sha256(str(item['text']).encode('utf-8')).hexdigest()
9297
  expected_semantic_hash = cls._source_semantic_claim_frame(item['text'], section_name).get('frame_hash', '')
9298
  expected_object_hash = cls._translation_semantic_object_frame(item['text'], section_name).get('object_hash', '')
9299
+ valid = bool(
9300
+ isinstance(row, dict)
9301
+ and row.get('source_hash') == expected_hash
9302
+ and str(row.get('target_text') or '').strip()
9303
+ and str(row.get('semantic_object_version') or '') == cls.TRANSLATION_SEMANTIC_OBJECT_VERSION
9304
+ and str(row.get('semantic_object_hash') or '') == expected_object_hash
9305
+ )
9306
+ if valid:
9307
+ stored_semantic_hash = str(row.get('semantic_frame_hash') or '')
9308
+ if stored_semantic_hash and stored_semantic_hash != expected_semantic_hash:
9309
+ valid = False
9310
+ if not valid:
9311
+ missing.append(claim_id)
9312
+ continue
9313
  segments[claim_id] = str(row.get('target_text') or '').strip()
9314
+ reusable_claims += 1
9315
  if str(row.get('qa_status') or '') == 'repair_pending':
9316
+ codes = [
9317
+ str(code or '').strip().upper() for code in (row.get('issue_codes') or [])
9318
+ if str(code or '').strip()
9319
+ ]
9320
  if codes:
9321
  pending.setdefault(str(block_id), {})[str(claim_id)] = codes[:5]
9322
+ blocks[str(block_id)] = {
9323
+ 'order': order, 'segments': segments, 'missing': missing,
9324
+ 'source_claims': {item['claim_id']: item['text'] for item in source_claims},
9325
+ }
9326
+ return blocks, pending, reusable_claims
9327
+
9328
+ @classmethod
9329
+ def _restore_translation_claim_states(cls, source_blocks, claim_states, section_name=''):
9330
+ """Compatibility wrapper: full resume succeeds only when every claim is durable."""
9331
+ if not dict(claim_states or {}):
9332
+ return {}, {}, False
9333
+ blocks, pending, reusable_claims = cls._restore_translation_claim_state_segments(
9334
+ source_blocks, claim_states, section_name=section_name
9335
+ )
9336
+ if reusable_claims <= 0 or any(row.get('missing') for row in blocks.values()):
9337
+ return {}, {}, False
9338
+ translations = {
9339
+ str(block_id): cls._compose_translation_claim_segments(row['order'], row['segments'])
9340
+ for block_id, row in blocks.items()
9341
+ }
9342
  return translations, pending, True
9343
 
9344
+ def _audit_translation_claim_subset_with_retry(
9345
+ self, source_blocks, translations, target_lang, glossary, claim_pairs, section_name=''
9346
+ ):
9347
+ """QA only newly translated claims; durable passed siblings never re-enter the model."""
9348
+ synthetic_sources = {}
9349
+ synthetic_targets = {}
9350
+ reverse = {}
9351
+ for index, (block_id, claim_id) in enumerate(claim_pairs, start=1):
9352
+ source_claims = {
9353
+ item['claim_id']: item['text'] for item in self._translation_claim_units(source_blocks[block_id])
9354
+ }
9355
+ order = list(source_claims)
9356
+ _order, target_claims = self._parse_translation_claim_segments(translations[block_id], order)
9357
+ synthetic_id = f'{900000 + index:06d}'
9358
+ synthetic_sources[synthetic_id] = source_claims[claim_id]
9359
+ synthetic_targets[synthetic_id] = self._compose_translation_claim_segments(
9360
+ ['C000001'], {'C000001': target_claims[claim_id]}
9361
+ )
9362
+ reverse[(synthetic_id, 'C000001')] = (str(block_id), str(claim_id))
9363
+ if not synthetic_sources:
9364
+ return {}
9365
+ synthetic_issues = self._audit_translation_blocks_with_retry(
9366
+ synthetic_sources, synthetic_targets, target_lang, glossary,
9367
+ section_name=section_name or 'frozen section', max_attempts=2,
9368
+ )
9369
+ issues = {}
9370
+ for synthetic_id, claim_map in dict(synthetic_issues or {}).items():
9371
+ for synthetic_claim_id, codes in dict(claim_map or {}).items():
9372
+ original = reverse.get((str(synthetic_id), str(synthetic_claim_id)))
9373
+ if not original:
9374
+ continue
9375
+ block_id, claim_id = original
9376
+ synthetic_evidence_key = (str(synthetic_id), str(synthetic_claim_id))
9377
+ evidence = dict(self._translation_claim_qa_evidence.pop(synthetic_evidence_key, {}) or {})
9378
+ if evidence:
9379
+ self._translation_claim_qa_evidence[(block_id, claim_id)] = evidence
9380
+ issues.setdefault(block_id, {})[claim_id] = list(codes or [])
9381
+ return issues
9382
+
9383
  def _adjudicate_persistent_translation_issues(
9384
  self, sources, translations, issues, target_lang, glossary=None, section_name=''
9385
  ):
 
9858
  sum(len(v or {}) for v in quality_issues.values()),
9859
  )
9860
  else:
9861
+ partial_blocks, partial_pending, reusable_claims = self._restore_translation_claim_state_segments(
9862
+ source_blocks, claim_states, section_name=section_name or 'frozen section'
 
 
 
 
 
 
 
 
 
 
 
9863
  )
9864
+ if reusable_claims > 0:
9865
+ translations = {}
9866
+ quality_issues = {
9867
+ str(block_id): {str(claim_id): list(codes or []) for claim_id, codes in dict(claim_map or {}).items()}
9868
+ for block_id, claim_map in dict(partial_pending or {}).items()
9869
+ }
9870
+ newly_translated_pairs = []
9871
+ for block_id, row in partial_blocks.items():
9872
+ segments = dict(row.get('segments') or {})
9873
+ source_claims = dict(row.get('source_claims') or {})
9874
+ for claim_id in list(row.get('missing') or []):
9875
+ segments[claim_id] = self._translate_single_claim_with_recovery(
9876
+ block_id, claim_id, source_claims[claim_id], system_prompt, target_lang,
9877
+ ['DURABLE_RESUME_MISSING'], repair_strategy='SOURCE_REGENERATE',
9878
+ )
9879
+ newly_translated_pairs.append((block_id, claim_id))
9880
+ translations[block_id] = self._compose_translation_claim_segments(row['order'], segments)
9881
+ if newly_translated_pairs:
9882
+ emit_stage('qa')
9883
+ fresh_issues = self._audit_translation_claim_subset_with_retry(
9884
+ source_blocks, translations, target_lang, frozen_glossary,
9885
+ newly_translated_pairs, section_name=section_name or 'frozen section',
9886
+ )
9887
+ for block_id, claim_map in dict(fresh_issues or {}).items():
9888
+ for claim_id, codes in dict(claim_map or {}).items():
9889
+ quality_issues.setdefault(block_id, {})[claim_id] = list(codes or [])
9890
+ logging.info(
9891
+ 'Translation partial durable claim resume | section=%s reused=%s translated_missing=%s pending=%s',
9892
+ section_name or 'section', reusable_claims, len(newly_translated_pairs),
9893
+ sum(len(v or {}) for v in quality_issues.values()),
9894
+ )
9895
+ else:
9896
+ pending = list(pending_blocks)
9897
+ while pending:
9898
+ batch, pending = self._take_adaptive_translation_batch(
9899
+ pending, target_lang, token_budget
9900
+ )
9901
+ damage_before = self._translation_batch_damage_score(target_lang)
9902
+ translations.update(self._translate_marked_batch_with_recovery(batch, system_prompt, target_lang))
9903
+ # R6-R13: persist every completed claim batch immediately.
9904
+ # If a later block fails deterministically, already-passed
9905
+ # claims survive the runner's repair_pending checkpoint.
9906
+ self._last_translation_claim_states = self._snapshot_translation_claim_states(
9907
+ source_blocks, translations, {}, self._translation_claim_qa_evidence,
9908
+ self._translation_claim_repair_meta, section_name=section_name or 'frozen section',
9909
+ )
9910
+ if self._translation_batch_damage_score(target_lang) == damage_before and damage_before > 0:
9911
+ self._relax_translation_batch_damage(target_lang)
9912
+ emit_stage('qa')
9913
+ quality_issues = self._audit_translation_blocks_with_retry(
9914
+ source_blocks, translations, target_lang, frozen_glossary,
9915
+ section_name=section_name or 'frozen section', max_attempts=2,
9916
+ )
9917
  self._last_translation_claim_states = self._snapshot_translation_claim_states(
9918
  source_blocks, translations, quality_issues, self._translation_claim_qa_evidence,
9919
  self._translation_claim_repair_meta, section_name=section_name or 'frozen section',