fix: structured OCR follows raw tables/text for soil and leaf

Prefer header-mapped Document AI/Excel tables over shifted positional text parses. Sliding-window alignment fixes leaf N/P column shifts and soil leading junk. Flatten nested leaf Dry Matter structures; preserve SP Lab sample IDs.

modules/upload.py

@@ -536,76 +536,37 @@ def show_ocr_preview(file, file_type: str, container_type: str) -> None:
                             samples_data = {}
                             
                             for sample in samples:
-                                sample_id = sample.get('sample_id', sample.get('Sample No.', sample.get('Lab No.', '')))
-                                if not sample_id:
+                                sid, sample_data = _flatten_ocr_sample(sample, container_type)
+                                if not sid or not sample_data:
                                     continue
-                                
-                                # Ensure sample_id is a string
-                                sample_id = str(sample_id) if sample_id is not None else ''
-                                
-                                # For leaf data, use 'L' or 'P' prefix format
+                                sample_id = sid
+
+                                # Preserve SP Lab IDs (S218/25, P220/25); normalize farm IDs only
                                 if container_type == 'leaf':
-                                    # Keep original format if it starts with P (SP Lab format)
-                                    if sample_id.startswith('P') and '/' in sample_id:
-                                        pass  # Keep as is (e.g., P220/25)
+                                    if re.match(r'^P\d{1,4}/\d{2}$', sample_id, re.I):
+                                        sample_id = sample_id[0].upper() + sample_id[1:]
                                     elif sample_id.startswith(('L', 'P')) and sample_id[1:].isdigit():
-                                        sample_num = int(sample_id[1:])
-                                        sample_id = f'L{sample_num:03d}'
+                                        sample_id = f'L{int(sample_id[1:]):03d}'
                                     elif sample_id.isdigit():
                                         sample_id = f'L{int(sample_id):03d}'
-                                    else:
-                                        # Try to extract number from sample_id
-                                        num_match = re.search(r'(\d+)', sample_id)
-                                        if num_match:
-                                            sample_id = f'L{int(num_match.group(1)):03d}'
-                                        else:
-                                            continue
                                 else:
-                                    # For soil data, use 'S' prefix
-                                    if sample_id.startswith('S') and sample_id[1:].isdigit():
-                                        sample_num = int(sample_id[1:])
-                                        sample_id = f'S{sample_num:03d}'
+                                    if re.match(r'^S\d{1,4}/\d{2}$', sample_id, re.I):
+                                        sample_id = sample_id[0].upper() + sample_id[1:]
+                                    elif sample_id.startswith('S') and sample_id[1:].isdigit():
+                                        sample_id = f'S{int(sample_id[1:]):03d}'
                                     elif sample_id.isdigit():
                                         sample_id = f'S{int(sample_id):03d}'
-                                    else:
-                                        # Try to extract number from sample_id
-                                        num_match = re.search(r'(\d+)', sample_id)
-                                        if num_match:
-                                            sample_id = f'S{int(num_match.group(1)):03d}'
-                                        else:
-                                            continue
-                                
-                                # Extract sample data (exclude metadata keys)
-                                sample_data = {}
-                                for key, value in sample.items():
-                                    if key not in ['sample_id', 'Sample No.', 'Lab No.', 'sample_no', 'lab_no'] and value is not None:
-                                        try:
-                                            # Try to convert to float if numeric
-                                            if isinstance(value, (int, float)):
-                                                sample_data[key] = float(value)
-                                            elif isinstance(value, str):
-                                                # Try to parse numeric strings
-                                                cleaned = re.sub(r'[^\d\.\-]', '', value)
-                                                if cleaned:
-                                                    sample_data[key] = float(cleaned)
-                                                else:
-                                                    sample_data[key] = value
-                                            else:
-                                                sample_data[key] = value
-                                        except (ValueError, TypeError):
-                                            sample_data[key] = value
-                                
-                                if sample_data:
-                                    samples_data[sample_id] = sample_data
+
+                                samples_data[sample_id] = sample_data
                             
-                            # Determine container key
+                            # Determine container key (SP Lab if any ID has / or SP/LAB markers)
                             if container_type == 'soil':
-                                if any('SP' in str(sid).upper() or 'LAB' in str(sid).upper() for sid in samples_data.keys()):
+                                if any('/' in str(sid) or 'SP' in str(sid).upper() or 'LAB' in str(sid).upper() for sid in samples_data.keys()):
                                     container_key = "SP_Lab_Test_Report"
                                 else:
                                     container_key = "Farm_Soil_Test_Data"
                             else:
-                                if any('P' in str(sid) and '/' in str(sid) for sid in samples_data.keys()):
+                                if any(('/' in str(sid) and str(sid).upper().startswith('P')) for sid in samples_data.keys()):
                                     container_key = "SP_Lab_Test_Report"
                                 else:
                                     container_key = "Farm_Leaf_Test_Data"
@@ -1348,42 +1309,177 @@ def _parse_numeric_token(token: str):
         return None
 
 
-def _align_soil_numbers_to_params(numbers: list, params: list, sample_id: str = '') -> dict:
-    """Map a number list onto soil/leaf params.
-
-    SP Lab OCR often injects leading junk before pH (lab serial, year, page no.).
-    For soil, pH is first and must fall in a realistic range (~2.5–9.5).
+def _score_soil_window(window: list) -> float:
+    """Higher = more realistic oil-palm soil lab row (pH, N, OrgC, P, cations, CEC)."""
+    if len(window) < 5:
+        return -100.0
+    score = 0.0
+    ph = window[0]
+    # pH must look like pH
+    if 3.0 <= ph <= 7.5:
+        score += 5.0
+    elif 2.5 <= ph <= 9.0:
+        score += 2.0
+    else:
+        score -= 8.0
+    # N (%) typically 0.03–0.40
+    if len(window) > 1:
+        n = window[1]
+        if 0.02 <= n <= 0.5:
+            score += 3.0
+        elif n > 2:
+            score -= 4.0
+    # Org C (%) typically 0.2–5
+    if len(window) > 2:
+        oc = window[2]
+        if 0.1 <= oc <= 6.0:
+            score += 2.0
+        elif oc > 20:
+            score -= 3.0
+    # Total P mg/kg often 5–400
+    if len(window) > 3:
+        tp = window[3]
+        if 1 <= tp <= 800:
+            score += 2.0
+        elif 0 < tp < 1:
+            score -= 2.0
+    # Avail P
+    if len(window) > 4:
+        ap = window[4]
+        if 0 <= ap <= 80:
+            score += 1.5
+    # Exch cations small
+    for idx in (5, 6, 7):
+        if len(window) > idx and 0 <= window[idx] <= 15:
+            score += 0.5
+    # CEC usually > exch cations and 1–40
+    if len(window) > 8:
+        cec = window[8]
+        if 1 <= cec <= 50:
+            score += 2.0
+        if len(window) > 5 and cec >= window[5]:
+            score += 0.5
+    return score
+
+
+def _score_leaf_window(window: list) -> float:
+    """Higher = more realistic oil-palm leaf lab row (N,P,K,Mg,Ca,B,Cu,Zn)."""
+    if len(window) < 5:
+        return -100.0
+    score = 0.0
+    n = window[0]
+    # Leaf N % typically ~1.8–3.2
+    if 1.6 <= n <= 3.5:
+        score += 6.0
+    elif 1.2 <= n <= 4.0:
+        score += 3.0
+    else:
+        score -= 8.0
+    # P % typically 0.08–0.25
+    if len(window) > 1:
+        p = window[1]
+        if 0.08 <= p <= 0.30:
+            score += 4.0
+        elif p > 1.0:
+            score -= 5.0  # shifted N sitting in P slot
+    # K % typically 0.4–1.5
+    if len(window) > 2:
+        k = window[2]
+        if 0.35 <= k <= 2.0:
+            score += 2.0
+        elif k > 5:
+            score -= 2.0
+    # Mg, Ca %
+    if len(window) > 3 and 0.05 <= window[3] <= 0.8:
+        score += 1.0
+    if len(window) > 4 and 0.2 <= window[4] <= 1.5:
+        score += 1.0
+    # B mg/kg typically 8–30; Cu 1–10; Zn 5–30
+    if len(window) > 5:
+        b = window[5]
+        if 5 <= b <= 50:
+            score += 3.0
+        elif b < 1:
+            score -= 3.0  # Ca sitting in B slot
+    if len(window) > 6 and 0 <= window[6] <= 15:
+        score += 1.0
+    if len(window) > 7 and 3 <= window[7] <= 40:
+        score += 1.0
+    return score
+
+
+def _align_numbers_to_params(numbers: list, params: list, sample_id: str = '', kind: str = 'soil') -> dict:
+    """Map OCR number sequence onto params by sliding-window fitness (no hardcoded samples).
+
+    Handles leading junk (lab serial, row index, year) that otherwise shifts all columns —
+    e.g. leaf N=2.13 becoming P because a leading 1.0 was treated as N.
     """
     if not numbers or not params:
         return {}
 
     nums = list(numbers)
-    # Drop numbers that are clearly the lab/year digits already in the sample ID
+    # Drop large digits that repeat the sample lab number (e.g. 218 from S218/25)
     if sample_id:
         id_digits = re.findall(r'\d+', sample_id)
-        while nums and any(abs(nums[0] - float(d)) < 1e-9 for d in id_digits if float(d) > 9):
-            # Only strip large ID parts (e.g. 218), not small year-like 25 if it could be Avail P
+        while nums and any(abs(nums[0] - float(d)) < 1e-9 for d in id_digits if float(d) >= 10):
             if nums[0] >= 10:
                 nums = nums[1:]
             else:
                 break
 
-    # Soil: first param is pH — skip leading values outside soil pH range
-    if params and params[0] == 'pH':
-        start = 0
-        while start < len(nums) and not (2.5 <= nums[start] <= 9.5):
-            start += 1
-        if start < len(nums):
-            nums = nums[start:]
-        # If still no plausible pH, keep original (better partial than empty)
+    n_params = len(params)
+    scorer = _score_soil_window if kind == 'soil' else _score_leaf_window
+
+    if len(nums) < 3:
+        return {params[i]: nums[i] for i in range(len(nums))}
+
+    best_score = -1e9
+    best_window = nums[:n_params]
+
+    # Try every alignment window (extra leading/trailing OCR numbers)
+    max_start = max(0, len(nums) - n_params)
+    for start in range(0, max_start + 1):
+        window = nums[start:start + n_params]
+        if len(window) < min(5, n_params):
+            continue
+        # pad short windows
+        while len(window) < n_params:
+            window = window  # no pad for scoring; scorer handles short
+            break
+        s = scorer(window)
+        # prefer earlier windows slightly when scores tie
+        s -= start * 0.01
+        if s > best_score:
+            best_score = s
+            best_window = window
+
+    # If best score is still terrible, fall back to first plausible anchor
+    if kind == 'soil' and best_score < 0:
+        for start, val in enumerate(nums):
+            if 2.5 <= val <= 9.5:
+                best_window = nums[start:start + n_params]
+                break
+    elif kind == 'leaf' and best_score < 0:
+        for start, val in enumerate(nums):
+            if 1.5 <= val <= 3.5:
+                # next should look like P if available
+                if start + 1 < len(nums) and nums[start + 1] > 1.0:
+                    continue
+                best_window = nums[start:start + n_params]
+                break
 
     sample_data = {}
     for i, param in enumerate(params):
-        if i < len(nums):
-            sample_data[param] = nums[i]
+        if i < len(best_window):
+            sample_data[param] = best_window[i]
     return sample_data
 
 
+def _align_soil_numbers_to_params(numbers: list, params: list, sample_id: str = '') -> dict:
+    """Backward-compatible soil aligner."""
+    return _align_numbers_to_params(numbers, params, sample_id, kind='soil')
+
+
 def _extract_sp_lab_samples_from_text(text: str, sample_prefix: str, params: list) -> dict:
     """
     Parse SP Lab samples (e.g. S218/25 or P220/25) from real OCR text.
@@ -1540,6 +1636,8 @@ def _extract_sp_lab_samples_from_text(text: str, sample_prefix: str, params: lis
         else:
             return transposed
 
+    kind = 'soil' if is_soil else 'leaf'
+
     def _parse_between_ids(source_text: str) -> dict:
         """Split source on sample IDs and map numbers after each ID to params."""
         found = {}
@@ -1554,7 +1652,7 @@ def _extract_sp_lab_samples_from_text(text: str, sample_prefix: str, params: lis
             tokens = re.split(r'\s+', chunk.strip()) if chunk.strip() else []
             numbers = _tokens_to_numbers(tokens)
             if len(numbers) >= min_needed:
-                sample_data = _align_soil_numbers_to_params(numbers, params, sample_id)
+                sample_data = _align_numbers_to_params(numbers, params, sample_id, kind=kind)
                 if sample_data:
                     found[sample_id] = sample_data
         return found
@@ -1729,17 +1827,10 @@ def format_raw_text_as_structured_json(raw_text: str, container_type: str) -> di
                 for match in matches[:12]:
                     sample_num, sample_values = match
                     sample_id = f"S{sample_num.zfill(3)}"
-                    numbers = re.findall(r'(\d+\.?\d*)', sample_values)
+                    numbers = [float(x) for x in re.findall(r'(\d+\.?\d*)', sample_values)]
 
                     if len(numbers) >= 3:
-                        sample_data = {}
-                        for i, param in enumerate(soil_params):
-                            if i < len(numbers):
-                                try:
-                                    sample_data[param] = float(numbers[i])
-                                except (ValueError, TypeError):
-                                    sample_data[param] = 0.0
-
+                        sample_data = _align_numbers_to_params(numbers, soil_params, sample_id, kind='soil')
                         if sample_data and any(v != 0.0 for v in sample_data.values()):
                             samples_data[sample_id] = sample_data
 
@@ -1751,36 +1842,23 @@ def format_raw_text_as_structured_json(raw_text: str, container_type: str) -> di
                 for match in matches[:12]:
                     sample_num, sample_values = match
                     sample_id = f"S{sample_num.zfill(3)}"
-                    numbers = re.findall(r'(\d+\.?\d*)', sample_values)
+                    numbers = [float(x) for x in re.findall(r'(\d+\.?\d*)', sample_values)]
 
                     if len(numbers) >= 3:
-                        sample_data = {}
-                        for i, param in enumerate(soil_params):
-                            if i < len(numbers):
-                                try:
-                                    sample_data[param] = float(numbers[i])
-                                except (ValueError, TypeError):
-                                    sample_data[param] = 0.0
-
+                        sample_data = _align_numbers_to_params(numbers, soil_params, sample_id, kind='soil')
                         if sample_data and any(v != 0.0 for v in sample_data.values()):
                             samples_data[sample_id] = sample_data
 
             # Strategy 4: Chunk all numbers into groups of 9 (farm tabular dump only)
             if not samples_data and not is_sp_soil:
-                all_numbers = re.findall(r'(\d+\.?\d*)', all_text)
+                all_numbers = [float(x) for x in re.findall(r'(\d+\.?\d*)', all_text)]
                 if len(all_numbers) >= 27:
                     num_samples = min(12, len(all_numbers) // 9)
                     for i in range(num_samples):
                         sample_id = f"S{i+1:03d}"
-                        sample_data = {}
                         start_idx = i * 9
-                        for j, param in enumerate(soil_params):
-                            if start_idx + j < len(all_numbers):
-                                try:
-                                    sample_data[param] = float(all_numbers[start_idx + j])
-                                except (ValueError, TypeError):
-                                    sample_data[param] = 0.0
-
+                        chunk = all_numbers[start_idx:start_idx + 9]
+                        sample_data = _align_numbers_to_params(chunk, soil_params, sample_id, kind='soil')
                         if sample_data and any(v != 0.0 for v in sample_data.values()):
                             samples_data[sample_id] = sample_data
 
@@ -1847,36 +1925,23 @@ def format_raw_text_as_structured_json(raw_text: str, container_type: str) -> di
                     break
                 sample_num, sample_values = match
                 sample_id = f"L{str(sample_num).zfill(3)}"
-                numbers = re.findall(r'(\d+\.?\d*)', sample_values)
+                numbers = [float(x) for x in re.findall(r'(\d+\.?\d*)', sample_values)]
 
                 if len(numbers) >= 5:
-                    sample_data = {}
-                    for i, param in enumerate(leaf_params):
-                        if i < len(numbers):
-                            try:
-                                sample_data[param] = float(numbers[i])
-                            except (ValueError, TypeError):
-                                sample_data[param] = 0.0
-
+                    sample_data = _align_numbers_to_params(numbers, leaf_params, sample_id, kind='leaf')
                     if sample_data and any(v != 0.0 for v in sample_data.values()):
                         samples_data[sample_id] = sample_data
                         processed_count += 1
 
             # Tabular number chunking for farm leaf only
             if not samples_data and not is_sp_leaf:
-                all_numbers = re.findall(r'(\d+\.?\d*)', all_text)
+                all_numbers = [float(x) for x in re.findall(r'(\d+\.?\d*)', all_text)]
                 if len(all_numbers) >= 40:
                     for i in range(min(12, len(all_numbers) // 8)):
                         sample_id = f"L{i+1:03d}"
-                        sample_data = {}
                         start_idx = i * 8
-                        for j, param in enumerate(leaf_params):
-                            if start_idx + j < len(all_numbers):
-                                try:
-                                    sample_data[param] = float(all_numbers[start_idx + j])
-                                except (ValueError, TypeError):
-                                    sample_data[param] = 0.0
-
+                        chunk = all_numbers[start_idx:start_idx + 8]
+                        sample_data = _align_numbers_to_params(chunk, leaf_params, sample_id, kind='leaf')
                         if sample_data and any(v != 0.0 for v in sample_data.values()):
                             samples_data[sample_id] = sample_data
 
@@ -1933,6 +1998,129 @@ def format_raw_text_as_structured_json(raw_text: str, container_type: str) -> di
         return {"Farm_Leaf_Test_Data": {}}
 
 
+def _flatten_ocr_sample(sample: dict, container_type: str) -> tuple:
+    """Return (sample_id, flat_param_dict) from a Document AI / Excel sample object."""
+    if not isinstance(sample, dict):
+        return None, {}
+
+    sample_id = (
+        sample.get('sample_id')
+        or sample.get('Sample ID')
+        or sample.get('Sample No.')
+        or sample.get('Lab No.')
+        or sample.get('lab_no')
+    )
+
+    # Nested soil: {sample_id, data: {...}}
+    data = sample.get('data') if isinstance(sample.get('data'), dict) else None
+
+    # Nested leaf Document AI shape
+    dry = sample.get('% Dry Matter') if isinstance(sample.get('% Dry Matter'), dict) else None
+    mgkg = sample.get('mg/kg Dry Matter') if isinstance(sample.get('mg/kg Dry Matter'), dict) else None
+
+    flat = {}
+    if data:
+        for k, v in data.items():
+            if v is None or isinstance(v, dict):
+                continue
+            num = _parse_numeric_token(v) if not isinstance(v, (int, float)) else float(v)
+            flat[k] = num if num is not None else v
+    elif dry or mgkg:
+        # Flatten leaf nested structure into standard keys
+        key_map_dry = {
+            'N': 'N (%)', 'P': 'P (%)', 'K': 'K (%)', 'MG': 'Mg (%)', 'Mg': 'Mg (%)',
+            'CA': 'Ca (%)', 'Ca': 'Ca (%)',
+        }
+        key_map_mg = {
+            'B': 'B (mg/kg)', 'CU': 'Cu (mg/kg)', 'Cu': 'Cu (mg/kg)',
+            'ZN': 'Zn (mg/kg)', 'Zn': 'Zn (mg/kg)',
+        }
+        if dry:
+            for k, v in dry.items():
+                out_k = key_map_dry.get(str(k).upper() if len(str(k)) <= 2 else str(k), key_map_dry.get(str(k), None))
+                if not out_k:
+                    # try direct
+                    ku = str(k).upper()
+                    out_k = {'N': 'N (%)', 'P': 'P (%)', 'K': 'K (%)', 'MG': 'Mg (%)', 'CA': 'Ca (%)'}.get(ku)
+                if out_k:
+                    num = _parse_numeric_token(v) if not isinstance(v, (int, float)) else float(v)
+                    if num is not None:
+                        flat[out_k] = num
+        if mgkg:
+            for k, v in mgkg.items():
+                ku = str(k).upper()
+                out_k = {'B': 'B (mg/kg)', 'CU': 'Cu (mg/kg)', 'ZN': 'Zn (mg/kg)'}.get(ku)
+                if out_k:
+                    num = _parse_numeric_token(v) if not isinstance(v, (int, float)) else float(v)
+                    if num is not None:
+                        flat[out_k] = num
+    else:
+        skip = {
+            'sample_id', 'Sample ID', 'Sample No.', 'Lab No.', 'sample_no', 'lab_no',
+            'data', '% Dry Matter', 'mg/kg Dry Matter',
+        }
+        for k, v in sample.items():
+            if k in skip or v is None or isinstance(v, dict):
+                continue
+            num = _parse_numeric_token(v) if not isinstance(v, (int, float)) else float(v)
+            flat[k] = num if num is not None else v
+
+    # Normalize common soil param aliases to compact keys
+    if container_type.lower() == 'soil' and flat:
+        alias = {
+            'Nitrogen (%)': 'N (%)',
+            'Organic Carbon (%)': 'Org. C (%)',
+            'Available P (mg/kg)': 'Avail P (mg/kg)',
+            'C.E.C (meq%)': 'CEC (meq%)',
+            'CEC (meq/100 g)': 'CEC (meq%)',
+        }
+        flat = {alias.get(k, k): v for k, v in flat.items()}
+
+    return (str(sample_id) if sample_id is not None else None), flat
+
+
+def _score_structured_samples(samples: dict, container_type: str) -> float:
+    """Score how realistic a structured sample dict is (detect shifted columns)."""
+    if not samples:
+        return -1e9
+    total = 0.0
+    count = 0
+    for data in samples.values():
+        if not isinstance(data, dict) or not data:
+            continue
+        if container_type.lower() == 'soil':
+            order = [
+                'pH', 'N (%)', 'Org. C (%)', 'Total P (mg/kg)', 'Avail P (mg/kg)',
+                'Exch. K (meq%)', 'Exch. Ca (meq%)', 'Exch. Mg (meq%)', 'CEC (meq%)',
+            ]
+            # also accept expanded aliases already normalized
+            window = []
+            for p in order:
+                v = data.get(p)
+                if isinstance(v, (int, float)):
+                    window.append(float(v))
+                else:
+                    break
+            if len(window) >= 5:
+                total += _score_soil_window(window)
+                count += 1
+        else:
+            order = ['N (%)', 'P (%)', 'K (%)', 'Mg (%)', 'Ca (%)', 'B (mg/kg)', 'Cu (mg/kg)', 'Zn (mg/kg)']
+            window = []
+            for p in order:
+                v = data.get(p)
+                if isinstance(v, (int, float)):
+                    window.append(float(v))
+                else:
+                    break
+            if len(window) >= 5:
+                total += _score_leaf_window(window)
+                count += 1
+    if count == 0:
+        return -1e9
+    return total / count
+
+
 def _structured_from_ocr_tables(ocr_result: dict, container_type: str) -> dict:
     """Build structured samples from Document AI / OCR table objects when present."""
     if not ocr_result:
@@ -1940,101 +2128,138 @@ def _structured_from_ocr_tables(ocr_result: dict, container_type: str) -> dict:
     tables = ocr_result.get('tables') or []
     samples_data = {}
     for table in tables:
-        if str(table.get('type', '')).lower() != container_type.lower():
-            continue
-        for sample in table.get('samples') or []:
-            if not isinstance(sample, dict):
+        ttype = str(table.get('type', '')).lower()
+        if ttype not in (container_type.lower(), 'unknown', ''):
+            # still try unknown tables if empty so far
+            if ttype not in ('soil', 'leaf') and samples_data:
                 continue
-            sample_id = sample.get('sample_id') or sample.get('Sample ID') or sample.get('Lab No.')
-            data = sample.get('data') if isinstance(sample.get('data'), dict) else None
-            if data is None:
-                data = {
-                    k: v for k, v in sample.items()
-                    if k not in ('sample_id', 'Sample ID', 'Lab No.', 'sample_no', 'lab_no', 'data')
-                    and v is not None
-                }
-            if sample_id and data:
-                samples_data[str(sample_id)] = data
+            if ttype in ('soil', 'leaf') and ttype != container_type.lower():
+                continue
+        for sample in table.get('samples') or []:
+            sid, flat = _flatten_ocr_sample(sample, container_type)
+            if sid and flat:
+                samples_data[sid] = flat
+
     if not samples_data:
-        # extraction_details SP_Lab_Test_Report path
         details = (ocr_result.get('raw_data') or {}).get('extraction_details') or {}
-        for key in ('SP_Lab_Test_Report', 'Farm_Soil_Test_Data', 'Farm_3_Soil_Test_Data', 'Farm_Leaf_Test_Data'):
+        for key in ('SP_Lab_Test_Report', 'Farm_Soil_Test_Data', 'Farm_3_Soil_Test_Data', 'Farm_Leaf_Test_Data', 'leaf_data'):
             block = details.get(key)
-            if isinstance(block, dict) and block:
-                samples_data = block
-                break
+            if isinstance(block, dict) and block.get('samples'):
+                for sample in block['samples']:
+                    sid, flat = _flatten_ocr_sample(sample, container_type)
+                    if sid and flat:
+                        samples_data[sid] = flat
+            elif isinstance(block, dict) and block:
+                # map of sample_id -> data
+                for sid, data in block.items():
+                    if sid in ('total_samples', 'sample_ids', 'parameters', 'report_type', 'samples'):
+                        continue
+                    if isinstance(data, dict):
+                        _, flat = _flatten_ocr_sample({'sample_id': sid, **data} if 'data' not in data else {'sample_id': sid, 'data': data.get('data', data)}, container_type)
+                        if not flat and isinstance(data, dict):
+                            flat = {k: v for k, v in data.items() if not isinstance(v, dict)}
+                        if flat:
+                            samples_data[str(sid)] = flat
+
     if not samples_data:
         return {}
-    # Normalize key
+
     ids = list(samples_data.keys())
-    is_sp = any('/' in str(i) for i in ids)
-    if is_sp or 'SP LAB' in str(ocr_result.get('raw_data', {}).get('text', '')).upper():
+    is_sp = any('/' in str(i) for i in ids) or 'SP LAB' in str(
+        (ocr_result.get('raw_data') or {}).get('text', '')
+    ).upper()
+    if is_sp:
         return {"SP_Lab_Test_Report": samples_data}
     if container_type.lower() == 'soil':
         return {"Farm_Soil_Test_Data": samples_data}
     return {"Farm_Leaf_Test_Data": samples_data}
 
 
-def _prefer_text_structured(text_struct: dict, table_struct: dict, raw_text: str, container_type: str) -> dict:
-    """Prefer raw-text parse; merge table data only when text is missing samples/params.
+def _merge_structured_sources(text_struct: dict, table_struct: dict, raw_text: str, container_type: str) -> dict:
+    """Pick the best structured source so output matches real OCR (tables or text).
 
-    Always re-apply pH from raw text for soil SP Lab samples so structured matches Raw OCR.
+    Document AI header-mapped tables usually beat positional text parsing.
+    Bad text parses (shifted columns) must not overwrite good tables.
     """
     text_key = next(iter(text_struct), None) if text_struct else None
     table_key = next(iter(table_struct), None) if table_struct else None
     text_samples = (text_struct or {}).get(text_key or '', {}) or {}
     table_samples = (table_struct or {}).get(table_key or '', {}) or {}
 
-    if not text_samples and table_samples:
-        chosen_key = table_key or text_key or 'SP_Lab_Test_Report'
-        merged = dict(table_samples)
+    text_score = _score_structured_samples(text_samples, container_type)
+    table_score = _score_structured_samples(table_samples, container_type)
+
+    # Prefer higher score; if close, prefer more samples
+    prefer_table = False
+    if table_samples and not text_samples:
+        prefer_table = True
     elif text_samples and not table_samples:
-        chosen_key = text_key or 'SP_Lab_Test_Report'
-        merged = dict(text_samples)
-    elif text_samples and table_samples:
-        # Prefer text for samples it has; fill gaps from tables
-        chosen_key = text_key if text_key else table_key
-        merged = dict(table_samples)
-        merged.update(text_samples)  # text wins on overlap
+        prefer_table = False
+    elif table_samples and text_samples:
+        if table_score > text_score + 0.5:
+            prefer_table = True
+        elif text_score > table_score + 0.5:
+            prefer_table = False
+        else:
+            # tie-breaker: more complete params / more samples
+            def completeness(samples):
+                if not samples:
+                    return 0
+                return sum(len(v) for v in samples.values() if isinstance(v, dict)) / max(len(samples), 1)
+            prefer_table = completeness(table_samples) >= completeness(text_samples)
     else:
         return text_struct or table_struct or {}
 
-    # Force pH (and full row) consistency from raw OCR for SP Lab soil
-    if container_type.lower() == 'soil' and raw_text:
-        soil_params = [
-            'pH', 'N (%)', 'Org. C (%)', 'Total P (mg/kg)', 'Avail P (mg/kg)',
-            'Exch. K (meq%)', 'Exch. Ca (meq%)', 'Exch. Mg (meq%)', 'CEC (meq%)',
-        ]
-        from_text = _extract_sp_lab_samples_from_text(raw_text, 'S', soil_params)
-        for sid, data in from_text.items():
-            if sid not in merged:
-                merged[sid] = data
-            else:
-                # Text values win — especially pH
-                merged[sid] = {**merged[sid], **data}
+    if prefer_table:
+        chosen_key = table_key or text_key or 'SP_Lab_Test_Report'
+        merged = dict(table_samples)
+        # Fill missing samples from text if text score is ok
+        if text_score > 0:
+            for sid, data in text_samples.items():
+                if sid not in merged:
+                    merged[sid] = data
+    else:
+        chosen_key = text_key or table_key or 'SP_Lab_Test_Report'
+        merged = dict(text_samples)
+        if table_score > 0:
+            for sid, data in table_samples.items():
+                if sid not in merged:
+                    merged[sid] = data
+                else:
+                    # fill missing keys from table
+                    for k, v in data.items():
+                        if k not in merged[sid] or merged[sid][k] in (None, '', 0):
+                            merged[sid][k] = v
 
     return {chosen_key: merged}
 
 
 def _show_raw_text_as_json(raw_text: str, container_type: str, ocr_result: dict = None) -> None:
-    """Display raw extracted text data in structured JSON format.
+    """Display structured OCR data that faithfully follows the uploaded document.
 
-    Structured OCR Data is always derived from Raw OCR Text (never hardcoded samples).
+    Priority: Document AI / Excel tables (header-mapped) when realistic, else raw-text
+    parse with sliding-window alignment. Never uses hardcoded sample templates.
     """
-    # Primary: parse real OCR text
     structured_data = format_raw_text_as_structured_json(raw_text, container_type)
 
-    # Secondary: Document AI tables (may help fill gaps; text wins on conflict)
     try:
         table_struct = _structured_from_ocr_tables(ocr_result or {}, container_type)
         if table_struct:
-            structured_data = _prefer_text_structured(
+            structured_data = _merge_structured_sources(
                 structured_data, table_struct, raw_text, container_type
             )
     except Exception:
         pass
 
-    # Raw Extracted Text Data - display as structured JSON
+    # Final pass: for SP Lab IDs, re-parse raw text and keep the higher-scoring version
+    try:
+        text_only = format_raw_text_as_structured_json(raw_text, container_type)
+        structured_data = _merge_structured_sources(
+            text_only, structured_data, raw_text, container_type
+        )
+    except Exception:
+        pass
+
     st.markdown(f"### 📝 {t('upload_raw_text', 'Raw Extracted Text Data')}")
     st.markdown(f"#### 📊 {t('upload_structured_data', 'Structured OCR Data (JSON Format)')}")
     st.markdown(f"**{t('upload_ai_analysis_note', 'This data will be used by the AI for analysis. Each sample ID contains its parameter values:')}**")