fix: parse SP Lab vertical OCR layout for accurate soil structured data

Add column-major vertical Document AI parser for SP Lab reports (parameter label then one value per sample line). Fix Total P/Available P digits being skipped as column indices. Verified exact match on all 10 samples from user OCR including N.D.

modules/upload.py

@@ -1480,24 +1480,303 @@ def _align_soil_numbers_to_params(numbers: list, params: list, sample_id: str =
     return _align_numbers_to_params(numbers, params, sample_id, kind='soil')
 
 
+def _parse_sp_lab_vertical_document(text: str, sample_prefix: str, params: list) -> dict:
+    """Parse SP Lab reports where Document AI dumps one token per line.
+
+    Real layout (user OCR)::
+
+        S218/25
+        S219/25
+        ...
+        1
+        2
+        3
+        PH
+        5.0
+        4.3
+        ...
+        Nitrogen
+        0.10
+        ...
+
+    Sample IDs form columns; each parameter label is followed by one value
+    per sample (same order). Multiple table sections are supported.
+    """
+    if not text:
+        return {}
+
+    lines = [ln.strip() for ln in text.split('\n') if ln.strip()]
+    prefix = sample_prefix.upper()
+    id_re = re.compile(rf'^{prefix}\d{{1,4}}\s*/\s*\d{{2}}$', re.I)
+    id_find = re.compile(rf'\b({prefix}\d{{1,4}}\s*/\s*\d{{2}})\b', re.I)
+
+    def canon(sid: str) -> str:
+        sid = re.sub(r'\s+', '', sid.strip())
+        return sid[0].upper() + sid[1:] if sid else sid
+
+    # Soil / leaf parameter label matchers (order matters for specificity)
+    if params and params[0] == 'pH':
+        label_specs = [
+            (re.compile(r'^pH$', re.I), 'pH'),
+            (re.compile(r'^PH$', re.I), 'pH'),
+            (re.compile(r'^Nitrogen$', re.I), 'N (%)'),
+            (re.compile(r'^N\s*\(%\)$', re.I), 'N (%)'),
+            (re.compile(r'^Organic\s*Carbon$', re.I), 'Org. C (%)'),
+            (re.compile(r'^Org\.?\s*C', re.I), 'Org. C (%)'),
+            (re.compile(r'^Total\s*P', re.I), 'Total P (mg/kg)'),
+            (re.compile(r'^Available\s*P', re.I), 'Avail P (mg/kg)'),
+            (re.compile(r'^Avail\.?\s*P', re.I), 'Avail P (mg/kg)'),
+            (re.compile(r'^Exch\.?\s*K', re.I), 'Exch. K (meq%)'),
+            (re.compile(r'^Exch\.?\s*Ca', re.I), 'Exch. Ca (meq%)'),
+            (re.compile(r'^Exch\.?\s*Mg', re.I), 'Exch. Mg (meq%)'),
+            (re.compile(r'^C\.?E\.?C\.?$', re.I), 'CEC (meq%)'),
+        ]
+    else:
+        label_specs = [
+            (re.compile(r'^N(?:itrogen)?$', re.I), 'N (%)'),
+            (re.compile(r'^P(?:hos(?:phorus)?)?$', re.I), 'P (%)'),
+            (re.compile(r'^K(?:alium|potassium)?$', re.I), 'K (%)'),
+            (re.compile(r'^Mg(?:agnesium)?$', re.I), 'Mg (%)'),
+            (re.compile(r'^Ca(?:lcium)?$', re.I), 'Ca (%)'),
+            (re.compile(r'^B(?:oron)?$', re.I), 'B (mg/kg)'),
+            (re.compile(r'^Cu(?:pper)?$', re.I), 'Cu (mg/kg)'),
+            (re.compile(r'^Zn(?:inc)?$', re.I), 'Zn (mg/kg)'),
+        ]
+
+    skip_labels = re.compile(
+        r'^(Exchangeable|Parameter|Lab\s*No.*|Analysis|Date|Page|Serial|'
+        r'Your\s*Ref|TEST\s*REPORT|SP\s*LAB|Sarawak|Non-?Detected|'
+        r'%|mg/kg|meq%|meq/?%?)$',
+        re.I,
+    )
+
+    def match_param_label(line: str):
+        for preg, key in label_specs:
+            if preg.match(line.strip()):
+                return key
+        return None
+
+    def is_pure_index(line: str) -> bool:
+        """Column index rows like 1,2,3...10 that appear under sample IDs."""
+        return bool(re.fullmatch(r'\d{1,2}', line.strip()))
+
+    def is_value_token(line: str) -> bool:
+        s = line.strip()
+        if not s:
+            return False
+        if id_re.match(s) or id_find.search(s):
+            return False
+        if match_param_label(s):
+            return False
+        if skip_labels.match(s):
+            return False
+        # N.D. / ND
+        if re.fullmatch(r'N\.?\s*D\.?', s, re.I):
+            return True
+        if re.fullmatch(r'<?\d+\.?\d*', s):
+            return True
+        return False
+
+    def parse_value(line: str):
+        s = line.strip()
+        if re.fullmatch(r'N\.?\s*D\.?', s, re.I):
+            return 0.0
+        return _parse_numeric_token(s)
+
+    # Split document into table sections (each starts near sample IDs after "Lab No" / "Parameter")
+    sections = []
+    current = []
+    for line in lines:
+        if re.search(r'Lab\s*No', line, re.I) and current:
+            sections.append(current)
+            current = [line]
+        else:
+            current.append(line)
+    if current:
+        sections.append(current)
+
+    # If no clear section markers, treat whole doc as one section
+    if len(sections) <= 1:
+        # Also try splitting when a new block of sample IDs appears after parameters
+        sections = [lines]
+
+    all_samples = {}
+
+    def parse_section(sec_lines: list) -> dict:
+        # Collect sample IDs in order of appearance (within this section)
+        sample_ids = []
+        seen = set()
+        for line in sec_lines:
+            for m in id_find.finditer(line):
+                sid = canon(m.group(1))
+                if sid not in seen:
+                    seen.add(sid)
+                    sample_ids.append(sid)
+        if len(sample_ids) < 1:
+            return {}
+
+        n = len(sample_ids)
+        # param -> list of n values
+        param_cols = {p: None for p in params}
+
+        i = 0
+        while i < len(sec_lines):
+            line = sec_lines[i]
+            # Skip sample id lines, indices, units, headers
+            if id_find.search(line) and not match_param_label(line):
+                i += 1
+                continue
+            if is_pure_index(line):
+                i += 1
+                continue
+            if skip_labels.match(line):
+                i += 1
+                continue
+
+            pkey = match_param_label(line)
+            if pkey and pkey in param_cols:
+                # Collect the next `n` value tokens after this parameter label.
+                # IMPORTANT: after a label, digits like 59 / 2 / 1 are REAL lab
+                # values (Total P, Available P), NOT column indices 1..10.
+                values = []
+                j = i + 1
+                while j < len(sec_lines) and len(values) < n:
+                    nxt = sec_lines[j]
+                    # Stop if we hit next param label
+                    if match_param_label(nxt):
+                        break
+                    # Units / section words only — do not treat digits as indices here
+                    if skip_labels.match(nxt):
+                        j += 1
+                        # If we already have values and hit a unit line, we're done
+                        if values:
+                            break
+                        continue
+                    if id_find.search(nxt):
+                        break
+                    if re.fullmatch(r'N\.?\s*D\.?', nxt, re.I) or re.fullmatch(r'<?\d+\.?\d*', nxt):
+                        val = parse_value(nxt)
+                        if val is not None:
+                            values.append(val)
+                        j += 1
+                    else:
+                        j += 1
+                        if values:
+                            break
+                if len(values) >= n:
+                    param_cols[pkey] = values[:n]
+                elif len(values) > 0:
+                    # Accept partial only if nearly complete (e.g. trailing missing)
+                    if len(values) >= n - 1:
+                        param_cols[pkey] = values
+                i = j
+                continue
+            i += 1
+
+        # Require at least pH/N + a few more params filled
+        filled = [p for p in params if param_cols.get(p)]
+        if len(filled) < 3:
+            return {}
+
+        # Normalize lengths: use min available
+        out = {sid: {} for sid in sample_ids}
+        for p in params:
+            col = param_cols.get(p)
+            if not col:
+                continue
+            for idx, sid in enumerate(sample_ids):
+                if idx < len(col):
+                    out[sid][p] = col[idx]
+        # Drop empty
+        return {sid: data for sid, data in out.items() if data}
+
+    # Parse each Lab-No section separately (critical: each table has its own
+    # sample count — do NOT parse the whole doc as one 10-column table).
+    for sec in sections:
+        part = parse_section(sec)
+        for sid, data in part.items():
+            if sid not in all_samples:
+                all_samples[sid] = data
+            else:
+                # Merge params; prefer non-empty / longer
+                for k, v in data.items():
+                    if k not in all_samples[sid]:
+                        all_samples[sid][k] = v
+
+    # If Lab-No split failed, try sample-id cluster split
+    if len(all_samples) < 2:
+        clusters = []
+        cluster = []
+        saw_id = False
+        for line in lines:
+            has_id = bool(id_find.search(line))
+            if has_id:
+                if saw_id is False and cluster and any(id_find.search(x) for x in cluster):
+                    # new cluster after params
+                    clusters.append(cluster)
+                    cluster = []
+                saw_id = True
+                cluster.append(line)
+            else:
+                if cluster:
+                    cluster.append(line)
+                # reset saw_id after we leave a run of IDs and hit a param
+                if match_param_label(line):
+                    saw_id = False
+        if cluster:
+            clusters.append(cluster)
+        for cl in clusters:
+            part = parse_section(cl)
+            all_samples.update(part)
+
+    # Last resort: whole document only if still empty
+    if not all_samples:
+        all_samples = parse_section(lines)
+
+    return all_samples
+
+
 def _extract_sp_lab_samples_from_text(text: str, sample_prefix: str, params: list) -> dict:
     """
     Parse SP Lab samples (e.g. S218/25 or P220/25) from real OCR text.
     Never uses hardcoded values — structured data must match Raw OCR Text.
 
     Handles:
+    - Vertical Document AI dumps (one token per line) — SP Lab official layout
     - Row-per-sample:  S218/25 5.26 0.12 0.91 ...
-    - Column-major / transposed (common Document AI reading order):
-        S218/25 S219/25 S220/25
-        pH 5.26 5.10 4.88
-        N 0.12 0.11 0.08
-        ...
+    - Column-major / transposed (samples on one line, params as rows)
     - Leading junk numbers before pH (lab serial bleeding into values)
     """
     samples = {}
     if not text:
         return samples
 
+    # ------------------------------------------------------------------
+    # Strategy V (highest priority): vertical one-token-per-line SP Lab OCR
+    # ------------------------------------------------------------------
+    vertical = _parse_sp_lab_vertical_document(text, sample_prefix, params)
+    if vertical:
+        # Validate soil pH columns look sane before accepting
+        if params and params[0] == 'pH':
+            ph_ok = sum(
+                1 for d in vertical.values()
+                if isinstance(d.get('pH'), (int, float)) and 2.5 <= d['pH'] <= 9.5
+            )
+            if ph_ok >= max(1, len(vertical) // 2):
+                return vertical
+        else:
+            # leaf: N in range
+            n_ok = sum(
+                1 for d in vertical.values()
+                if isinstance(d.get('N (%)'), (int, float)) and 1.2 <= d['N (%)'] <= 4.0
+            )
+            if n_ok >= max(1, len(vertical) // 2) or len(vertical) >= 1:
+                return vertical
+        # even if validation weak, prefer vertical if it found many samples with many params
+        avg_params = sum(len(d) for d in vertical.values()) / max(len(vertical), 1)
+        if len(vertical) >= 3 and avg_params >= 5:
+            return vertical
+
     lines = [ln.strip() for ln in text.split('\n') if ln.strip()]
     joined = ' '.join(lines)
     joined_norm = re.sub(r'[^\w\s\.\-\(\)\[\]\/<>%]', ' ', joined)
@@ -1527,9 +1806,7 @@ def _extract_sp_lab_samples_from_text(text: str, sample_prefix: str, params: lis
     is_soil = bool(params and params[0] == 'pH')
 
     # ------------------------------------------------------------------
-    # Strategy T (priority): transposed / column-major SP Lab layout
-    # Sample IDs appear first (often one line), then parameter rows with
-    # one value per sample in the same column order.
+    # Strategy T: transposed / column-major SP Lab layout (values on same line)
     # ------------------------------------------------------------------
     def _parse_transposed() -> dict:
         # Collect ordered unique sample IDs as they appear