structural_parser.py

from __future__ import annotations

import re
from typing import Any


ARTICLE_RE = re.compile(
    r"(?m)^\s*(?P<kind>Ek Madde|Geçici Madde|Madde)\s+"
    r"(?P<number>\d+(?:/[A-ZÇĞİÖŞÜ])?|[0-9]+[A-ZÇĞİÖŞÜ]?)\s*[–-]?"
)

SCHEDULE_RE = re.compile(
    r"(?m)^\s*(?P<title>EK\s+GÖSTERGE\s+CETVELİ|MAKAM\s+TAZMİNATI\s+CETVELİ[^\n]*)\s*:?\s*$",
    flags=re.IGNORECASE,
)

HIERARCHY_HEADING_RE = re.compile(
    r"(?m)^\s*(?P<label>[A-ZÇĞİÖŞÜ0-9]+(?:\s+[A-ZÇĞİÖŞÜ0-9]+){0,4}\s+"
    r"(?P<kind>KISIM|BÖLÜM|ALT BÖLÜM))\s*$"
)

CLAUSE_MARKER_RES = [
    # A true lettered clause is written ``a) ...``.  The letter in a
    # parenthesized cross-reference such as ``40 ıncı maddenin (b) fıkrası``
    # is not a new clause boundary; accepting ``(`` here used to cut a legal
    # sentence into two invalid normative units.
    re.compile(r"(?<![\w/(])([a-zçğıöşü])\)\s", flags=re.IGNORECASE),
    re.compile(r"(?<![\w/])([a-zçğıöşü])\.\s+(?=[A-ZÇĞİÖŞÜ])", flags=re.IGNORECASE),
    re.compile(r"\((\d+)\)\s"),
]


def parse_structural_units(text: str, document_id: str = "TR-KANUN-2547") -> dict[str, Any]:
    articles = []
    clauses = []
    schedules = []
    matches = list(ARTICLE_RE.finditer(text or ""))
    schedule_matches = list(SCHEDULE_RE.finditer(text or ""))

    for idx, match in enumerate(matches):
        start = match.start()
        end = matches[idx + 1].start() if idx + 1 < len(matches) else len(text)
        next_schedule = _first_schedule_start_after(schedule_matches, start, end)
        if next_schedule is not None:
            end = next_schedule
        article_text = text[start:end].strip()
        article_id = f"{match.group('kind')} {match.group('number')}"
        title = _article_title(article_text, text, start)
        heading_path = _heading_path(text, start)

        article = {
            "document_id": document_id,
            "article_id": article_id,
            "article_type": _article_type(match.group("kind")),
            "title": title,
            "heading_path": heading_path,
            "article_heading": title,
            "source_span": {"char_start": start, "char_end": end},
            "source_text": article_text,
        }
        articles.append(article)

        for ordinal, clause in enumerate(split_article_clauses(article_text), start=1):
            label, c_start, c_end = clause
            clauses.append(
                {
                    "document_id": document_id,
                    "article_id": article_id,
                    "clause_id": f"{_slug(article_id)}__{ordinal:03d}_{_slug(label)}",
                    "label": label,
                    "source_span": {
                        "article_char_start": c_start,
                        "article_char_end": c_end,
                        "document_char_start": start + c_start,
                        "document_char_end": start + c_end,
                    },
                    "source_text": article_text[c_start:c_end].strip(),
                }
        )

    for idx, match in enumerate(schedule_matches):
        start = match.start()
        next_article_start = _first_article_start_after(matches, start)
        next_schedule_start = schedule_matches[idx + 1].start() if idx + 1 < len(schedule_matches) else None
        candidates = [value for value in [next_article_start, next_schedule_start, len(text)] if value is not None]
        end = min(candidates) if candidates else len(text)
        title = re.sub(r"\s+", " ", match.group("title")).strip().rstrip(":")
        schedule_text = text[start:end].strip()
        schedule_id = f"{document_id}__{_slug(title).upper()}"
        schedules.append(
            {
                "class": "ScheduleUnit",
                "document_id": document_id,
                "schedule_id": schedule_id,
                "article_id": title,
                "title": title,
                "source_span": {"char_start": start, "char_end": end},
                "source_text": schedule_text,
            }
        )

    if not articles:
        articles, clauses = _parse_generic_sections(text or "", document_id)

    return {
        "document_id": document_id,
        "article_count": len(articles),
        "clause_count": len(clauses),
        "schedule_count": len(schedules),
        "articles": articles,
        "clauses": clauses,
        "schedules": schedules,
    }


GENERIC_HEADING_RE = re.compile(
    r"(?m)^\s*(?:(?P<number>\d+(?:\.\d+){0,3})[.)-]\s+)?"
    r"(?P<title>[A-ZÇĞİÖŞÜ][^\n]{2,119})\s*$"
)


def _parse_generic_sections(text: str, document_id: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
    """Parse non-legislative institutional TXT files without changing law parsing.

    Numbered headings and short uppercase headings become stable section units. If
    the document has no visible hierarchy, non-empty paragraphs become units so
    every source span remains traceable and reviewable.
    """
    headings = []
    for match in GENERIC_HEADING_RE.finditer(text):
        title = re.sub(r"\s+", " ", match.group("title")).strip().rstrip(":")
        if not _looks_like_generic_heading(title, bool(match.group("number"))):
            continue
        headings.append((match.start(), match.end(), match.group("number") or "", title))

    spans: list[tuple[str, str, int, int]] = []
    if headings:
        for index, (start, _heading_end, number, title) in enumerate(headings, start=1):
            end = headings[index][0] if index < len(headings) else len(text)
            section_id = f"Bölüm {number or index}"
            spans.append((section_id, title, start, end))
    else:
        for index, match in enumerate(re.finditer(r"(?ms)(?:^|\n\s*\n)(\S.*?)(?=\n\s*\n|\Z)", text), start=1):
            body = match.group(1).strip()
            if len(body) < 20:
                continue
            title = re.sub(r"\s+", " ", body).strip()[:120]
            spans.append((f"Birim {index}", title, match.start(1), match.end(1)))

    articles: list[dict[str, Any]] = []
    clauses: list[dict[str, Any]] = []
    for index, (article_id, title, start, end) in enumerate(spans, start=1):
        source_text = text[start:end].strip()
        article = {
            "document_id": document_id,
            "article_id": article_id,
            "article_type": "section",
            "title": title,
            "heading_path": [],
            "article_heading": title,
            "source_span": {"char_start": start, "char_end": end},
            "source_text": source_text,
        }
        articles.append(article)
        clauses.append(
            {
                "document_id": document_id,
                "article_id": article_id,
                "clause_id": f"{_slug(article_id)}__001_section",
                "label": "section",
                "source_span": {
                    "article_char_start": 0,
                    "article_char_end": len(source_text),
                    "document_char_start": start,
                    "document_char_end": end,
                },
                "source_text": source_text,
            }
        )
    return articles, clauses


def _looks_like_generic_heading(title: str, numbered: bool) -> bool:
    if numbered:
        return not title.endswith((".", ";", ","))
    words = title.split()
    if not 1 <= len(words) <= 10:
        return False
    letters = [char for char in title if char.isalpha()]
    return bool(letters) and sum(char.isupper() for char in letters) / len(letters) >= 0.75


def _heading_path(text: str, article_start: int) -> list[dict[str, str]]:
    """Return the active part/section headings governing an article.



    Turkish legislation normally places a hierarchy marker on one line and its

    subject on the next. Keeping that path prevents article titles from being

    interpreted without their document context.

    """
    active: dict[str, tuple[int, dict[str, str]]] = {}
    for match in HIERARCHY_HEADING_RE.finditer(text, 0, article_start):
        title = _next_heading_title(text, match.end(), article_start)
        kind = match.group("kind").lower().replace(" ", "_")
        active[kind] = (
            match.start(),
            {
                "level": kind,
                "label": re.sub(r"\s+", " ", match.group("label")).strip(),
                "title": title,
            },
        )
    return [item for _position, item in sorted(active.values(), key=lambda value: value[0])]


def _next_heading_title(text: str, start: int, limit: int) -> str:
    for line in text[start:limit].splitlines()[:4]:
        candidate = re.sub(r"\s+", " ", line).strip().rstrip(":")
        candidate = re.sub(r"(?<=[A-Za-zÇĞİÖŞÜçğıöşü])\d+$", "", candidate).strip()
        if candidate and not ARTICLE_RE.match(candidate):
            return candidate
    return ""


def _first_schedule_start_after(matches: list[re.Match], start: int, end: int) -> int | None:
    starts = [match.start() for match in matches if start < match.start() < end]
    return min(starts) if starts else None


def _first_article_start_after(matches: list[re.Match], start: int) -> int | None:
    starts = [match.start() for match in matches if match.start() > start]
    return min(starts) if starts else None


def split_article_clauses(article_text: str) -> list[tuple[str, int, int]]:
    """Return authoritative legal-clause boundaries for every MCKF layer.

    Runtime ontology generation and structural exports must call this same
    function.  Duplicated marker regexes previously allowed the two artifacts
    to disagree about where a provision began and ended.
    """
    markers: list[tuple[int, str]] = []
    for pattern in CLAUSE_MARKER_RES:
        for match in pattern.finditer(article_text):
            if _looks_like_date_context(article_text, match.start()):
                continue
            markers.append((match.start(), match.group(1)))

    markers = sorted(set(markers), key=lambda item: item[0])
    if not markers:
        return [("article", 0, len(article_text))]

    spans: list[tuple[str, int, int]] = []
    if markers[0][0] > 0:
        spans.append(("preamble", 0, markers[0][0]))

    for idx, (start, label) in enumerate(markers):
        end = markers[idx + 1][0] if idx + 1 < len(markers) else len(article_text)
        spans.append((label, start, end))
    return spans


def _split_article_clauses(article_text: str) -> list[tuple[str, int, int]]:
    """Backward-compatible alias for older checks and integrations."""
    return split_article_clauses(article_text)


def _previous_heading(text: str, article_start: int) -> str:
    before = text[:article_start].rstrip().splitlines()
    for line in reversed(before[-5:]):
        cleaned = line.strip()
        if (
            cleaned
            and len(cleaned) <= 120
            and not cleaned.isupper()
            and not cleaned.startswith(("Kanun ", "Yayımlandığı", "Madde ", "Ek Madde ", "Geçici Madde "))
            and not cleaned.endswith((".", ";", ","))
        ):
            return cleaned.rstrip(":")
    return ""


def _article_title(article_text: str, full_text: str, article_start: int) -> str:
    first_line = (article_text or "").splitlines()[0].strip()
    remainder = ARTICLE_RE.sub("", first_line, count=1).strip(" –-")
    if remainder and not remainder.startswith("("):
        colon = remainder.find(":")
        semicolon = remainder.find(";")
        boundary_candidates = [value for value in (colon, semicolon) if 0 < value <= 140]
        if boundary_candidates:
            candidate = remainder[:min(boundary_candidates)].strip()
            if 2 <= len(candidate) <= 140:
                return candidate
    return _previous_heading(full_text, article_start)


def _article_type(kind: str) -> str:
    if kind == "Ek Madde":
        return "additional"
    if kind == "Geçici Madde":
        return "temporary"
    return "main"


def _looks_like_date_context(text: str, position: int) -> bool:
    window = text[max(0, position - 16):position + 16]
    return bool(re.search(r"\d{1,2}/\d{1,2}/\d{4}", window))


def _slug(value: str) -> str:
    cleaned = re.sub(r"\W+", "_", value.lower(), flags=re.UNICODE).strip("_")
    return cleaned or "unit"