from __future__ import annotations import re from typing import Any ARTICLE_RE = re.compile( r"(?m)^\s*(?PEk Madde|Geçici Madde|Madde)\s+" r"(?P\d+(?:/[A-ZÇĞİÖŞÜ])?|[0-9]+[A-ZÇĞİÖŞÜ]?)\s*[–-]?" ) SCHEDULE_RE = re.compile( r"(?m)^\s*(?PEK\s+GÖSTERGE\s+CETVELİ|MAKAM\s+TAZMİNATI\s+CETVELİ[^\n]*)\s*:?\s*$", flags=re.IGNORECASE, ) HIERARCHY_HEADING_RE = re.compile( r"(?m)^\s*(?P<label>[A-ZÇĞİÖŞÜ0-9]+(?:\s+[A-ZÇĞİÖŞÜ0-9]+){0,4}\s+" r"(?P<kind>KISIM|BÖLÜM|ALT BÖLÜM))\s*$" ) CLAUSE_MARKER_RES = [ # A true lettered clause is written ``a) ...``. The letter in a # parenthesized cross-reference such as ``40 ıncı maddenin (b) fıkrası`` # is not a new clause boundary; accepting ``(`` here used to cut a legal # sentence into two invalid normative units. re.compile(r"(?<![\w/(])([a-zçğıöşü])\)\s", flags=re.IGNORECASE), re.compile(r"(?<![\w/])([a-zçğıöşü])\.\s+(?=[A-ZÇĞİÖŞÜ])", flags=re.IGNORECASE), re.compile(r"\((\d+)\)\s"), ] def parse_structural_units(text: str, document_id: str = "TR-KANUN-2547") -> dict[str, Any]: articles = [] clauses = [] schedules = [] matches = list(ARTICLE_RE.finditer(text or "")) schedule_matches = list(SCHEDULE_RE.finditer(text or "")) for idx, match in enumerate(matches): start = match.start() end = matches[idx + 1].start() if idx + 1 < len(matches) else len(text) next_schedule = _first_schedule_start_after(schedule_matches, start, end) if next_schedule is not None: end = next_schedule article_text = text[start:end].strip() article_id = f"{match.group('kind')} {match.group('number')}" title = _article_title(article_text, text, start) heading_path = _heading_path(text, start) article = { "document_id": document_id, "article_id": article_id, "article_type": _article_type(match.group("kind")), "title": title, "heading_path": heading_path, "article_heading": title, "source_span": {"char_start": start, "char_end": end}, "source_text": article_text, } articles.append(article) for ordinal, clause in enumerate(split_article_clauses(article_text), start=1): label, c_start, c_end = clause clauses.append( { "document_id": document_id, "article_id": article_id, "clause_id": f"{_slug(article_id)}__{ordinal:03d}_{_slug(label)}", "label": label, "source_span": { "article_char_start": c_start, "article_char_end": c_end, "document_char_start": start + c_start, "document_char_end": start + c_end, }, "source_text": article_text[c_start:c_end].strip(), } ) for idx, match in enumerate(schedule_matches): start = match.start() next_article_start = _first_article_start_after(matches, start) next_schedule_start = schedule_matches[idx + 1].start() if idx + 1 < len(schedule_matches) else None candidates = [value for value in [next_article_start, next_schedule_start, len(text)] if value is not None] end = min(candidates) if candidates else len(text) title = re.sub(r"\s+", " ", match.group("title")).strip().rstrip(":") schedule_text = text[start:end].strip() schedule_id = f"{document_id}__{_slug(title).upper()}" schedules.append( { "class": "ScheduleUnit", "document_id": document_id, "schedule_id": schedule_id, "article_id": title, "title": title, "source_span": {"char_start": start, "char_end": end}, "source_text": schedule_text, } ) if not articles: articles, clauses = _parse_generic_sections(text or "", document_id) return { "document_id": document_id, "article_count": len(articles), "clause_count": len(clauses), "schedule_count": len(schedules), "articles": articles, "clauses": clauses, "schedules": schedules, } GENERIC_HEADING_RE = re.compile( r"(?m)^\s*(?:(?P<number>\d+(?:\.\d+){0,3})[.)-]\s+)?" r"(?P<title>[A-ZÇĞİÖŞÜ][^\n]{2,119})\s*$" ) def _parse_generic_sections(text: str, document_id: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: """Parse non-legislative institutional TXT files without changing law parsing. Numbered headings and short uppercase headings become stable section units. If the document has no visible hierarchy, non-empty paragraphs become units so every source span remains traceable and reviewable. """ headings = [] for match in GENERIC_HEADING_RE.finditer(text): title = re.sub(r"\s+", " ", match.group("title")).strip().rstrip(":") if not _looks_like_generic_heading(title, bool(match.group("number"))): continue headings.append((match.start(), match.end(), match.group("number") or "", title)) spans: list[tuple[str, str, int, int]] = [] if headings: for index, (start, _heading_end, number, title) in enumerate(headings, start=1): end = headings[index][0] if index < len(headings) else len(text) section_id = f"Bölüm {number or index}" spans.append((section_id, title, start, end)) else: for index, match in enumerate(re.finditer(r"(?ms)(?:^|\n\s*\n)(\S.*?)(?=\n\s*\n|\Z)", text), start=1): body = match.group(1).strip() if len(body) < 20: continue title = re.sub(r"\s+", " ", body).strip()[:120] spans.append((f"Birim {index}", title, match.start(1), match.end(1))) articles: list[dict[str, Any]] = [] clauses: list[dict[str, Any]] = [] for index, (article_id, title, start, end) in enumerate(spans, start=1): source_text = text[start:end].strip() article = { "document_id": document_id, "article_id": article_id, "article_type": "section", "title": title, "heading_path": [], "article_heading": title, "source_span": {"char_start": start, "char_end": end}, "source_text": source_text, } articles.append(article) clauses.append( { "document_id": document_id, "article_id": article_id, "clause_id": f"{_slug(article_id)}__001_section", "label": "section", "source_span": { "article_char_start": 0, "article_char_end": len(source_text), "document_char_start": start, "document_char_end": end, }, "source_text": source_text, } ) return articles, clauses def _looks_like_generic_heading(title: str, numbered: bool) -> bool: if numbered: return not title.endswith((".", ";", ",")) words = title.split() if not 1 <= len(words) <= 10: return False letters = [char for char in title if char.isalpha()] return bool(letters) and sum(char.isupper() for char in letters) / len(letters) >= 0.75 def _heading_path(text: str, article_start: int) -> list[dict[str, str]]: """Return the active part/section headings governing an article. Turkish legislation normally places a hierarchy marker on one line and its subject on the next. Keeping that path prevents article titles from being interpreted without their document context. """ active: dict[str, tuple[int, dict[str, str]]] = {} for match in HIERARCHY_HEADING_RE.finditer(text, 0, article_start): title = _next_heading_title(text, match.end(), article_start) kind = match.group("kind").lower().replace(" ", "_") active[kind] = ( match.start(), { "level": kind, "label": re.sub(r"\s+", " ", match.group("label")).strip(), "title": title, }, ) return [item for _position, item in sorted(active.values(), key=lambda value: value[0])] def _next_heading_title(text: str, start: int, limit: int) -> str: for line in text[start:limit].splitlines()[:4]: candidate = re.sub(r"\s+", " ", line).strip().rstrip(":") candidate = re.sub(r"(?<=[A-Za-zÇĞİÖŞÜçğıöşü])\d+$", "", candidate).strip() if candidate and not ARTICLE_RE.match(candidate): return candidate return "" def _first_schedule_start_after(matches: list[re.Match], start: int, end: int) -> int | None: starts = [match.start() for match in matches if start < match.start() < end] return min(starts) if starts else None def _first_article_start_after(matches: list[re.Match], start: int) -> int | None: starts = [match.start() for match in matches if match.start() > start] return min(starts) if starts else None def split_article_clauses(article_text: str) -> list[tuple[str, int, int]]: """Return authoritative legal-clause boundaries for every MCKF layer. Runtime ontology generation and structural exports must call this same function. Duplicated marker regexes previously allowed the two artifacts to disagree about where a provision began and ended. """ markers: list[tuple[int, str]] = [] for pattern in CLAUSE_MARKER_RES: for match in pattern.finditer(article_text): if _looks_like_date_context(article_text, match.start()): continue markers.append((match.start(), match.group(1))) markers = sorted(set(markers), key=lambda item: item[0]) if not markers: return [("article", 0, len(article_text))] spans: list[tuple[str, int, int]] = [] if markers[0][0] > 0: spans.append(("preamble", 0, markers[0][0])) for idx, (start, label) in enumerate(markers): end = markers[idx + 1][0] if idx + 1 < len(markers) else len(article_text) spans.append((label, start, end)) return spans def _split_article_clauses(article_text: str) -> list[tuple[str, int, int]]: """Backward-compatible alias for older checks and integrations.""" return split_article_clauses(article_text) def _previous_heading(text: str, article_start: int) -> str: before = text[:article_start].rstrip().splitlines() for line in reversed(before[-5:]): cleaned = line.strip() if ( cleaned and len(cleaned) <= 120 and not cleaned.isupper() and not cleaned.startswith(("Kanun ", "Yayımlandığı", "Madde ", "Ek Madde ", "Geçici Madde ")) and not cleaned.endswith((".", ";", ",")) ): return cleaned.rstrip(":") return "" def _article_title(article_text: str, full_text: str, article_start: int) -> str: first_line = (article_text or "").splitlines()[0].strip() remainder = ARTICLE_RE.sub("", first_line, count=1).strip(" –-") if remainder and not remainder.startswith("("): colon = remainder.find(":") semicolon = remainder.find(";") boundary_candidates = [value for value in (colon, semicolon) if 0 < value <= 140] if boundary_candidates: candidate = remainder[:min(boundary_candidates)].strip() if 2 <= len(candidate) <= 140: return candidate return _previous_heading(full_text, article_start) def _article_type(kind: str) -> str: if kind == "Ek Madde": return "additional" if kind == "Geçici Madde": return "temporary" return "main" def _looks_like_date_context(text: str, position: int) -> bool: window = text[max(0, position - 16):position + 16] return bool(re.search(r"\d{1,2}/\d{1,2}/\d{4}", window)) def _slug(value: str) -> str: cleaned = re.sub(r"\W+", "_", value.lower(), flags=re.UNICODE).strip("_") return cleaned or "unit"