app/renpy/script.py

"""Find the translatable strings in a .rpy script, with exact source offsets.

Ren'Py scripts mix dialogue with executable Python, asset filenames, style
definitions and screen layout in one file, and the difference between them is
the difference between a translated game and a game that will not start. So
this parser is built to be *certain* about what it touches:

* Every string literal in the file is located in one pass that understands
  comments, escapes and triple quotes. Nothing downstream ever re-scans raw
  text, so a `#` inside dialogue or a quote inside a comment cannot shift an
  offset.
* Statements are classified from a "code view" of each line - the same line
  with the inside of every string blanked out - so a keyword mentioned inside
  dialogue is never mistaken for the statement's own keyword. This only looks
  at bracket-depth-0 text, on purpose: a string sitting inside some other
  call (`action=ShowMenu("save")`) is never mistaken for a label.
* A second, independent pass then finds every string wrapped in `_(...)` or
  `__(...)`, at *any* depth, wherever it sits - inside an `Achievement(...)`
  constructor, a dict three levels deep in a list inside `init python:`.
  `_()` is Ren'Py's own "translate this" marker, so unlike everything else
  here it needs no statement classification: the developer already answered
  the question by writing it. Only a `translate` block's own body is held
  back from this pass, even one that happens to contain a `_()` call.
* Anything not positively recognised as dialogue, a menu choice, screen text
  or `_()`-marked is left alone. Over-skipping costs a line of translation;
  under-skipping corrupts a script.

The offsets are what makes writing back safe: a translation replaces exactly
the characters between one pair of quotes, so no amount of re-flowing by the
model can disturb the code around it.
"""

from __future__ import annotations

import re
from dataclasses import dataclass

from .text import EscapeError, is_translatable, unescape

# Statements whose entire indented body holds no dialogue. Skipping the block
# outright is safer than trusting per-line rules inside it: `style` bodies name
# fonts, `image` bodies name files, `python` bodies are code, and `translate`
# bodies are somebody else's finished translation.
_BLOCK_SKIP = {
    "python",
    "init",
    "style",
    "transform",
    "image",
    "layeredimage",
    "testcase",
    "translate",
    "camera",
}

# Single statements that carry no dialogue. Their arguments are label names,
# asset paths, transitions or expressions.
_LINE_SKIP = {
    "define",
    "default",
    "jump",
    "call",
    "scene",
    "show",
    "hide",
    "with",
    "play",
    "stop",
    "queue",
    "voice",
    "pause",
    "return",
    "pass",
    "window",
    "nvl",
    "use",
    "add",
    "on",
    "key",
    "timer",
    "imagemap",
    "hotspot",
    "hotbar",
    "drag",
    "draggroup",
    "mousearea",
    "imagebutton",
    "input",
    "bar",
    "vbar",
    "viewport",
    "side",
    "grid",
    "vpgrid",
    "fixed",
    "hbox",
    "vbox",
    "frame",
    "null",
    "has",
    "at",
    "audio",
    "sound",
    "music",
    "movie",
    "transclude",
    "default_focus",
}

# Screen statements whose *first* string is shown to the player. The first, not
# the last: `textbutton "OK" action Return()` and
# `text "Hi" style "big"` both put the visible text first and configuration
# after it.
_UI_KEYWORDS = {"text", "textbutton", "tooltip", "caption", "alt", "label"}

# Prefix words allowed in front of a say statement's dialogue, beyond the
# speaker's own name and image attributes.
_SAY_PREFIX_WORDS = {"extend", "nvl"}

_IDENT_RE = re.compile(r"^[A-Za-z_][A-Za-z_0-9.]*$")
_FIRST_WORD_RE = re.compile(r"^([A-Za-z_][A-Za-z_0-9]*)")


@dataclass
class Literal:
    """One string literal, located exactly in the file."""

    start: int          # offset of the first character inside the quotes
    end: int            # offset just past the last character inside the quotes
    quote: str          # the quote character, for re-escaping
    triple: bool
    line: int           # 0-based index of the line the literal opens on
    depth: int          # bracket nesting at the opening quote
    marked: bool         # sits directly inside a `_(...)` / `__(...)` call


@dataclass
class Found:
    """A translatable string plus where to write its translation back."""

    text: str
    start: int
    end: int
    quote: str
    kind: str           # dialogue | menu | ui
    line: int


def _is_underscore_call(source: str, paren_index: int) -> bool:
    """True when the '(' at `paren_index` is `_(` or `__(`.

    Walks back over whitespace, then over one contiguous identifier, and
    compares that whole token - not just the character before the paren - so
    a real name that merely ends in an underscore (`get_text_ (x)`) is never
    mistaken for the marker.
    """
    cursor = paren_index
    while cursor > 0 and source[cursor - 1] in " \t":
        cursor -= 1
    end = cursor
    while cursor > 0 and (source[cursor - 1].isalnum() or source[cursor - 1] == "_"):
        cursor -= 1
    return source[cursor:end] in ("_", "__")


def scan_literals(source: str) -> tuple[list[Literal], list[str]]:
    """Locate every string literal, and build the blanked-out code view.

    Returns the literals and one "code line" per source line: the same text
    with string contents and comments replaced by spaces, so positions still
    line up with the original but no statement rule can be fooled by them.
    """
    literals: list[Literal] = []
    code = list(source)
    # A byte-order mark is not whitespace, so it would glue itself to the
    # first statement's keyword and make line 1 unrecognisable. Blanking it in
    # the code view fixes the classification while leaving the real offsets -
    # and the mark itself - untouched.
    if code and code[0] == "":
        code[0] = " "
    line_index = 0
    depth = 0
    index = 0
    length = len(source)
    # One entry per currently-open '(' ')' pair, tracking whether it was
    # opened by a bare `_` or `__` - Ren'Py's own "translate this" marker.
    # `_("Continue")`, `Achievement(description=_("..."))` and
    # `flavor=_("..."))` inside a dict two calls deep are all reached this way,
    # regardless of what statement or block they sit inside; see `marked` on
    # Literal.
    paren_marks: list[bool] = []

    while index < length:
        char = source[index]

        if char == "\n":
            line_index += 1
            index += 1
            continue

        if char == "#":
            # Comment: blank to end of line. Ren'Py has no block comments.
            while index < length and source[index] != "\n":
                code[index] = " "
                index += 1
            continue

        if char == "(":
            paren_marks.append(_is_underscore_call(source, index))
            depth += 1
            index += 1
            continue
        if char == ")":
            if paren_marks:
                paren_marks.pop()
            depth = max(0, depth - 1)
            index += 1
            continue
        if char in "[{":
            depth += 1
            index += 1
            continue
        if char in "]}":
            depth = max(0, depth - 1)
            index += 1
            continue

        if char not in "\"'":
            index += 1
            continue

        # A string opens here. Triple quotes first, so `"""` is never read as
        # an empty string followed by a stray quote.
        quote = char
        triple = source.startswith(quote * 3, index)
        marker = quote * 3 if triple else quote
        open_line = line_index
        inner_start = index + len(marker)

        cursor = inner_start
        closed = False
        while cursor < length:
            current = source[cursor]
            if current == "\\":
                # Skip the escaped character so an escaped quote cannot close
                # the string. Newlines inside the escape still count for
                # line numbering.
                if cursor + 1 < length:
                    if source[cursor + 1] == "\n":
                        line_index += 1
                    cursor += 2
                    continue
                cursor += 1
                continue
            if current == "\n":
                line_index += 1
                if not triple:
                    # An unterminated single-quoted string: treat the line end
                    # as the end rather than swallowing the rest of the file.
                    break
                cursor += 1
                continue
            if source.startswith(marker, cursor):
                closed = True
                break
            cursor += 1

        inner_end = cursor
        for position in range(inner_start, min(inner_end, length)):
            if code[position] != "\n":
                code[position] = " "

        if closed:
            literals.append(
                Literal(
                    start=inner_start,
                    end=inner_end,
                    quote=quote,
                    triple=triple,
                    line=open_line,
                    depth=depth,
                    marked=bool(paren_marks) and paren_marks[-1],
                )
            )
            index = inner_end + len(marker)
        else:
            index = inner_end

    code_lines = "".join(code).split("\n")
    return literals, code_lines


def _prefix_is_speaker(prefix: str) -> bool:
    """True when the text before a say statement's dialogue is just a speaker.

    A say statement's prefix is the character variable plus any image
    attributes - all bare words. Anything with an operator or a call in it is
    some other statement that happens to contain a string.
    """
    if not prefix:
        return True
    for word in prefix.split():
        if word in _SAY_PREFIX_WORDS:
            continue
        if not _IDENT_RE.match(word):
            return False
    return True


def _line_bounds(source: str) -> list[int]:
    """Start offset of every line, so a line index can address the file."""
    starts = [0]
    for index, char in enumerate(source):
        if char == "\n":
            starts.append(index + 1)
    return starts


def find_translatable(source: str) -> list[Found]:
    """Return every translatable string in one .rpy file."""
    literals, code_lines = scan_literals(source)
    line_starts = _line_bounds(source)

    by_line: dict[int, list[Literal]] = {}
    for literal in literals:
        by_line.setdefault(literal.line, []).append(literal)

    found: list[Found] = []
    skip_indent: int | None = None
    skip_kind: str | None = None
    menu_indent: int | None = None
    continuation = False
    depth = 0
    # Lines that belong to a `translate` block - somebody else's finished
    # translation - so the marked-literal pass below can steer clear of it
    # the same way the main classification does.
    translate_lines: set[int] = set()

    for index, code_line in enumerate(code_lines):
        stripped = code_line.strip()
        line_depth_before = depth
        depth += sum(code_line.count(c) for c in "([{")
        depth -= sum(code_line.count(c) for c in ")]}")
        depth = max(0, depth)

        was_continuation = continuation
        # A statement continues onto the next line while brackets are open or
        # the line ends in a backslash.
        continuation = depth > 0 or code_line.rstrip().endswith("\\")

        if not stripped:
            continue

        indent = len(code_line) - len(code_line.lstrip())

        # A continuation line is part of the statement above it, not a
        # statement of its own - classifying it would read `"Eileen")` from a
        # multi-line Character() call as narration.
        if was_continuation or line_depth_before > 0:
            continue

        if skip_indent is not None:
            if indent > skip_indent:
                if skip_kind == "translate":
                    translate_lines.add(index)
                continue
            skip_indent = None
            skip_kind = None

        if menu_indent is not None and indent <= menu_indent:
            menu_indent = None

        if stripped.startswith("$"):
            continue

        match = _FIRST_WORD_RE.match(stripped)
        first = match.group(1) if match else ""

        if first in _BLOCK_SKIP:
            skip_indent = indent
            skip_kind = first
            if first == "translate":
                translate_lines.add(index)
            continue

        if first == "menu":
            menu_indent = indent
            continue

        if first == "screen":
            # Screens hold player-visible text, so only the header is skipped.
            continue

        candidates = [lit for lit in by_line.get(index, []) if lit.depth == 0]
        if not candidates:
            continue

        # `label` is two different statements: `label start:` is a script
        # label, `label "Text"` inside a screen is a caption.
        if first == "label" and not stripped[len("label"):].lstrip().startswith(
            ("'", '"')
        ):
            continue

        if first in _UI_KEYWORDS:
            chosen = candidates[0]
            kind = "ui"
        elif first in _LINE_SKIP:
            continue
        else:
            # Everything before the opening quote of the first string. The
            # quote itself is excluded - it is not part of the speaker.
            opening = candidates[0].start - (3 if candidates[0].triple else 1)
            prefix = code_line[: opening - line_starts[index]].strip()
            # An assignment that escaped the `$` and define checks is not
            # dialogue, whatever it looks like.
            if "=" in prefix:
                continue
            if not _prefix_is_speaker(prefix):
                continue
            # With two strings and no prefix the line is `"Speaker" "text"`,
            # so the dialogue is the last one either way.
            chosen = candidates[-1]
            kind = "menu" if menu_indent is not None and not prefix else "dialogue"

        item = _literal_to_found(source, chosen, kind, index + 1)
        if item is not None:
            found.append(item)

    # `_()` / `__()` is Ren'Py's own "translate this" marker, and it means
    # exactly that wherever it appears - inside a `python:`/`init python:`
    # block, an `Achievement(...)` constructor, a dict buried in a list two
    # levels deep. The scan above only classifies statements at bracket-depth
    # 0 (so a style name or an action sitting next to a label is never
    # mistaken for one), which is also why it cannot see into any of those
    # places. This second pass doesn't classify anything; the developer
    # already did, explicitly, by writing `_(...)`. Depth-0 literals can never
    # be marked (marking requires an open paren), so this cannot re-find
    # anything the loop above already collected.
    seen = {(f.start, f.end) for f in found}
    for literal in literals:
        if not literal.marked or literal.line in translate_lines:
            continue
        if (literal.start, literal.end) in seen:
            continue
        item = _literal_to_found(source, literal, "ui", literal.line + 1)
        if item is not None:
            found.append(item)

    return found


def _literal_to_found(source: str, literal: Literal, kind: str, line: int) -> Found | None:
    raw = source[literal.start : literal.end]
    try:
        value = unescape(raw)
    except EscapeError:
        # An escape this tool will not round-trip. Leaving the line in the
        # source untranslated is always recoverable; writing back a guess is
        # not.
        return None
    if not value.strip() or not is_translatable(value):
        return None
    if literal.triple:
        # Triple-quoted bodies are found and skipped rather than translated:
        # their content is re-indented by Ren'Py and a rewritten body would
        # change the layout it depends on.
        return None
    return Found(
        text=value,
        start=literal.start,
        end=literal.end,
        quote=literal.quote,
        kind=kind,
        line=line,
    )