Add Rust LM Studio title-boundary reviewer

tools/rust_dmhy_template_apply/README.md

@@ -53,6 +53,22 @@ cargo run --release --manifest-path tools\rust_dmhy_template_apply\Cargo.toml --
   --audit-max-count 50
 ```
 
+Review ambiguous title-boundary decisions with a local LM Studio model. This is
+an optional final-pass reviewer: it writes a decision cache JSONL and does not
+modify recipes or training data directly.
+
+```powershell
+cargo run --release --manifest-path tools\rust_dmhy_template_apply\Cargo.toml -- `
+  --review-title-boundaries-lmstudio `
+  --recipes reports\dmhy_template_recipes.full_top5000.seed.jsonl `
+  --title-boundary-decisions-output reports\dmhy_title_boundary_lmstudio_decisions.jsonl `
+  --lmstudio-base-url http://localhost:1234/v1 `
+  --lmstudio-model qwen `
+  --title-boundary-min-similarity 0.25 `
+  --title-boundary-max-similarity 0.85 `
+  --limit 50
+```
+
 Optional controls:
 
 ```powershell

tools/rust_dmhy_template_apply/src/main.rs

@@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize};
 use serde_json::{json, Value};
 use std::collections::{HashMap, HashSet};
 use std::fs::{self, File};
-use std::io::{BufRead, BufReader, BufWriter, Write};
+use std::io::{BufRead, BufReader, BufWriter, Read, Write};
 use std::path::PathBuf;
 use std::sync::atomic::{AtomicUsize, Ordering};
 
@@ -23,6 +23,8 @@ struct Args {
     verify_generated_output: bool,
     #[arg(long)]
     rich_annotations: bool,
+    #[arg(long)]
+    review_title_boundaries_lmstudio: bool,
     #[arg(long, default_value = "datasets/AnimeName/dmhy_list.jsonl")]
     input: PathBuf,
     #[arg(long, default_value = "reports/dmhy_template_recipes.seed.jsonl")]
@@ -57,6 +59,23 @@ struct Args {
     audit_output: PathBuf,
     #[arg(long, default_value = "reports/dmhy_rich_annotations.rust.jsonl")]
     rich_output: PathBuf,
+    #[arg(
+        long,
+        default_value = "reports/dmhy_title_boundary_lmstudio_decisions.jsonl"
+    )]
+    title_boundary_decisions_output: PathBuf,
+    #[arg(long, default_value = "http://localhost:1234/v1")]
+    lmstudio_base_url: String,
+    #[arg(long, default_value = "qwen")]
+    lmstudio_model: String,
+    #[arg(long, default_value_t = 0.25)]
+    title_boundary_min_similarity: f64,
+    #[arg(long, default_value_t = 0.85)]
+    title_boundary_max_similarity: f64,
+    #[arg(long, default_value_t = 0.0)]
+    lmstudio_temperature: f64,
+    #[arg(long, default_value_t = 120)]
+    lmstudio_timeout_seconds: u64,
     #[arg(long, default_value_t = 50)]
     audit_max_count: u64,
     #[arg(long)]
@@ -310,6 +329,9 @@ fn main() -> Result<()> {
     if args.rich_annotations {
         return run_rich_annotations(&args);
     }
+    if args.review_title_boundaries_lmstudio {
+        return run_review_title_boundaries_lmstudio(&args);
+    }
     if args.expand != "all" && args.expand != "sample" {
         bail!("--expand must be all or sample");
     }
@@ -1147,6 +1169,285 @@ fn run_rich_annotations(args: &Args) -> Result<()> {
     Ok(())
 }
 
+fn run_review_title_boundaries_lmstudio(args: &Args) -> Result<()> {
+    if !(0.0..=1.0).contains(&args.title_boundary_min_similarity)
+        || !(0.0..=1.0).contains(&args.title_boundary_max_similarity)
+        || args.title_boundary_min_similarity > args.title_boundary_max_similarity
+    {
+        bail!("title boundary similarity thresholds must satisfy 0 <= min <= max <= 1");
+    }
+    let recipes = load_recipe_values(&args.recipes)?;
+    if recipes.is_empty() {
+        bail!("no recipe rows found in {}", args.recipes.display());
+    }
+    if let Some(parent) = args.title_boundary_decisions_output.parent() {
+        fs::create_dir_all(parent)?;
+    }
+    let mut writer = BufWriter::new(File::create(&args.title_boundary_decisions_output)?);
+    let mut reviewed = 0usize;
+    let mut skipped = 0usize;
+    let endpoint = format!(
+        "{}/chat/completions",
+        args.lmstudio_base_url.trim_end_matches('/')
+    );
+
+    for recipe in recipes {
+        let template_id = recipe
+            .get("template_id")
+            .and_then(Value::as_str)
+            .unwrap_or("unknown");
+        let decisions = recipe
+            .get("title_boundary_decisions")
+            .and_then(Value::as_array)
+            .cloned()
+            .unwrap_or_default();
+        for decision in decisions {
+            let similarity = decision
+                .get("similarity")
+                .and_then(Value::as_f64)
+                .unwrap_or(0.0);
+            let heuristic_decision = decision
+                .get("decision")
+                .and_then(Value::as_str)
+                .unwrap_or("unknown");
+            if heuristic_decision.contains("path_boundary")
+                || heuristic_decision.contains("structural_boundary")
+                || similarity < args.title_boundary_min_similarity
+                || similarity > args.title_boundary_max_similarity
+            {
+                skipped += 1;
+                continue;
+            }
+            if let Some(limit) = args.limit {
+                if reviewed >= limit {
+                    break;
+                }
+            }
+            let prompt = title_boundary_review_prompt(&recipe, &decision);
+            let response = call_lmstudio_title_boundary(
+                &endpoint,
+                &args.lmstudio_model,
+                args.lmstudio_temperature,
+                args.lmstudio_timeout_seconds,
+                &prompt,
+            )
+            .with_context(|| {
+                format!(
+                    "LM Studio title-boundary review failed for template {template_id} via {endpoint}"
+                )
+            })?;
+            let content = response
+                .get("choices")
+                .and_then(Value::as_array)
+                .and_then(|choices| choices.first())
+                .and_then(|choice| choice.get("message"))
+                .and_then(|message| message.get("content"))
+                .and_then(Value::as_str)
+                .unwrap_or("");
+            let parsed = parse_lmstudio_json_object(content).unwrap_or_else(|| {
+                json!({
+                    "decision": "skip",
+                    "kind": "parse_error",
+                    "confidence": 0.0,
+                    "reason": content,
+                })
+            });
+            let output = json!({
+                "template_id": recipe.get("template_id").cloned().unwrap_or(Value::Null),
+                "template": recipe.get("template").cloned().unwrap_or(Value::Null),
+                "left_role_index": decision.get("left_role_index").cloned().unwrap_or(Value::Null),
+                "right_role_index": decision.get("right_role_index").cloned().unwrap_or(Value::Null),
+                "similarity": similarity,
+                "heuristic_decision": heuristic_decision,
+                "model_decision": parsed.get("decision").cloned().unwrap_or(Value::Null),
+                "model_kind": parsed.get("kind").cloned().unwrap_or(Value::Null),
+                "confidence": parsed.get("confidence").cloned().unwrap_or(Value::Null),
+                "reason": parsed.get("reason").cloned().unwrap_or(Value::Null),
+                "source": "lmstudio",
+                "model": args.lmstudio_model,
+                "reviewed_at": Utc::now().to_rfc3339(),
+            });
+            serde_json::to_writer(&mut writer, &output)?;
+            writer.write_all(b"\n")?;
+            reviewed += 1;
+        }
+    }
+    writer.flush()?;
+    let summary = json!({
+        "recipes": args.recipes.to_string_lossy(),
+        "output": args.title_boundary_decisions_output.to_string_lossy(),
+        "reviewed": reviewed,
+        "skipped": skipped,
+        "model": args.lmstudio_model,
+        "base_url": args.lmstudio_base_url,
+        "similarity_range": [args.title_boundary_min_similarity, args.title_boundary_max_similarity],
+        "generated_at": Utc::now().to_rfc3339(),
+    });
+    println!("{}", serde_json::to_string_pretty(&summary)?);
+    Ok(())
+}
+
+fn load_recipe_values(path: &PathBuf) -> Result<Vec<Value>> {
+    let file = File::open(path).with_context(|| format!("failed to open {}", path.display()))?;
+    let reader = BufReader::new(file);
+    let mut rows = Vec::new();
+    for (line_number, line) in reader.lines().enumerate() {
+        let line = line.with_context(|| format!("failed to read {}", path.display()))?;
+        if line.trim().is_empty() {
+            continue;
+        }
+        let value: Value = serde_json::from_str(&line)
+            .with_context(|| format!("invalid JSON at {}:{}", path.display(), line_number + 1))?;
+        rows.push(value);
+    }
+    Ok(rows)
+}
+
+fn title_boundary_review_prompt(recipe: &Value, decision: &Value) -> String {
+    let left = decision
+        .get("left_role_index")
+        .and_then(Value::as_u64)
+        .unwrap_or(0) as usize;
+    let right = decision
+        .get("right_role_index")
+        .and_then(Value::as_u64)
+        .unwrap_or(0) as usize;
+    let position_literals = recipe
+        .get("position_top_literals")
+        .and_then(Value::as_array);
+    let left_literals = position_literals
+        .and_then(|items| items.get(left))
+        .cloned()
+        .unwrap_or_else(|| json!([]));
+    let right_literals = position_literals
+        .and_then(|items| items.get(right))
+        .cloned()
+        .unwrap_or_else(|| json!([]));
+    let examples = recipe
+        .get("examples")
+        .and_then(Value::as_array)
+        .map(|items| items.iter().take(8).cloned().collect::<Vec<_>>())
+        .unwrap_or_default();
+    let payload = json!({
+        "task": "Classify whether two adjacent TITLE slots in an anime filename template should be one logical title span or kept separate.",
+        "allowed_decisions": {
+            "merge": "same title continuation; use one logical title span",
+            "keep": "separate title slots such as duplicate/path title, alias, multilingual title, or distinct fields",
+            "skip": "uncertain or unsafe for synthetic replacement"
+        },
+        "template_id": recipe.get("template_id"),
+        "template": recipe.get("template"),
+        "roles": recipe.get("roles"),
+        "boundary": decision,
+        "left_position_top_literals": left_literals,
+        "right_position_top_literals": right_literals,
+        "examples": examples,
+        "output_schema": {
+            "decision": "merge|keep|skip",
+            "kind": "same_title_continuation|duplicate_title|multilingual_alias|path_duplicate|distinct_fields|uncertain",
+            "confidence": "number from 0 to 1",
+            "reason": "short explanation"
+        }
+    });
+    format!(
+        "Return only one compact JSON object matching output_schema. No markdown.\n{}",
+        serde_json::to_string_pretty(&payload).unwrap_or_else(|_| "{}".to_string())
+    )
+}
+
+fn call_lmstudio_title_boundary(
+    endpoint: &str,
+    model: &str,
+    temperature: f64,
+    timeout_seconds: u64,
+    prompt: &str,
+) -> Result<Value> {
+    let body = json!({
+        "model": model,
+        "temperature": temperature,
+        "messages": [
+            {
+                "role": "system",
+                "content": "You are a strict data-labeling reviewer for anime release filename templates. Reply with JSON only."
+            },
+            {"role": "user", "content": prompt}
+        ]
+    });
+    post_json_http(endpoint, &body, timeout_seconds)
+}
+
+fn post_json_http(endpoint: &str, body: &Value, timeout_seconds: u64) -> Result<Value> {
+    let (host, port, path) = parse_http_endpoint(endpoint)?;
+    let host_port = format!("{host}:{port}");
+    let mut stream = std::net::TcpStream::connect((&*host, port))
+        .with_context(|| format!("failed to connect to {host_port}"))?;
+    let timeout = std::time::Duration::from_secs(timeout_seconds);
+    stream.set_read_timeout(Some(timeout))?;
+    stream.set_write_timeout(Some(timeout))?;
+    let body_text = serde_json::to_string(body)?;
+    let request = format!(
+        "POST {path} HTTP/1.1\r\nHost: {host_port}\r\nContent-Type: application/json\r\nAccept: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
+        body_text.len(),
+        body_text
+    );
+    stream.write_all(request.as_bytes())?;
+    let mut response = String::new();
+    stream.read_to_string(&mut response)?;
+    let (head, response_body) = response
+        .split_once("\r\n\r\n")
+        .ok_or_else(|| anyhow::anyhow!("invalid HTTP response from {endpoint}"))?;
+    let status_line = head.lines().next().unwrap_or_default();
+    let status_code = status_line
+        .split_whitespace()
+        .nth(1)
+        .and_then(|value| value.parse::<u16>().ok())
+        .unwrap_or(0);
+    if !(200..300).contains(&status_code) {
+        bail!("LM Studio returned HTTP {status_code}: {response_body}");
+    }
+    serde_json::from_str(response_body).with_context(|| "invalid JSON response from LM Studio")
+}
+
+fn parse_http_endpoint(endpoint: &str) -> Result<(String, u16, String)> {
+    let rest = endpoint.strip_prefix("http://").ok_or_else(|| {
+        anyhow::anyhow!("only http:// LM Studio endpoints are supported: {endpoint}")
+    })?;
+    let (authority, path) = match rest.split_once('/') {
+        Some((authority, path)) => (authority, format!("/{path}")),
+        None => (rest, "/".to_string()),
+    };
+    if authority.is_empty() {
+        bail!("missing host in endpoint {endpoint}");
+    }
+    let (host, port) = match authority.rsplit_once(':') {
+        Some((host, port)) if !host.is_empty() => {
+            let port = port
+                .parse::<u16>()
+                .with_context(|| format!("invalid port in {endpoint}"))?;
+            (host.to_string(), port)
+        }
+        _ => (authority.to_string(), 80),
+    };
+    Ok((host, port, path))
+}
+
+fn parse_lmstudio_json_object(content: &str) -> Option<Value> {
+    if let Ok(value) = serde_json::from_str::<Value>(content.trim()) {
+        return if value.is_object() { Some(value) } else { None };
+    }
+    let start = content.find('{')?;
+    let end = content.rfind('}')?;
+    if end <= start {
+        return None;
+    }
+    let value = serde_json::from_str::<Value>(&content[start..=end]).ok()?;
+    if value.is_object() {
+        Some(value)
+    } else {
+        None
+    }
+}
+
 fn rich_annotation_for(original: &str) -> Value {
     let (training_filename, path_trimmed) = training_filename_for(original);
     let parts: Vec<&str> = original