fix(dataset): repair training-set alignment + relabel teneighty as GROUP; add Rust repair tool

.gitignore

@@ -22,3 +22,8 @@ reports/
 reports/generated/
 target/
 **/target/
+
+# 本地工作/审计产物，不入库
+.workbuddy/
+data/repair_report.json
+data/dmhy_weak_char.repaired.manifest.json

datasets/AnimeName

@@ -1 +1 @@
-Subproject commit 6d94ea67d836aa79325e280f4600f91689ba0e4a
+Subproject commit 061e59eddd03a157cb6b2deb13c4dc964630d4a4

tools/repair_bracket_mismatch/Cargo.lock

@@ -0,0 +1,256 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "anifilebert-repair-bracket-mismatch"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "clap",
+ "serde",
+ "serde_json",
+]
+
+[[package]]
+name = "anstream"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
+dependencies = [
+ "anstyle",
+ "anstyle-parse",
+ "anstyle-query",
+ "anstyle-wincon",
+ "colorchoice",
+ "is_terminal_polyfill",
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle"
+version = "1.0.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
+
+[[package]]
+name = "anstyle-parse"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
+dependencies = [
+ "utf8parse",
+]
+
+[[package]]
+name = "anstyle-query"
+version = "1.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
+dependencies = [
+ "windows-sys",
+]
+
+[[package]]
+name = "anstyle-wincon"
+version = "3.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
+dependencies = [
+ "anstyle",
+ "once_cell_polyfill",
+ "windows-sys",
+]
+
+[[package]]
+name = "anyhow"
+version = "1.0.104"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
+
+[[package]]
+name = "clap"
+version = "4.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "301b56658598e48f3648647ac6fc887be7e7108eddfa4e9b63fcf3ec58c0cadf"
+dependencies = [
+ "clap_builder",
+ "clap_derive",
+]
+
+[[package]]
+name = "clap_builder"
+version = "4.6.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94a65403d1a1bd28f7dc68eb8506e8874808ee5eecb59298de588e2e1407a078"
+dependencies = [
+ "anstream",
+ "anstyle",
+ "clap_lex",
+ "strsim",
+]
+
+[[package]]
+name = "clap_derive"
+version = "4.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "clap_lex"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
+
+[[package]]
+name = "colorchoice"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "is_terminal_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
+
+[[package]]
+name = "itoa"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
+
+[[package]]
+name = "memchr"
+version = "2.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98"
+
+[[package]]
+name = "once_cell_polyfill"
+version = "1.70.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.107"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.47"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba"
+dependencies = [
+ "serde_core",
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_core"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.229"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.151"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14"
+dependencies = [
+ "itoa",
+ "memchr",
+ "serde",
+ "serde_core",
+ "zmij",
+]
+
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
+[[package]]
+name = "syn"
+version = "3.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75"
+
+[[package]]
+name = "utf8parse"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
+
+[[package]]
+name = "windows-link"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
+
+[[package]]
+name = "windows-sys"
+version = "0.61.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
+dependencies = [
+ "windows-link",
+]
+
+[[package]]
+name = "zmij"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b"

tools/repair_bracket_mismatch/Cargo.toml

@@ -0,0 +1,10 @@
+[package]
+name = "anifilebert-repair-bracket-mismatch"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+anyhow = "1.0"
+clap = { version = "4.5", features = ["derive"] }
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"

tools/repair_bracket_mismatch/src/main.rs

@@ -0,0 +1,428 @@
+use std::collections::BTreeMap;
+use std::fs::{self, File};
+use std::io::{self, BufRead, BufReader, BufWriter, Write};
+use std::path::PathBuf;
+use std::sync::LazyLock;
+
+use anyhow::{Context, Result};
+use clap::Parser;
+use serde::Deserialize;
+use serde_json::{json, Value};
+
+#[derive(Parser)]
+#[command(about = "Repair bracket/case misalignment and mislabeled release groups in the DMHY char-level dataset")]
+struct Args {
+    #[arg(long, default_value = "datasets\\AnimeName\\dmhy_weak_char.jsonl")]
+    input: PathBuf,
+    #[arg(long, default_value = "data\\dmhy_weak_char.repaired.jsonl")]
+    output: PathBuf,
+    #[arg(long, default_value = "data\\repair_report.json")]
+    report: PathBuf,
+    /// Compute and print statistics without writing the repaired dataset.
+    #[arg(long)]
+    dry_run: bool,
+}
+
+#[derive(Deserialize)]
+struct Record {
+    filename: String,
+    #[serde(default)]
+    tokens: Vec<String>,
+    labels: Vec<String>,
+    #[serde(default)]
+    template: Option<String>,
+    #[serde(default)]
+    tokenizer_variant: Option<String>,
+}
+
+/// Release groups that appear un-bracketed as a filename prefix and were found
+/// mislabeled as TITLE. Extend this list as more groups are confirmed.
+static KNOWN_GROUPS: LazyLock<Vec<String>> = LazyLock::new(|| vec!["teneighty".to_string()]);
+
+#[derive(Clone, Copy, PartialEq)]
+enum AlignKind {
+    Bracket,
+    Case,
+}
+
+#[derive(Default)]
+struct Stats {
+    total: usize,
+    already_ok: usize,
+    normalized: usize,
+    repaired_bracket: usize,
+    repaired_case: usize,
+    unrepairable: usize,
+    group_relabeled: usize,
+    examples: BTreeMap<String, Vec<Value>>,
+}
+
+fn main() -> Result<()> {
+    let args = Args::parse();
+
+    let input = File::open(&args.input)
+        .with_context(|| format!("failed to open {}", args.input.display()))?;
+    let reader = BufReader::new(input);
+
+    if !args.dry_run {
+        if let Some(parent) = args.output.parent() {
+            if !parent.as_os_str().is_empty() {
+                fs::create_dir_all(parent)
+                    .with_context(|| format!("failed to create {}", parent.display()))?;
+            }
+        }
+    }
+
+    let mut out: Box<dyn Write> = if args.dry_run {
+        Box::new(io::sink())
+    } else {
+        let f = File::create(&args.output)
+            .with_context(|| format!("failed to create {}", args.output.display()))?;
+        Box::new(BufWriter::new(f))
+    };
+
+    let mut stats = Stats::default();
+
+    for (lineno, line) in reader.lines().enumerate() {
+        let line = line.with_context(|| format!("failed to read line {}", lineno + 1))?;
+        if line.trim().is_empty() {
+            continue;
+        }
+        let rec: Record = serde_json::from_str(&line)
+            .with_context(|| format!("failed to parse line {}", lineno + 1))?;
+        stats.total += 1;
+
+        let old_joined: String = rec.tokens.concat();
+        let fn_chars: Vec<char> = rec.filename.chars().collect();
+        let perfectly_aligned = rec.filename == old_joined
+            && rec.tokens.len() == fn_chars.len()
+            && rec.labels.len() == rec.tokens.len();
+
+        if perfectly_aligned {
+            let new_tokens = rec.tokens.clone();
+            let mut new_labels = rec.labels.clone();
+            if relabel_groups(&new_tokens, &mut new_labels) {
+                stats.group_relabeled += 1;
+                push_example(
+                    &mut stats,
+                    "group_relabeled",
+                    &rec.filename,
+                    &rec.tokens,
+                    &rec.labels,
+                    &new_tokens,
+                    &new_labels,
+                );
+            }
+            stats.already_ok += 1;
+            if !args.dry_run {
+                write_record(
+                    &mut out,
+                    &rec.filename,
+                    &new_tokens,
+                    &new_labels,
+                    rec.template.as_deref(),
+                    rec.tokenizer_variant.as_deref(),
+                )?;
+            }
+            continue;
+        }
+
+        match align_to_filename(&rec.filename, &rec.tokens, &rec.labels) {
+            Some((new_tokens, new_labels, kind)) => {
+                let mut final_labels = new_labels;
+                let grp = relabel_groups(&new_tokens, &mut final_labels);
+
+                if rec.filename == old_joined && rec.tokens.len() != fn_chars.len() {
+                    // Content identical but token granularity was not char-level.
+                    stats.normalized += 1;
+                    push_example(
+                        &mut stats,
+                        "normalized",
+                        &rec.filename,
+                        &rec.tokens,
+                        &rec.labels,
+                        &new_tokens,
+                        &final_labels,
+                    );
+                } else if kind == AlignKind::Case {
+                    stats.repaired_case += 1;
+                    push_example(
+                        &mut stats,
+                        "repaired_case",
+                        &rec.filename,
+                        &rec.tokens,
+                        &rec.labels,
+                        &new_tokens,
+                        &final_labels,
+                    );
+                } else {
+                    stats.repaired_bracket += 1;
+                    push_example(
+                        &mut stats,
+                        "repaired_bracket",
+                        &rec.filename,
+                        &rec.tokens,
+                        &rec.labels,
+                        &new_tokens,
+                        &final_labels,
+                    );
+                }
+
+                if grp {
+                    stats.group_relabeled += 1;
+                    push_example(
+                        &mut stats,
+                        "group_relabeled",
+                        &rec.filename,
+                        &rec.tokens,
+                        &rec.labels,
+                        &new_tokens,
+                        &final_labels,
+                    );
+                }
+
+                if !args.dry_run {
+                    write_record(
+                        &mut out,
+                        &rec.filename,
+                        &new_tokens,
+                        &final_labels,
+                        rec.template.as_deref(),
+                        rec.tokenizer_variant.as_deref(),
+                    )?;
+                }
+            }
+            None => {
+                stats.unrepairable += 1;
+                push_example(
+                    &mut stats,
+                    "unrepairable",
+                    &rec.filename,
+                    &rec.tokens,
+                    &rec.labels,
+                    &rec.tokens,
+                    &rec.labels,
+                );
+                if !args.dry_run {
+                    write_record(
+                        &mut out,
+                        &rec.filename,
+                        &rec.tokens,
+                        &rec.labels,
+                        rec.template.as_deref(),
+                        rec.tokenizer_variant.as_deref(),
+                    )?;
+                }
+            }
+        }
+    }
+    out.flush()?;
+
+    let summary = json!({
+        "total": stats.total,
+        "already_ok": stats.already_ok,
+        "normalized": stats.normalized,
+        "repaired_bracket": stats.repaired_bracket,
+        "repaired_case": stats.repaired_case,
+        "unrepairable": stats.unrepairable,
+        "group_relabeled": stats.group_relabeled,
+    });
+
+    if args.dry_run {
+        let report = json!({
+            "input": args.input.display().to_string(),
+            "dry_run": true,
+            "summary": summary,
+            "examples": stats.examples,
+        });
+        println!("{}", serde_json::to_string_pretty(&report)?);
+    } else {
+        let report = json!({
+            "input": args.input.display().to_string(),
+            "output": args.output.display().to_string(),
+            "dry_run": false,
+            "summary": summary,
+            "examples": stats.examples,
+        });
+        let rf = File::create(&args.report)
+            .with_context(|| format!("failed to create {}", args.report.display()))?;
+        serde_json::to_writer_pretty(BufWriter::new(rf), &report)?;
+        println!("{}", serde_json::to_string_pretty(&summary)?);
+        println!("wrote repaired dataset -> {}", args.output.display());
+        println!("wrote report -> {}", args.report.display());
+    }
+
+    Ok(())
+}
+
+/// Reconstruct character-level tokens/labels from `filename`, projecting the
+/// old character-level labels onto the filename characters. Characters present
+/// in `filename` but absent from the joined old tokens (e.g. dropped brackets)
+/// become `O`; case-only differences are tolerated.
+fn align_to_filename(
+    filename: &str,
+    old_tokens: &[String],
+    old_labels: &[String],
+) -> Option<(Vec<String>, Vec<String>, AlignKind)> {
+    let mut old_chars: Vec<char> = Vec::new();
+    for t in old_tokens {
+        for c in t.chars() {
+            old_chars.push(c);
+        }
+    }
+    if old_chars.len() != old_labels.len() {
+        return None;
+    }
+    let fn_chars: Vec<char> = filename.chars().collect();
+    if fn_chars.len() < old_chars.len() {
+        // filename is missing characters relative to the labels -> cannot repair
+        return None;
+    }
+
+    let mut new_labels = Vec::with_capacity(fn_chars.len());
+    let mut o_ptr = 0usize;
+    let mut had_case = false;
+
+    for &ch in &fn_chars {
+        if o_ptr < old_chars.len() && old_chars[o_ptr] == ch {
+            new_labels.push(old_labels[o_ptr].clone());
+            o_ptr += 1;
+        } else if o_ptr < old_chars.len()
+            && old_chars[o_ptr].to_lowercase().eq(ch.to_lowercase())
+            && old_chars[o_ptr] != ch
+        {
+            // case-only difference (e.g. filename `s02e01` vs labels `S02E01`)
+            new_labels.push(old_labels[o_ptr].clone());
+            o_ptr += 1;
+            had_case = true;
+        } else {
+            // character inserted relative to old (typically a bracket)
+            new_labels.push("O".to_string());
+        }
+    }
+
+    if o_ptr != old_chars.len() {
+        // some old characters were never matched -> real deletion -> unrepairable
+        return None;
+    }
+
+    let new_tokens = fn_chars.iter().map(|c| c.to_string()).collect();
+    let kind = if had_case {
+        AlignKind::Case
+    } else {
+        AlignKind::Bracket
+    };
+    Some((new_tokens, new_labels, kind))
+}
+
+/// Rewrite a leading known release-group span (currently labeled TITLE_*) as
+/// GROUP. Returns true if any labels were changed.
+fn relabel_groups(tokens: &[String], labels: &mut [String]) -> bool {
+    let n = tokens.len();
+    if n == 0 {
+        return false;
+    }
+    let start = match (0..n).find(|&i| labels[i] != "O") {
+        Some(s) => s,
+        None => return false,
+    };
+    let lab0 = labels[start].clone();
+    if !lab0.starts_with("B-") {
+        // Not at the start of a BIO span; leave it alone.
+        return false;
+    }
+    let entity = lab0[2..].to_string();
+    let continuation = format!("I-{entity}");
+    let mut end = start;
+    while end + 1 < n && labels[end + 1] == continuation {
+        end += 1;
+    }
+    let span_text: String = tokens[start..=end].concat();
+    if !KNOWN_GROUPS.iter().any(|g| g == &span_text) {
+        return false;
+    }
+    labels[start] = "B-GROUP".to_string();
+    for i in (start + 1)..=end {
+        labels[i] = "I-GROUP".to_string();
+    }
+    true
+}
+
+fn write_record<W: Write>(
+    w: &mut W,
+    filename: &str,
+    tokens: &[String],
+    labels: &[String],
+    template: Option<&str>,
+    variant: Option<&str>,
+) -> Result<()> {
+    let obj = json!({
+        "filename": filename,
+        "tokens": tokens,
+        "labels": labels,
+        "template": template,
+        "tokenizer_variant": variant,
+    });
+    serde_json::to_writer(&mut *w, &obj)?;
+    w.write_all(b"\n")?;
+    Ok(())
+}
+
+fn push_example(
+    stats: &mut Stats,
+    key: &str,
+    filename: &str,
+    before_tokens: &[String],
+    before_labels: &[String],
+    after_tokens: &[String],
+    after_labels: &[String],
+) {
+    let arr = stats.examples.entry(key.to_string()).or_default();
+    if arr.len() >= 25 {
+        return;
+    }
+    arr.push(json!({
+        "filename": filename,
+        "before": spans_text(before_tokens, before_labels),
+        "after": spans_text(after_tokens, after_labels),
+    }));
+}
+
+fn spans_text(tokens: &[String], labels: &[String]) -> String {
+    let mut out = String::new();
+    let mut i = 0;
+    while i < labels.len() {
+        let lab = labels[i].clone();
+        if lab == "O" {
+            let mut j = i;
+            while j + 1 < labels.len() && labels[j + 1] == "O" {
+                j += 1;
+            }
+            let text: String = tokens[i..=j].concat();
+            out.push_str(&format!(" {text}[O]"));
+            i = j + 1;
+            continue;
+        }
+        if !lab.starts_with("B-") {
+            // lone I- without a preceding B-: render as its own span
+            let mut j = i;
+            while j + 1 < labels.len() && labels[j + 1] == lab {
+                j += 1;
+            }
+            let text: String = tokens[i..=j].concat();
+            out.push_str(&format!(" {text}[{lab}]"));
+            i = j + 1;
+            continue;
+        }
+        let entity = lab[2..].to_string();
+        let continuation = format!("I-{entity}");
+        let mut j = i;
+        while j + 1 < labels.len() && labels[j + 1] == continuation {
+            j += 1;
+        }
+        let text: String = tokens[i..=j].concat();
+        out.push_str(&format!(" {text}[{lab}]"));
+        i = j + 1;
+    }
+    out.trim_start().to_string()
+}