Token Classification
Transformers
ONNX
Safetensors
English
Japanese
Chinese
bert
anime
filename-parsing
Eval Results (legacy)
Instructions to use ModerRAS/AniFileBERT with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use ModerRAS/AniFileBERT with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("token-classification", model="ModerRAS/AniFileBERT")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForTokenClassification tokenizer = AutoTokenizer.from_pretrained("ModerRAS/AniFileBERT") model = AutoModelForTokenClassification.from_pretrained("ModerRAS/AniFileBERT", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Include title position literals in Rust recipes
Browse files
tools/rust_dmhy_template_apply/src/main.rs
CHANGED
|
@@ -947,6 +947,8 @@ fn recipe_row(row: &Value, confidence: &str) -> Value {
|
|
| 947 |
"roles": row["suggested_roles"],
|
| 948 |
"title_spans": row.get("title_spans").cloned().unwrap_or_else(|| json!([])),
|
| 949 |
"title_boundary_decisions": row.get("title_boundary_decisions").cloned().unwrap_or_else(|| json!([])),
|
|
|
|
|
|
|
| 950 |
"confidence": confidence,
|
| 951 |
"count": row["count"],
|
| 952 |
"examples": row["examples"],
|
|
@@ -1328,25 +1330,36 @@ fn title_boundary_review_prompt(recipe: &Value, decision: &Value) -> String {
|
|
| 1328 |
.and_then(Value::as_array)
|
| 1329 |
.map(|items| items.iter().take(8).cloned().collect::<Vec<_>>())
|
| 1330 |
.unwrap_or_default();
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1331 |
let payload = json!({
|
| 1332 |
-
"task": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1333 |
"allowed_decisions": {
|
| 1334 |
"merge": "same title continuation; use one logical title span",
|
| 1335 |
-
"keep": "separate title slots such as duplicate/path title, alias, multilingual title, or distinct fields",
|
| 1336 |
"skip": "uncertain or unsafe for synthetic replacement"
|
| 1337 |
},
|
| 1338 |
"template_id": recipe.get("template_id"),
|
| 1339 |
"template": recipe.get("template"),
|
| 1340 |
"roles": recipe.get("roles"),
|
| 1341 |
-
"boundary":
|
| 1342 |
"left_position_top_literals": left_literals,
|
| 1343 |
"right_position_top_literals": right_literals,
|
| 1344 |
"examples": examples,
|
| 1345 |
"output_schema": {
|
| 1346 |
"decision": "merge|keep|skip",
|
| 1347 |
-
"kind": "same_title_continuation|duplicate_title|multilingual_alias|path_duplicate|distinct_fields|uncertain",
|
| 1348 |
"confidence": "number from 0 to 1",
|
| 1349 |
-
"reason": "short explanation"
|
| 1350 |
}
|
| 1351 |
});
|
| 1352 |
format!(
|
|
|
|
| 947 |
"roles": row["suggested_roles"],
|
| 948 |
"title_spans": row.get("title_spans").cloned().unwrap_or_else(|| json!([])),
|
| 949 |
"title_boundary_decisions": row.get("title_boundary_decisions").cloned().unwrap_or_else(|| json!([])),
|
| 950 |
+
"position_top_literals": row.get("position_top_literals").cloned().unwrap_or_else(|| json!([])),
|
| 951 |
+
"top_literals": row.get("top_literals").cloned().unwrap_or_else(|| json!([])),
|
| 952 |
"confidence": confidence,
|
| 953 |
"count": row["count"],
|
| 954 |
"examples": row["examples"],
|
|
|
|
| 1330 |
.and_then(Value::as_array)
|
| 1331 |
.map(|items| items.iter().take(8).cloned().collect::<Vec<_>>())
|
| 1332 |
.unwrap_or_default();
|
| 1333 |
+
let boundary = json!({
|
| 1334 |
+
"left_role_index": left,
|
| 1335 |
+
"right_role_index": right,
|
| 1336 |
+
"literal_similarity": decision.get("similarity").cloned().unwrap_or(Value::Null),
|
| 1337 |
+
});
|
| 1338 |
let payload = json!({
|
| 1339 |
+
"task": "Independently classify whether two adjacent TITLE slots in an anime filename template should be one logical title span or kept separate.",
|
| 1340 |
+
"rules": [
|
| 1341 |
+
"Choose merge only when the two slots are contiguous parts of the same title/subtitle phrase.",
|
| 1342 |
+
"Choose keep when slots are duplicate titles, path/leaf repeats, multilingual aliases, category tags plus title, or distinct title candidates.",
|
| 1343 |
+
"Choose skip when examples/top literals are mixed or insufficient.",
|
| 1344 |
+
"Do not infer from any heuristic decision; use examples and position literals."
|
| 1345 |
+
],
|
| 1346 |
"allowed_decisions": {
|
| 1347 |
"merge": "same title continuation; use one logical title span",
|
| 1348 |
+
"keep": "separate title slots such as duplicate/path title, alias, multilingual title, category tag plus title, or distinct fields",
|
| 1349 |
"skip": "uncertain or unsafe for synthetic replacement"
|
| 1350 |
},
|
| 1351 |
"template_id": recipe.get("template_id"),
|
| 1352 |
"template": recipe.get("template"),
|
| 1353 |
"roles": recipe.get("roles"),
|
| 1354 |
+
"boundary": boundary,
|
| 1355 |
"left_position_top_literals": left_literals,
|
| 1356 |
"right_position_top_literals": right_literals,
|
| 1357 |
"examples": examples,
|
| 1358 |
"output_schema": {
|
| 1359 |
"decision": "merge|keep|skip",
|
| 1360 |
+
"kind": "same_title_continuation|duplicate_title|multilingual_alias|path_duplicate|category_plus_title|distinct_fields|uncertain",
|
| 1361 |
"confidence": "number from 0 to 1",
|
| 1362 |
+
"reason": "short explanation citing examples or top literals"
|
| 1363 |
}
|
| 1364 |
});
|
| 1365 |
format!(
|