Create ner_logic.py

ner_logic.py

@@ -0,0 +1,46 @@
+import torch
+import wikipedia
+import re
+import json
+import spaces
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
+from labels_config import CUSTOM_LABELS
+
+# Modellerin Yüklenmesi
+ner_pipe = pipeline("ner", model="xlm-roberta-large-finetuned-conll03-english", aggregation_strategy="simple", device=-1)
+model_id = "Qwen/Qwen2.5-1.5B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(model_id)
+llm_model = AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map="auto", trust_remote_code=True)
+
+def get_wiki_summary(term):
+    try:
+        wikipedia.set_lang("tr")
+        search_results = wikipedia.search(term)
+        if not search_results: return None
+        return wikipedia.summary(search_results[0], sentences=2, auto_suggest=False)
+    except: return None
+
+@spaces.GPU
+def refine_labels_batch(misc_items, full_sentence):
+    if not misc_items: return []
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    
+    label_rules = "".join([f"### {k}\n- TANIM: {v['tanim']}\n" for k, v in CUSTOM_LABELS.items()])
+    targets_text = "".join([f"\nVARLIK {i+1}: {item['word']}\nWIKI: {item['wiki']}\n" for i, item in enumerate(misc_items)])
+
+    prompt = f"""Sen semantik analiz uzmanısın. 4 adımlı CoT protokolüyle analiz yap.
+CÜMLE: "{full_sentence}"
+VARLIKLAR: {targets_text}
+ETİKETLER: {label_rules}
+ÇIKTI: Sadece JSON listesi döndür. Her nesne 'varlik', 'analiz', 'karar' içermeli."""
+
+    messages = [{"role": "user", "content": prompt}]
+    text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+    model_inputs = tokenizer([text], return_tensors="pt").to(device)
+
+    try:
+        generated_ids = llm_model.generate(model_inputs.input_ids, max_new_tokens=1500, do_sample=False)
+        full_output = tokenizer.batch_decode(generated_ids[:, model_inputs.input_ids.shape[1]:], skip_special_tokens=True)[0].strip()
+        json_match = re.search(r'\[\s*\{.*\}\s*\]', full_output, re.DOTALL)
+        return json.loads(json_match.group(0)) if json_match else []
+    except: return []