--- license: mit language: - ko base_model: - lcw99/t5-base-korean-text-summary --- # KSSDS: Korean Sentence Splitter for Dialogue Systems KSSDS는 [lcw99/t5-base-korean-text-summary](https://huggingface.co/lcw99/t5-base-korean-text-summary)의 인코더를 기반으로 하여, AI Hub 데이터를 이용해 fine-tuning한 모델입니다. 이 모델은 한국어 대화 시스템 용 문장 분리기로, Whisper와 같은 STT 모델이 생성한 한국어 텍스트를 문장 단위로 분리하는 것을 목표로 만들어졌습니다. 자세한 설명은 [KSSDS GitHub repository](https://github.com/ggomarobot/KSSDS)를 참고해주세요. --- ## 사용 방법 ### 1. PyPI 또는 GitHub 설치를 통한 사용 (권장) - PyPI 또는 GitHub를 통해 설치하면 더욱 더 편리하게 사용하실 수 있습니다. - [KSSDS GitHub repository](https://github.com/ggomarobot/KSSDS)에서 설정 및 사용 방법을 확인하세요. ### 2. Hugging Face Hub 네이티브 방식 사용 Hugging Face Hub에서 모델과 T5 인코더를 직접 다운로드하여 사용할 수 있습니다. 아래는 문장 분리를 수행하는 전체 파이프라인의 예제 코드입니다. ```python # 필요한 라이브러리 및 모듈 불러오기 from huggingface_hub import hf_hub_download # Hugging Face Hub에서 파일 다운로드 import sys import os from transformers import AutoTokenizer # Hugging Face Tokenizer import torch from typing import List # Hugging Face Hub에서 T5_encoder.py와 모델 다운로드 model_name = "ggomarobot/KSSDS" file_path = hf_hub_download(repo_id=model_name, filename="T5_encoder.py") # T5_encoder.py 파일 다운로드 module_dir = os.path.dirname(file_path) # T5_encoder.py 파일이 있는 디렉터리 경로 sys.path.append(module_dir) # Python 경로에 T5_encoder 디렉터리 추가 from T5_encoder import T5ForTokenClassification # 커스텀 T5 인코더 모델 tokenizer = AutoTokenizer.from_pretrained(model_name) # 토크나이저 불러오기 model = T5ForTokenClassification.from_pretrained(model_name) # 커스텀 T5 인코더 모델 불러오기 # Check for GPU availability device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # Move the model to GPU model = model.to(device) model.eval() # 모델을 평가 모드로 설정 # 최대 입력 길이 설정 max_length = 512 def preprocess_text(input_text: str) -> List[dict]: """ 긴 입력 텍스트를 처리 가능한 청크로 나누는 함수. Args: input_text (str): 처리할 긴 입력 텍스트. Returns: List[dict]: 청크 단위의 input_ids와 attention_mask를 포함한 리스트. """ tokenized_sequence = tokenizer.encode(input_text, add_special_tokens=False) chunks = [tokenized_sequence[i:i + max_length] for i in range(0, len(tokenized_sequence), max_length)] processed_data = [{"input_ids": chunk, "attention_mask": [1] * len(chunk)} for chunk in chunks] return processed_data def handle_repetitions(sentences: List[str], max_repeats: int = 60, detection_threshold: int = 70, max_phrase_length: int = 2) -> List[str]: """ Handles single-word and phrase repetitions in a list of sentences, ensuring proper order and separation. Args: sentences (List[str]): List of input sentences to process. max_repeats (int): Maximum number of phrase repetitions to allow before splitting. detection_threshold (int): Minimum length of text for repetition detection. max_phrase_length (int): Maximum length of a phrase to consider for repetition detection. Returns: List[str]: The processed text split into sentences. """ processed_sentences = [] for sentence in sentences: words = sentence.split() if len(words) <= detection_threshold: processed_sentences.append(sentence) continue result_sentences = [] current_sentence = [] current_repetition = [] def flush_sentence(): """Flush the current sentence into result_sentences.""" if current_sentence: result_sentences.append(" ".join(current_sentence)) current_sentence.clear() def flush_repetition(phrase_length): """Flush the current repetition into result_sentences.""" for i in range(0, len(current_repetition), phrase_length * max_repeats): chunk = current_repetition[i:i + phrase_length * max_repeats] result_sentences.append(" ".join(chunk)) current_repetition.clear() def find_repeating_phrase(start_idx): """Find the smallest repeating phrase starting at the given index.""" for phrase_length in range(1, max_phrase_length + 1): phrase = words[start_idx:start_idx + phrase_length] next_idx = start_idx + phrase_length if next_idx + phrase_length <= len(words) and words[next_idx:next_idx + phrase_length] == phrase: return phrase return None i = 0 while i < len(words): repeating_phrase = find_repeating_phrase(i) if repeating_phrase: # Flush any ongoing sentence before handling repetition flush_sentence() # Accumulate repeating phrases phrase_length = len(repeating_phrase) while i + phrase_length <= len(words) and words[i:i + phrase_length] == repeating_phrase: current_repetition.extend(repeating_phrase) i += phrase_length # Flush accumulated repetition if it reaches the threshold if len(current_repetition) >= phrase_length * max_repeats: flush_repetition(phrase_length) else: # Add non-repeating words to the current sentence if current_repetition: # Flush repetition before starting a new sentence flush_repetition(1) # Default to single-word repetition current_sentence.append(words[i]) i += 1 # Flush any remaining tokens flush_sentence() flush_repetition(1) # Default to single-word repetition for the last chunk processed_sentences.extend(result_sentences) return processed_sentences def segment_predictions(input_ids: List[int], predictions: List[int]) -> List[List[int]]: """ Segment predictions into sentences based on label 1 (sentence-ending). Args: input_ids (List[int]): List of input token IDs. predictions (List[int]): Corresponding prediction labels. Returns: List[List[int]]: Segmented sentences as lists of token IDs. """ segments = [] current_segment = [] for token, label in zip(input_ids, predictions): if label == 1: # Sentence-ending label if current_segment: current_segment.append(token) segments.append(current_segment) current_segment = [] else: segments.append([token]) else: current_segment.append(token) if current_segment: # Append any remaining tokens segments.append(current_segment) return segments def decode_predictions(input_ids: List[int], predictions: List[int], tokenizer, carry_over=None): """ Decode model predictions into sentences, handling carry-over tokens across chunks. Args: input_ids (List[int]): Input token IDs. predictions (List[int]): Prediction labels. tokenizer: Hugging Face tokenizer instance. carry_over (List[int], optional): Tokens carried over from the previous chunk. Returns: Tuple[List[str], List[int]]: Decoded sentences and remaining carry-over tokens. """ if carry_over is None: carry_over = [] sentences = [] tokens = carry_over + input_ids # Include carry-over tokens labels = [0] * len(carry_over) + predictions # Carry-over tokens have label 0 segmented = segment_predictions(tokens, labels) for segment in segmented[:-1]: # Decode all segments except the last one sentence = tokenizer.decode(segment, skip_special_tokens=False, clean_up_tokenization_spaces=False).strip() if sentence: sentences.append(sentence) # Handle carry-over for the last segment carry_over = segmented[-1] if segmented[-1] and labels[len(tokens) - len(segmented[-1])] != 1 else [] return sentences, carry_over def inference(input_text: str) -> List[str]: """ Perform sentence splitting on input text using the HF Hub model. Args: input_text (str): Input text to split into sentences. Returns: List[str]: List of split sentences. """ chunks = preprocess_text(input_text) carry_over_tokens = [] sentences = [] for chunk in chunks: # Prepare inputs for the model input_ids_tensor = torch.tensor([chunk["input_ids"]], dtype=torch.long, device=device) attention_mask_tensor = torch.tensor([chunk["attention_mask"]], dtype=torch.long, device=device) with torch.no_grad(): outputs = model(input_ids=input_ids_tensor, attention_mask=attention_mask_tensor) predictions = outputs.logits.argmax(dim=-1).squeeze().tolist() # Ensure predictions is a list if isinstance(predictions, int): predictions = [predictions] input_ids = chunk["input_ids"] decoded_sentences, carry_over_tokens = decode_predictions(input_ids, predictions, tokenizer, carry_over_tokens) sentences.extend(decoded_sentences) # Process any remaining carry-over tokens if carry_over_tokens: remaining_sentence = tokenizer.decode(carry_over_tokens, skip_special_tokens=False, clean_up_tokenization_spaces=False).strip() if remaining_sentence: sentences.append(remaining_sentence) return handle_repetitions(sentences) # 사용 예제 input_text = "안녕하세요. 오늘 날씨가 참 좋네요. 저는 산책을 나갈 예정입니다." split_sentences = inference(input_text) # 결과 출력 for idx, sentence in enumerate(split_sentences): print(f"{idx + 1}: {sentence}") ``` ## KSSDS_NO_LF 모델 소개 [**KSSDS_NO_LF**](https://huggingface.co/ggomarobot/KSSDS_NO_LF)는 ablation study를 위해 Length Filter를 적용하지 않고 학습된 모델입니다. 이 모델은 긴 텍스트를 처리할 때 상대적으로 덜 정밀한 문장 분리를 수행합니다. 사용 방식은 본 모델과 동일하지만 연구 및 비교 목적 외에는 실제 사용에 적합하지 않을 수 있습니다. ### 사용 방법 위의 [Hugging Face Hub 네이티브 방식 사용](#2-hugging-face-hub-네이티브-방식-사용) 코드 스니펫에서 `model_name`을 `"ggomarobot/KSSDS_NO_LF"`로 교체하여 사용할 수 있습니다: ```python # Load tokenizer and model model_name = "ggomarobot/KSSDS_NO_LF" tokenizer = AutoTokenizer.from_pretrained(model_name) model = T5ForTokenClassification.from_pretrained(model_name) ```