Spaces:
Sleeping
Sleeping
Download app.py from soojeongcrystal/text: direct link, hf CLI and curl.
- Browser
- Download file 6.16 kB
-
https://huggingface.co/spaces/soojeongcrystal/text/resolve/main/app.py
- Command line
-
hf download hf://spaces/soojeongcrystal/text/app.py
-
curl -L -o app.py https://huggingface.co/spaces/soojeongcrystal/text/resolve/main/app.py
6.16 kB
| import streamlit as st | |
| import matplotlib.pyplot as plt | |
| from sklearn.feature_extraction.text import CountVectorizer | |
| from sklearn.decomposition import LatentDirichletAllocation | |
| import warnings | |
| from collections import Counter | |
| import traceback | |
| import os | |
| import jpype | |
| warnings.filterwarnings("ignore") | |
| # JPype μ€μ | |
| if not jpype.isJVMStarted(): | |
| jvm_path = jpype.getDefaultJVMPath() | |
| jpype.startJVM(jvm_path, "-Djava.class.path=/usr/local/lib/python3.10/site-packages/konlpy/java/*") | |
| # νκΈ ν°νΈ μ€μ (νκΉ νμ΄μ€ νκ²½μ λ§κ² μμ νμν μ μμ) | |
| plt.rcParams['font.family'] = 'NanumGothic' | |
| # μ΄κΈ° λΆμ©μ΄ λͺ©λ‘ | |
| default_stopwords = set([ | |
| "μλ€", "μλ€", "λλ€", "μ΄λ€", "νλ€", "κ°λ€", "μνλ€", "μλ", "νλ", | |
| "κ·Έλ¦¬κ³ ", "κ·Έλ°", "μ΄λ°", "μ λ°", "μ΄λ κ²", "μ λ κ²", "κ·Έλ κ²", | |
| "μ°λ¦¬", "μ λ", "μ κ°", "λ΄κ°", "λλ", "μμ΅λλ€", "μμμ΅λλ€", "λμμ΅λλ€", | |
| "νμμ΅λλ€", "ν΅ν΄", "μν΄", "λν", "ν¨κ»", "λ§μ", "μ΄λ€", "μ ν¬", "μ΄λ₯Ό", | |
| "λν", "κ·Έλμ", "κ·Έλ¦¬κ³ ", "νμ§λ§", "κ·Έλ¬λ", "λ°λΌμ", "λλ¬Έμ" | |
| ]) | |
| def load_okt(): | |
| from konlpy.tag import Okt | |
| return Okt() | |
| okt = load_okt() | |
| def extract_nouns(text): | |
| try: | |
| nouns = okt.nouns(text) | |
| return [noun for noun in nouns if len(noun) > 1] | |
| except Exception as e: | |
| st.error(f"λͺ μ¬ μΆμΆ μ€ μ€λ₯ λ°μ: {str(e)}") | |
| st.error(f"λ¬Έμ κ° λ°μν ν μ€νΈ: {text[:100]}...") | |
| return [] | |
| def preprocess_text(text, user_stopwords): | |
| try: | |
| if not text or not isinstance(text, str): | |
| st.warning(f"μ ν¨νμ§ μμ μ λ ₯: {type(text)}") | |
| return "" | |
| nouns = extract_nouns(text) | |
| nouns = [noun for noun in nouns if noun and noun not in user_stopwords] | |
| return ' '.join(nouns) | |
| except Exception as e: | |
| st.error(f"ν μ€νΈ μ μ²λ¦¬ μ€ μ€λ₯ λ°μ: {str(e)}") | |
| st.error(f"λ¬Έμ κ° λ°μν ν μ€νΈ: {text[:100]}...") | |
| return "" | |
| def topic_modeling(texts, n_components): | |
| stop_words_list = list(default_stopwords) | |
| vectorizer = CountVectorizer(stop_words=stop_words_list, max_features=1000) | |
| data_vectorized = vectorizer.fit_transform(texts) | |
| n_tokens = data_vectorized.shape[1] | |
| n_components = min(n_components, n_tokens) | |
| if n_components < 2: | |
| st.warning("μΆμΆλ κ³ μ λ¨μ΄κ° λ무 μ μ΅λλ€. λ κΈ΄ ν μ€νΈλ₯Ό μ¬μ©ν΄ μ£ΌμΈμ.") | |
| return {} | |
| lda = LatentDirichletAllocation(n_components=n_components, random_state=42, max_iter=50, | |
| learning_method='online', learning_offset=50., doc_topic_prior=0.1, topic_word_prior=0.01) | |
| lda.fit(data_vectorized) | |
| features = vectorizer.get_feature_names_out() | |
| topics = {} | |
| for topic_idx, topic in enumerate(lda.components_): | |
| top_words = [features[i] for i in topic.argsort()[:-11:-1]] | |
| topics[f"Topic {topic_idx + 1}"] = top_words | |
| return topics | |
| def generate_word_frequency_chart(text, color, n=20): | |
| nouns = extract_nouns(text) | |
| word_freq = Counter(nouns) | |
| top_words = dict(word_freq.most_common(n)) | |
| fig, ax = plt.subplots(figsize=(12, 6)) | |
| ax.barh(list(top_words.keys()), list(top_words.values()), color=color) | |
| ax.invert_yaxis() | |
| ax.set_title("μμ {} λ¨μ΄".format(n)) | |
| plt.tight_layout() | |
| return fig | |
| def get_top_trigrams(text, n=10): | |
| nouns = extract_nouns(text) | |
| trigrams = zip(nouns, nouns[1:], nouns[2:]) | |
| trigram_freq = Counter(trigrams) | |
| return trigram_freq.most_common(n) | |
| # μ€νΈλ¦Όλ¦Ώ UI μ€μ | |
| st.title("ν μ€νΈ λΆμ λꡬ") | |
| user_stopwords = st.sidebar.text_area("λΆμ©μ΄λ₯Ό μ λ ₯νμΈμ (μΌνλ‘ κ΅¬λΆ)", | |
| value=", ".join(default_stopwords)) | |
| user_stopwords = list(set(user_stopwords.split(", ")) | default_stopwords) | |
| uploaded_file = st.file_uploader("ν μ€νΈ νμΌ μ λ‘λ", type=['txt']) | |
| if uploaded_file is not None: | |
| try: | |
| with st.spinner('νμΌμ μ²λ¦¬ μ€μ λλ€...'): | |
| text = str(uploaded_file.read(), 'utf-8') | |
| st.text(f"νμΌ κΈΈμ΄: {len(text)} λ¬Έμ") | |
| preprocessed_text = preprocess_text(text, user_stopwords) | |
| st.text(f"μ²λ¦¬λ ν μ€νΈ κΈΈμ΄: {len(preprocessed_text)} λ¬Έμ") | |
| st.text(f"κ³ μ λ¨μ΄ μ: {len(set(preprocessed_text.split()))}") | |
| if not preprocessed_text: | |
| st.warning("μ²λ¦¬λ ν μ€νΈκ° μμ΅λλ€. λ€λ₯Έ νμΌμ μ λ‘λν΄ μ£ΌμΈμ.") | |
| else: | |
| st.subheader("ν ν½ λͺ¨λΈλ§ κ²°κ³Ό") | |
| n_topics = st.slider("ν ν½ μ μ ν", min_value=2, max_value=10, value=5) | |
| topics = topic_modeling([preprocessed_text], n_topics) | |
| if topics: | |
| for topic, words in topics.items(): | |
| st.write(f"{topic}: {', '.join(words)}") | |
| else: | |
| st.warning("ν ν½ λͺ¨λΈλ§μ μνν μ μμ΅λλ€. λ κΈ΄ ν μ€νΈλ₯Ό μ¬μ©ν΄ μ£ΌμΈμ.") | |
| st.subheader("μμ 10κ° Trigram") | |
| top_trigrams = get_top_trigrams(preprocessed_text) | |
| for trigram, count in top_trigrams: | |
| st.write(f"{' '.join(trigram)}: {count}") | |
| st.subheader("λ¨μ΄ λΉλ μ°¨νΈ") | |
| color = st.color_picker("λ§λ μμ μ ν", "#1f77b4") | |
| fig = generate_word_frequency_chart(preprocessed_text, color) | |
| st.pyplot(fig) | |
| except Exception as e: | |
| st.error(f"νμΌ μ²λ¦¬ μ€ μ€λ₯ λ°μ: {str(e)}") | |
| st.text("μ€λ₯ λ°μ μμΉ:") | |
| st.text(traceback.format_exc()) | |
| st.sidebar.markdown(""" | |
| ## μ¬μ© λ°©λ² | |
| 1. μ¬μ΄λλ°μμ λΆμ©μ΄λ₯Ό μΆκ°νκ±°λ μμ ν μ μμ΅λλ€. | |
| 2. ν μ€νΈ νμΌ(.txt)μ μ λ‘λνμΈμ. | |
| 3. ν ν½ λͺ¨λΈλ§μ ν ν½ μλ₯Ό μ ννμΈμ. | |
| 4. μμ 10κ° Trigramμ νμΈνμΈμ. | |
| 5. λ¨μ΄ λΉλ μ°¨νΈμ λ§λ μμμ μ νν μ μμ΅λλ€. | |
| """) |