Spaces:
Sleeping
Sleeping
| import streamlit as st | |
| import pandas as pd | |
| import numpy as np | |
| import plotly.express as px | |
| import matplotlib.pyplot as plt | |
| from collections import Counter | |
| import re | |
| import string | |
| import nltk | |
| from nltk.corpus import stopwords | |
| from nltk.tokenize import word_tokenize | |
| from nltk.stem import WordNetLemmatizer, PorterStemmer | |
| from wordcloud import WordCloud | |
| import io | |
| # Download NLTK resources if not already downloaded | |
| try: | |
| nltk.data.find('tokenizers/punkt') | |
| except LookupError: | |
| nltk.download('punkt', quiet=True) | |
| try: | |
| nltk.data.find('corpora/stopwords') | |
| except LookupError: | |
| nltk.download('stopwords', quiet=True) | |
| try: | |
| nltk.data.find('corpora/wordnet') | |
| except LookupError: | |
| nltk.download('wordnet', quiet=True) | |
| # Page configuration | |
| st.set_page_config( | |
| page_title="Natural Language Processing", | |
| page_icon="📝", | |
| layout="wide" | |
| ) | |
| # Page title | |
| st.title("📝 Natural Language Processing") | |
| st.markdown(""" | |
| Analyze text data using natural language processing techniques to extract insights, | |
| identify patterns, and visualize text characteristics. | |
| """) | |
| # Check if data is loaded | |
| if 'df' not in st.session_state or st.session_state.df is None: | |
| st.error("Please load a dataset first! Go back to the Home page.") | |
| st.stop() | |
| # Get the dataframe | |
| df = st.session_state.df | |
| # Function to preprocess text | |
| def preprocess_text(text, remove_stopwords=True, stem_words=False, lemmatize=False): | |
| if pd.isna(text) or not isinstance(text, str): | |
| return "" | |
| # Convert to lowercase | |
| text = text.lower() | |
| # Remove URLs | |
| text = re.sub(r'https?://\S+|www\.\S+', '', text) | |
| # Remove HTML tags | |
| text = re.sub(r'<.*?>', '', text) | |
| # Remove punctuation and numbers | |
| text = re.sub(f'[{re.escape(string.punctuation)}]', ' ', text) | |
| text = re.sub(r'\d+', '', text) | |
| # Tokenize | |
| tokens = word_tokenize(text) | |
| # Remove stopwords | |
| if remove_stopwords: | |
| stop_words = set(stopwords.words('english')) | |
| tokens = [word for word in tokens if word not in stop_words] | |
| # Stemming | |
| if stem_words: | |
| stemmer = PorterStemmer() | |
| tokens = [stemmer.stem(word) for word in tokens] | |
| # Lemmatization | |
| if lemmatize: | |
| lemmatizer = WordNetLemmatizer() | |
| tokens = [lemmatizer.lemmatize(word) for word in tokens] | |
| # Join tokens back to text | |
| processed_text = ' '.join(tokens) | |
| # Remove extra spaces | |
| processed_text = re.sub(r'\s+', ' ', processed_text).strip() | |
| return processed_text | |
| # Function to create word frequency distribution | |
| def get_word_freq(texts, top_n=20): | |
| all_words = ' '.join(texts).split() | |
| word_counts = Counter(all_words) | |
| return word_counts.most_common(top_n) | |
| # Function to create word cloud | |
| def generate_wordcloud(text): | |
| wordcloud = WordCloud( | |
| width=800, | |
| height=400, | |
| background_color='white', | |
| max_words=200, | |
| contour_width=3, | |
| contour_color='steelblue', | |
| collocations=False | |
| ).generate(text) | |
| return wordcloud | |
| # Create tabs for different NLP features | |
| tab1, tab2, tab3, tab4 = st.tabs([ | |
| "Text Preprocessing", | |
| "Word Clouds", | |
| "Text Statistics", | |
| "Sentiment Analysis" | |
| ]) | |
| # Text Preprocessing Tab | |
| with tab1: | |
| st.subheader("Text Preprocessing") | |
| # Select text column for analysis | |
| text_columns = df.select_dtypes(include=['object']).columns.tolist() | |
| if not text_columns: | |
| st.warning("No text columns found in the dataset. Please load a dataset with text data.") | |
| st.stop() | |
| selected_text_col = st.selectbox( | |
| "Select text column for analysis", | |
| text_columns | |
| ) | |
| # Preprocessing options | |
| st.subheader("Preprocessing Options") | |
| col1, col2, col3 = st.columns(3) | |
| with col1: | |
| remove_stopwords = st.checkbox("Remove Stopwords", value=True) | |
| with col2: | |
| stem_words = st.checkbox("Apply Stemming", value=False) | |
| with col3: | |
| lemmatize = st.checkbox("Apply Lemmatization", value=True) | |
| # Process the text | |
| if st.button("Preprocess Text"): | |
| with st.spinner("Processing text data..."): | |
| try: | |
| # Get text data from the selected column | |
| text_data = df[selected_text_col].fillna("").astype(str).tolist() | |
| # Preprocess each text | |
| processed_texts = [ | |
| preprocess_text( | |
| text, | |
| remove_stopwords=remove_stopwords, | |
| stem_words=stem_words, | |
| lemmatize=lemmatize | |
| ) for text in text_data | |
| ] | |
| # Store processed texts in session state | |
| st.session_state.text_data = text_data | |
| st.session_state.processed_texts = processed_texts | |
| st.session_state.selected_text_col = selected_text_col | |
| # Display sample before and after | |
| st.subheader("Sample Text Before and After Processing") | |
| sample_size = min(5, len(text_data)) | |
| samples = np.random.choice(len(text_data), sample_size, replace=False) | |
| for i in samples: | |
| col1, col2 = st.columns(2) | |
| with col1: | |
| st.text_area("Original Text", text_data[i], height=150, key=f"orig_{i}") | |
| with col2: | |
| st.text_area("Processed Text", processed_texts[i], height=150, key=f"proc_{i}") | |
| # Create a new dataframe with processed text | |
| new_col_name = f"{selected_text_col}_processed" | |
| df_processed = df.copy() | |
| df_processed[new_col_name] = processed_texts | |
| # Option to save the processed text as a new column | |
| if st.button("Add Processed Text as New Column"): | |
| st.session_state.df[new_col_name] = processed_texts | |
| st.success(f"Added processed text as new column '{new_col_name}'") | |
| st.rerun() | |
| # Display info | |
| st.info(f"Successfully processed {len(processed_texts)} texts. Now you can generate word clouds or analyze text statistics.") | |
| except Exception as e: | |
| st.error(f"Error processing text: {e}") | |
| # Word Clouds Tab | |
| with tab2: | |
| st.subheader("Word Cloud Visualization") | |
| if 'processed_texts' not in st.session_state: | |
| st.info("Please process your text data in the 'Text Preprocessing' tab first.") | |
| else: | |
| # Options for word cloud | |
| cloud_type = st.radio( | |
| "Word Cloud Type", | |
| ["All Documents", "By Category"], | |
| horizontal=True | |
| ) | |
| if cloud_type == "All Documents": | |
| # Generate word cloud for all documents | |
| if st.button("Generate Word Cloud"): | |
| with st.spinner("Generating word cloud..."): | |
| try: | |
| # Combine all processed texts | |
| all_text = ' '.join(st.session_state.processed_texts) | |
| if all_text.strip(): | |
| # Generate word cloud | |
| wordcloud = generate_wordcloud(all_text) | |
| # Convert to image for Streamlit | |
| fig, ax = plt.subplots(figsize=(10, 6)) | |
| ax.imshow(wordcloud, interpolation='bilinear') | |
| ax.axis('off') | |
| plt.tight_layout() | |
| # Convert to bytes for Streamlit | |
| buf = io.BytesIO() | |
| plt.savefig(buf, format='png', dpi=300) | |
| plt.close(fig) | |
| buf.seek(0) | |
| # Display the word cloud | |
| st.image(buf, use_column_width=True) | |
| # Word frequency table | |
| st.subheader("Top Words by Frequency") | |
| top_words = get_word_freq(st.session_state.processed_texts) | |
| # Create DataFrame for the table | |
| word_freq_df = pd.DataFrame(top_words, columns=['Word', 'Frequency']) | |
| # Display as a table | |
| st.dataframe(word_freq_df, use_container_width=True) | |
| # Bar chart of top words | |
| fig = px.bar( | |
| word_freq_df, | |
| x='Word', | |
| y='Frequency', | |
| title='Top 20 Words by Frequency', | |
| height=500 | |
| ) | |
| st.plotly_chart(fig, use_container_width=True) | |
| else: | |
| st.warning("No words left after preprocessing. Try adjusting preprocessing options.") | |
| except Exception as e: | |
| st.error(f"Error generating word cloud: {e}") | |
| else: # By Category | |
| # Select a category column | |
| categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist() | |
| categorical_cols = [col for col in categorical_cols if col != st.session_state.selected_text_col] | |
| if not categorical_cols: | |
| st.warning("No categorical columns found for grouping. Please choose 'All Documents' instead.") | |
| else: | |
| category_col = st.selectbox( | |
| "Select category column", | |
| categorical_cols | |
| ) | |
| # Generate word clouds by category | |
| if st.button("Generate Word Clouds by Category"): | |
| with st.spinner("Generating word clouds by category..."): | |
| try: | |
| # Get unique categories | |
| categories = df[category_col].dropna().unique() | |
| # For each category, create a word cloud | |
| for category in categories[:5]: # Limit to first 5 categories to avoid overload | |
| st.subheader(f"Word Cloud for {category_col} = {category}") | |
| # Get indices for this category | |
| category_indices = df[df[category_col] == category].index | |
| # Get processed texts for this category | |
| category_texts = [st.session_state.processed_texts[i] for i in category_indices if i < len(st.session_state.processed_texts)] | |
| if category_texts: | |
| # Combine all texts for this category | |
| category_text = ' '.join(category_texts) | |
| if category_text.strip(): | |
| # Generate word cloud | |
| wordcloud = generate_wordcloud(category_text) | |
| # Convert to image for Streamlit | |
| fig, ax = plt.subplots(figsize=(10, 6)) | |
| ax.imshow(wordcloud, interpolation='bilinear') | |
| ax.axis('off') | |
| plt.tight_layout() | |
| # Convert to bytes for Streamlit | |
| buf = io.BytesIO() | |
| plt.savefig(buf, format='png', dpi=300) | |
| plt.close(fig) | |
| buf.seek(0) | |
| # Display the word cloud | |
| st.image(buf, use_column_width=True) | |
| # Word frequency table for this category | |
| top_words = get_word_freq(category_texts, top_n=10) | |
| # Create DataFrame for the table | |
| word_freq_df = pd.DataFrame(top_words, columns=['Word', 'Frequency']) | |
| # Display as a table | |
| st.dataframe(word_freq_df, use_container_width=True) | |
| else: | |
| st.warning(f"No words left after preprocessing for category '{category}'.") | |
| else: | |
| st.info(f"No texts found for category '{category}'.") | |
| if len(categories) > 5: | |
| st.info(f"Only showing first 5 of {len(categories)} categories to avoid visual overload.") | |
| except Exception as e: | |
| st.error(f"Error generating word clouds by category: {e}") | |
| # Text Statistics Tab | |
| with tab3: | |
| st.subheader("Text Statistics and Analysis") | |
| if 'processed_texts' not in st.session_state: | |
| st.info("Please process your text data in the 'Text Preprocessing' tab first.") | |
| else: | |
| if st.button("Calculate Text Statistics"): | |
| with st.spinner("Analyzing text data..."): | |
| try: | |
| # Calculate statistics on original and processed texts | |
| original_texts = st.session_state.text_data | |
| processed_texts = st.session_state.processed_texts | |
| # Initialize lists for statistics | |
| orig_word_counts = [] | |
| proc_word_counts = [] | |
| orig_char_counts = [] | |
| proc_char_counts = [] | |
| orig_sent_counts = [] | |
| # Calculate for each document | |
| for orig, proc in zip(original_texts, processed_texts): | |
| # Word counts | |
| orig_word_counts.append(len(str(orig).split())) | |
| proc_word_counts.append(len(str(proc).split())) | |
| # Character counts | |
| orig_char_counts.append(len(str(orig))) | |
| proc_char_counts.append(len(str(proc))) | |
| # Sentence counts (original only) | |
| orig_sent_counts.append(len(nltk.sent_tokenize(str(orig)))) | |
| # Create a DataFrame with statistics | |
| stats_df = pd.DataFrame({ | |
| 'Original Word Count': orig_word_counts, | |
| 'Processed Word Count': proc_word_counts, | |
| 'Original Character Count': orig_char_counts, | |
| 'Processed Character Count': proc_char_counts, | |
| 'Sentence Count': orig_sent_counts | |
| }) | |
| # Add to the original dataframe | |
| df_with_stats = df.copy() | |
| for col in stats_df.columns: | |
| df_with_stats[col] = stats_df[col] | |
| # Display summary statistics | |
| st.subheader("Text Statistics Summary") | |
| st.dataframe(stats_df.describe(), use_container_width=True) | |
| # Display distribution of word counts | |
| st.subheader("Distribution of Word Counts") | |
| fig = px.histogram( | |
| stats_df, | |
| x='Original Word Count', | |
| nbins=50, | |
| title='Distribution of Word Counts in Documents', | |
| labels={'Original Word Count': 'Word Count'}, | |
| height=400 | |
| ) | |
| st.plotly_chart(fig, use_container_width=True) | |
| # Display summary table for individual documents | |
| st.subheader("Statistics for Individual Documents") | |
| st.dataframe(df_with_stats.head(10), use_container_width=True) | |
| # Option to add statistics to the original dataframe | |
| if st.button("Add Statistics to Dataset"): | |
| for col in stats_df.columns: | |
| st.session_state.df[col] = stats_df[col] | |
| st.success("Text statistics added to the dataset") | |
| st.rerun() | |
| except Exception as e: | |
| st.error(f"Error calculating text statistics: {e}") | |
| # Sentiment Analysis Tab | |
| with tab4: | |
| st.subheader("Basic Sentiment Analysis") | |
| if 'processed_texts' not in st.session_state: | |
| st.info("Please process your text data in the 'Text Preprocessing' tab first.") | |
| else: | |
| # Select sentiment analysis method | |
| method = st.selectbox( | |
| "Select Sentiment Analysis Method", | |
| ["NLTK Vader", "Simple Lexicon-based"] | |
| ) | |
| if st.button("Analyze Sentiment"): | |
| with st.spinner("Analyzing sentiment..."): | |
| try: | |
| if method == "NLTK Vader": | |
| try: | |
| from nltk.sentiment.vader import SentimentIntensityAnalyzer | |
| try: | |
| nltk.data.find('sentiment/vader_lexicon.zip') | |
| except LookupError: | |
| nltk.download('vader_lexicon', quiet=True) | |
| # Initialize the VADER sentiment analyzer | |
| sid = SentimentIntensityAnalyzer() | |
| # Get original texts | |
| original_texts = st.session_state.text_data | |
| # Calculate sentiment scores | |
| sentiment_scores = [] | |
| compound_scores = [] | |
| sentiments = [] | |
| for text in original_texts: | |
| if pd.isna(text) or not isinstance(text, str) or not text.strip(): | |
| sentiment_scores.append({'neg': 0, 'neu': 0, 'pos': 0, 'compound': 0}) | |
| compound_scores.append(0) | |
| sentiments.append('neutral') | |
| else: | |
| # Get sentiment scores | |
| scores = sid.polarity_scores(text) | |
| sentiment_scores.append(scores) | |
| compound_scores.append(scores['compound']) | |
| # Determine sentiment label | |
| if scores['compound'] >= 0.05: | |
| sentiments.append('positive') | |
| elif scores['compound'] <= -0.05: | |
| sentiments.append('negative') | |
| else: | |
| sentiments.append('neutral') | |
| # Create DataFrame with sentiment results | |
| sentiment_df = pd.DataFrame({ | |
| 'Text': original_texts, | |
| 'Compound Score': compound_scores, | |
| 'Sentiment': sentiments | |
| }) | |
| # Calculate sentiment distribution | |
| sentiment_counts = sentiment_df['Sentiment'].value_counts() | |
| # Display sentiment distribution | |
| st.subheader("Sentiment Distribution") | |
| fig = px.pie( | |
| values=sentiment_counts.values, | |
| names=sentiment_counts.index, | |
| title='Distribution of Sentiments', | |
| color_discrete_map={ | |
| 'positive': 'green', | |
| 'neutral': 'gray', | |
| 'negative': 'red' | |
| }, | |
| height=500 | |
| ) | |
| st.plotly_chart(fig, use_container_width=True) | |
| # Display sentiment scores for a sample of documents | |
| st.subheader("Sample Document Sentiment Scores") | |
| st.dataframe(sentiment_df.head(10), use_container_width=True) | |
| # Option to add sentiment to the dataset | |
| if st.button("Add Sentiment Analysis to Dataset"): | |
| st.session_state.df['Sentiment_Score'] = compound_scores | |
| st.session_state.df['Sentiment'] = sentiments | |
| st.success("Sentiment analysis results added to the dataset") | |
| st.rerun() | |
| except ImportError: | |
| st.error("NLTK Vader not available. Try using 'Simple Lexicon-based' method instead.") | |
| else: # Simple Lexicon-based | |
| # Load positive and negative word lists | |
| positive_words = [ | |
| 'good', 'great', 'excellent', 'positive', 'nice', 'wonderful', 'amazing', | |
| 'love', 'happy', 'best', 'better', 'awesome', 'fantastic', 'superior', | |
| 'perfect', 'outstanding', 'remarkable', 'superb', 'brilliant', 'exceptional' | |
| ] | |
| negative_words = [ | |
| 'bad', 'terrible', 'negative', 'poor', 'awful', 'horrible', 'worst', | |
| 'hate', 'sad', 'disappointing', 'inferior', 'mediocre', 'useless', | |
| 'unpleasant', 'failure', 'pathetic', 'terrible', 'dreadful', 'poor' | |
| ] | |
| # Get processed texts for better analysis | |
| processed_texts = st.session_state.processed_texts | |
| # Calculate sentiment scores | |
| positive_scores = [] | |
| negative_scores = [] | |
| sentiment_scores = [] | |
| sentiments = [] | |
| for text in processed_texts: | |
| words = text.split() | |
| # Count positive and negative words | |
| pos_count = sum(1 for word in words if word in positive_words) | |
| neg_count = sum(1 for word in words if word in negative_words) | |
| # Calculate score | |
| if len(words) > 0: | |
| pos_score = pos_count / len(words) | |
| neg_score = neg_count / len(words) | |
| sentiment_score = pos_score - neg_score | |
| else: | |
| pos_score = 0 | |
| neg_score = 0 | |
| sentiment_score = 0 | |
| positive_scores.append(pos_score) | |
| negative_scores.append(neg_score) | |
| sentiment_scores.append(sentiment_score) | |
| # Determine sentiment label | |
| if sentiment_score > 0.05: | |
| sentiments.append('positive') | |
| elif sentiment_score < -0.05: | |
| sentiments.append('negative') | |
| else: | |
| sentiments.append('neutral') | |
| # Create DataFrame with sentiment results | |
| sentiment_df = pd.DataFrame({ | |
| 'Text': st.session_state.text_data, | |
| 'Positive Score': positive_scores, | |
| 'Negative Score': negative_scores, | |
| 'Sentiment Score': sentiment_scores, | |
| 'Sentiment': sentiments | |
| }) | |
| # Calculate sentiment distribution | |
| sentiment_counts = sentiment_df['Sentiment'].value_counts() | |
| # Display sentiment distribution | |
| st.subheader("Sentiment Distribution") | |
| fig = px.pie( | |
| values=sentiment_counts.values, | |
| names=sentiment_counts.index, | |
| title='Distribution of Sentiments', | |
| color_discrete_map={ | |
| 'positive': 'green', | |
| 'neutral': 'gray', | |
| 'negative': 'red' | |
| }, | |
| height=500 | |
| ) | |
| st.plotly_chart(fig, use_container_width=True) | |
| # Display sentiment scores for a sample of documents | |
| st.subheader("Sample Document Sentiment Scores") | |
| st.dataframe(sentiment_df.head(10), use_container_width=True) | |
| # Option to add sentiment to the dataset | |
| if st.button("Add Sentiment Analysis to Dataset"): | |
| st.session_state.df['Positive_Score'] = positive_scores | |
| st.session_state.df['Negative_Score'] = negative_scores | |
| st.session_state.df['Sentiment_Score'] = sentiment_scores | |
| st.session_state.df['Sentiment'] = sentiments | |
| st.success("Sentiment analysis results added to the dataset") | |
| st.rerun() | |
| except Exception as e: | |
| st.error(f"Error analyzing sentiment: {e}") | |
| # Sidebar with educational content | |
| with st.sidebar: | |
| st.title("📚 NLP Learning Corner") | |
| st.markdown(""" | |
| ### What is NLP? | |
| Natural Language Processing (NLP) is a field of artificial intelligence that focuses on interactions between computers and human language. | |
| It enables computers to understand, interpret, and generate human language in useful ways. | |
| ### Key NLP Concepts | |
| **Text Preprocessing Steps:** | |
| - **Tokenization**: Breaking text into words, phrases, or other meaningful elements | |
| - **Stop Word Removal**: Removing common words that add little meaning (a, the, is, etc.) | |
| - **Stemming**: Reducing words to their root/stem form (running → run) | |
| - **Lemmatization**: Similar to stemming but ensures the result is a valid word (better → good) | |
| **Text Analysis Techniques:** | |
| - **Word Frequency**: Counting word occurrences to identify important terms | |
| - **Word Clouds**: Visual representations of word frequency | |
| - **Sentiment Analysis**: Determining if text expresses positive, negative, or neutral sentiment | |
| - **Topic Modeling**: Identifying themes or topics within a collection of documents | |
| ### Applications of NLP | |
| - **Sentiment Analysis**: Analyzing customer reviews, social media, surveys | |
| - **Text Classification**: Categorizing documents by topic, intent, or type | |
| - **Information Extraction**: Pulling structured data from unstructured text | |
| - **Text Summarization**: Creating concise summaries of longer documents | |
| - **Language Translation**: Converting text between languages | |
| - **Chatbots and Virtual Assistants**: Creating natural language interfaces | |
| """) |