import streamlit as st import pandas as pd import numpy as np import plotly.express as px import matplotlib.pyplot as plt from collections import Counter import re import string import nltk from nltk.corpus import stopwords from nltk.tokenize import word_tokenize from nltk.stem import WordNetLemmatizer, PorterStemmer from wordcloud import WordCloud import io # Download NLTK resources if not already downloaded try: nltk.data.find('tokenizers/punkt') except LookupError: nltk.download('punkt', quiet=True) try: nltk.data.find('corpora/stopwords') except LookupError: nltk.download('stopwords', quiet=True) try: nltk.data.find('corpora/wordnet') except LookupError: nltk.download('wordnet', quiet=True) # Page configuration st.set_page_config( page_title="Natural Language Processing", page_icon="📝", layout="wide" ) # Page title st.title("📝 Natural Language Processing") st.markdown(""" Analyze text data using natural language processing techniques to extract insights, identify patterns, and visualize text characteristics. """) # Check if data is loaded if 'df' not in st.session_state or st.session_state.df is None: st.error("Please load a dataset first! Go back to the Home page.") st.stop() # Get the dataframe df = st.session_state.df # Function to preprocess text def preprocess_text(text, remove_stopwords=True, stem_words=False, lemmatize=False): if pd.isna(text) or not isinstance(text, str): return "" # Convert to lowercase text = text.lower() # Remove URLs text = re.sub(r'https?://\S+|www\.\S+', '', text) # Remove HTML tags text = re.sub(r'<.*?>', '', text) # Remove punctuation and numbers text = re.sub(f'[{re.escape(string.punctuation)}]', ' ', text) text = re.sub(r'\d+', '', text) # Tokenize tokens = word_tokenize(text) # Remove stopwords if remove_stopwords: stop_words = set(stopwords.words('english')) tokens = [word for word in tokens if word not in stop_words] # Stemming if stem_words: stemmer = PorterStemmer() tokens = [stemmer.stem(word) for word in tokens] # Lemmatization if lemmatize: lemmatizer = WordNetLemmatizer() tokens = [lemmatizer.lemmatize(word) for word in tokens] # Join tokens back to text processed_text = ' '.join(tokens) # Remove extra spaces processed_text = re.sub(r'\s+', ' ', processed_text).strip() return processed_text # Function to create word frequency distribution def get_word_freq(texts, top_n=20): all_words = ' '.join(texts).split() word_counts = Counter(all_words) return word_counts.most_common(top_n) # Function to create word cloud def generate_wordcloud(text): wordcloud = WordCloud( width=800, height=400, background_color='white', max_words=200, contour_width=3, contour_color='steelblue', collocations=False ).generate(text) return wordcloud # Create tabs for different NLP features tab1, tab2, tab3, tab4 = st.tabs([ "Text Preprocessing", "Word Clouds", "Text Statistics", "Sentiment Analysis" ]) # Text Preprocessing Tab with tab1: st.subheader("Text Preprocessing") # Select text column for analysis text_columns = df.select_dtypes(include=['object']).columns.tolist() if not text_columns: st.warning("No text columns found in the dataset. Please load a dataset with text data.") st.stop() selected_text_col = st.selectbox( "Select text column for analysis", text_columns ) # Preprocessing options st.subheader("Preprocessing Options") col1, col2, col3 = st.columns(3) with col1: remove_stopwords = st.checkbox("Remove Stopwords", value=True) with col2: stem_words = st.checkbox("Apply Stemming", value=False) with col3: lemmatize = st.checkbox("Apply Lemmatization", value=True) # Process the text if st.button("Preprocess Text"): with st.spinner("Processing text data..."): try: # Get text data from the selected column text_data = df[selected_text_col].fillna("").astype(str).tolist() # Preprocess each text processed_texts = [ preprocess_text( text, remove_stopwords=remove_stopwords, stem_words=stem_words, lemmatize=lemmatize ) for text in text_data ] # Store processed texts in session state st.session_state.text_data = text_data st.session_state.processed_texts = processed_texts st.session_state.selected_text_col = selected_text_col # Display sample before and after st.subheader("Sample Text Before and After Processing") sample_size = min(5, len(text_data)) samples = np.random.choice(len(text_data), sample_size, replace=False) for i in samples: col1, col2 = st.columns(2) with col1: st.text_area("Original Text", text_data[i], height=150, key=f"orig_{i}") with col2: st.text_area("Processed Text", processed_texts[i], height=150, key=f"proc_{i}") # Create a new dataframe with processed text new_col_name = f"{selected_text_col}_processed" df_processed = df.copy() df_processed[new_col_name] = processed_texts # Option to save the processed text as a new column if st.button("Add Processed Text as New Column"): st.session_state.df[new_col_name] = processed_texts st.success(f"Added processed text as new column '{new_col_name}'") st.rerun() # Display info st.info(f"Successfully processed {len(processed_texts)} texts. Now you can generate word clouds or analyze text statistics.") except Exception as e: st.error(f"Error processing text: {e}") # Word Clouds Tab with tab2: st.subheader("Word Cloud Visualization") if 'processed_texts' not in st.session_state: st.info("Please process your text data in the 'Text Preprocessing' tab first.") else: # Options for word cloud cloud_type = st.radio( "Word Cloud Type", ["All Documents", "By Category"], horizontal=True ) if cloud_type == "All Documents": # Generate word cloud for all documents if st.button("Generate Word Cloud"): with st.spinner("Generating word cloud..."): try: # Combine all processed texts all_text = ' '.join(st.session_state.processed_texts) if all_text.strip(): # Generate word cloud wordcloud = generate_wordcloud(all_text) # Convert to image for Streamlit fig, ax = plt.subplots(figsize=(10, 6)) ax.imshow(wordcloud, interpolation='bilinear') ax.axis('off') plt.tight_layout() # Convert to bytes for Streamlit buf = io.BytesIO() plt.savefig(buf, format='png', dpi=300) plt.close(fig) buf.seek(0) # Display the word cloud st.image(buf, use_column_width=True) # Word frequency table st.subheader("Top Words by Frequency") top_words = get_word_freq(st.session_state.processed_texts) # Create DataFrame for the table word_freq_df = pd.DataFrame(top_words, columns=['Word', 'Frequency']) # Display as a table st.dataframe(word_freq_df, use_container_width=True) # Bar chart of top words fig = px.bar( word_freq_df, x='Word', y='Frequency', title='Top 20 Words by Frequency', height=500 ) st.plotly_chart(fig, use_container_width=True) else: st.warning("No words left after preprocessing. Try adjusting preprocessing options.") except Exception as e: st.error(f"Error generating word cloud: {e}") else: # By Category # Select a category column categorical_cols = df.select_dtypes(include=['object', 'category']).columns.tolist() categorical_cols = [col for col in categorical_cols if col != st.session_state.selected_text_col] if not categorical_cols: st.warning("No categorical columns found for grouping. Please choose 'All Documents' instead.") else: category_col = st.selectbox( "Select category column", categorical_cols ) # Generate word clouds by category if st.button("Generate Word Clouds by Category"): with st.spinner("Generating word clouds by category..."): try: # Get unique categories categories = df[category_col].dropna().unique() # For each category, create a word cloud for category in categories[:5]: # Limit to first 5 categories to avoid overload st.subheader(f"Word Cloud for {category_col} = {category}") # Get indices for this category category_indices = df[df[category_col] == category].index # Get processed texts for this category category_texts = [st.session_state.processed_texts[i] for i in category_indices if i < len(st.session_state.processed_texts)] if category_texts: # Combine all texts for this category category_text = ' '.join(category_texts) if category_text.strip(): # Generate word cloud wordcloud = generate_wordcloud(category_text) # Convert to image for Streamlit fig, ax = plt.subplots(figsize=(10, 6)) ax.imshow(wordcloud, interpolation='bilinear') ax.axis('off') plt.tight_layout() # Convert to bytes for Streamlit buf = io.BytesIO() plt.savefig(buf, format='png', dpi=300) plt.close(fig) buf.seek(0) # Display the word cloud st.image(buf, use_column_width=True) # Word frequency table for this category top_words = get_word_freq(category_texts, top_n=10) # Create DataFrame for the table word_freq_df = pd.DataFrame(top_words, columns=['Word', 'Frequency']) # Display as a table st.dataframe(word_freq_df, use_container_width=True) else: st.warning(f"No words left after preprocessing for category '{category}'.") else: st.info(f"No texts found for category '{category}'.") if len(categories) > 5: st.info(f"Only showing first 5 of {len(categories)} categories to avoid visual overload.") except Exception as e: st.error(f"Error generating word clouds by category: {e}") # Text Statistics Tab with tab3: st.subheader("Text Statistics and Analysis") if 'processed_texts' not in st.session_state: st.info("Please process your text data in the 'Text Preprocessing' tab first.") else: if st.button("Calculate Text Statistics"): with st.spinner("Analyzing text data..."): try: # Calculate statistics on original and processed texts original_texts = st.session_state.text_data processed_texts = st.session_state.processed_texts # Initialize lists for statistics orig_word_counts = [] proc_word_counts = [] orig_char_counts = [] proc_char_counts = [] orig_sent_counts = [] # Calculate for each document for orig, proc in zip(original_texts, processed_texts): # Word counts orig_word_counts.append(len(str(orig).split())) proc_word_counts.append(len(str(proc).split())) # Character counts orig_char_counts.append(len(str(orig))) proc_char_counts.append(len(str(proc))) # Sentence counts (original only) orig_sent_counts.append(len(nltk.sent_tokenize(str(orig)))) # Create a DataFrame with statistics stats_df = pd.DataFrame({ 'Original Word Count': orig_word_counts, 'Processed Word Count': proc_word_counts, 'Original Character Count': orig_char_counts, 'Processed Character Count': proc_char_counts, 'Sentence Count': orig_sent_counts }) # Add to the original dataframe df_with_stats = df.copy() for col in stats_df.columns: df_with_stats[col] = stats_df[col] # Display summary statistics st.subheader("Text Statistics Summary") st.dataframe(stats_df.describe(), use_container_width=True) # Display distribution of word counts st.subheader("Distribution of Word Counts") fig = px.histogram( stats_df, x='Original Word Count', nbins=50, title='Distribution of Word Counts in Documents', labels={'Original Word Count': 'Word Count'}, height=400 ) st.plotly_chart(fig, use_container_width=True) # Display summary table for individual documents st.subheader("Statistics for Individual Documents") st.dataframe(df_with_stats.head(10), use_container_width=True) # Option to add statistics to the original dataframe if st.button("Add Statistics to Dataset"): for col in stats_df.columns: st.session_state.df[col] = stats_df[col] st.success("Text statistics added to the dataset") st.rerun() except Exception as e: st.error(f"Error calculating text statistics: {e}") # Sentiment Analysis Tab with tab4: st.subheader("Basic Sentiment Analysis") if 'processed_texts' not in st.session_state: st.info("Please process your text data in the 'Text Preprocessing' tab first.") else: # Select sentiment analysis method method = st.selectbox( "Select Sentiment Analysis Method", ["NLTK Vader", "Simple Lexicon-based"] ) if st.button("Analyze Sentiment"): with st.spinner("Analyzing sentiment..."): try: if method == "NLTK Vader": try: from nltk.sentiment.vader import SentimentIntensityAnalyzer try: nltk.data.find('sentiment/vader_lexicon.zip') except LookupError: nltk.download('vader_lexicon', quiet=True) # Initialize the VADER sentiment analyzer sid = SentimentIntensityAnalyzer() # Get original texts original_texts = st.session_state.text_data # Calculate sentiment scores sentiment_scores = [] compound_scores = [] sentiments = [] for text in original_texts: if pd.isna(text) or not isinstance(text, str) or not text.strip(): sentiment_scores.append({'neg': 0, 'neu': 0, 'pos': 0, 'compound': 0}) compound_scores.append(0) sentiments.append('neutral') else: # Get sentiment scores scores = sid.polarity_scores(text) sentiment_scores.append(scores) compound_scores.append(scores['compound']) # Determine sentiment label if scores['compound'] >= 0.05: sentiments.append('positive') elif scores['compound'] <= -0.05: sentiments.append('negative') else: sentiments.append('neutral') # Create DataFrame with sentiment results sentiment_df = pd.DataFrame({ 'Text': original_texts, 'Compound Score': compound_scores, 'Sentiment': sentiments }) # Calculate sentiment distribution sentiment_counts = sentiment_df['Sentiment'].value_counts() # Display sentiment distribution st.subheader("Sentiment Distribution") fig = px.pie( values=sentiment_counts.values, names=sentiment_counts.index, title='Distribution of Sentiments', color_discrete_map={ 'positive': 'green', 'neutral': 'gray', 'negative': 'red' }, height=500 ) st.plotly_chart(fig, use_container_width=True) # Display sentiment scores for a sample of documents st.subheader("Sample Document Sentiment Scores") st.dataframe(sentiment_df.head(10), use_container_width=True) # Option to add sentiment to the dataset if st.button("Add Sentiment Analysis to Dataset"): st.session_state.df['Sentiment_Score'] = compound_scores st.session_state.df['Sentiment'] = sentiments st.success("Sentiment analysis results added to the dataset") st.rerun() except ImportError: st.error("NLTK Vader not available. Try using 'Simple Lexicon-based' method instead.") else: # Simple Lexicon-based # Load positive and negative word lists positive_words = [ 'good', 'great', 'excellent', 'positive', 'nice', 'wonderful', 'amazing', 'love', 'happy', 'best', 'better', 'awesome', 'fantastic', 'superior', 'perfect', 'outstanding', 'remarkable', 'superb', 'brilliant', 'exceptional' ] negative_words = [ 'bad', 'terrible', 'negative', 'poor', 'awful', 'horrible', 'worst', 'hate', 'sad', 'disappointing', 'inferior', 'mediocre', 'useless', 'unpleasant', 'failure', 'pathetic', 'terrible', 'dreadful', 'poor' ] # Get processed texts for better analysis processed_texts = st.session_state.processed_texts # Calculate sentiment scores positive_scores = [] negative_scores = [] sentiment_scores = [] sentiments = [] for text in processed_texts: words = text.split() # Count positive and negative words pos_count = sum(1 for word in words if word in positive_words) neg_count = sum(1 for word in words if word in negative_words) # Calculate score if len(words) > 0: pos_score = pos_count / len(words) neg_score = neg_count / len(words) sentiment_score = pos_score - neg_score else: pos_score = 0 neg_score = 0 sentiment_score = 0 positive_scores.append(pos_score) negative_scores.append(neg_score) sentiment_scores.append(sentiment_score) # Determine sentiment label if sentiment_score > 0.05: sentiments.append('positive') elif sentiment_score < -0.05: sentiments.append('negative') else: sentiments.append('neutral') # Create DataFrame with sentiment results sentiment_df = pd.DataFrame({ 'Text': st.session_state.text_data, 'Positive Score': positive_scores, 'Negative Score': negative_scores, 'Sentiment Score': sentiment_scores, 'Sentiment': sentiments }) # Calculate sentiment distribution sentiment_counts = sentiment_df['Sentiment'].value_counts() # Display sentiment distribution st.subheader("Sentiment Distribution") fig = px.pie( values=sentiment_counts.values, names=sentiment_counts.index, title='Distribution of Sentiments', color_discrete_map={ 'positive': 'green', 'neutral': 'gray', 'negative': 'red' }, height=500 ) st.plotly_chart(fig, use_container_width=True) # Display sentiment scores for a sample of documents st.subheader("Sample Document Sentiment Scores") st.dataframe(sentiment_df.head(10), use_container_width=True) # Option to add sentiment to the dataset if st.button("Add Sentiment Analysis to Dataset"): st.session_state.df['Positive_Score'] = positive_scores st.session_state.df['Negative_Score'] = negative_scores st.session_state.df['Sentiment_Score'] = sentiment_scores st.session_state.df['Sentiment'] = sentiments st.success("Sentiment analysis results added to the dataset") st.rerun() except Exception as e: st.error(f"Error analyzing sentiment: {e}") # Sidebar with educational content with st.sidebar: st.title("📚 NLP Learning Corner") st.markdown(""" ### What is NLP? Natural Language Processing (NLP) is a field of artificial intelligence that focuses on interactions between computers and human language. It enables computers to understand, interpret, and generate human language in useful ways. ### Key NLP Concepts **Text Preprocessing Steps:** - **Tokenization**: Breaking text into words, phrases, or other meaningful elements - **Stop Word Removal**: Removing common words that add little meaning (a, the, is, etc.) - **Stemming**: Reducing words to their root/stem form (running → run) - **Lemmatization**: Similar to stemming but ensures the result is a valid word (better → good) **Text Analysis Techniques:** - **Word Frequency**: Counting word occurrences to identify important terms - **Word Clouds**: Visual representations of word frequency - **Sentiment Analysis**: Determining if text expresses positive, negative, or neutral sentiment - **Topic Modeling**: Identifying themes or topics within a collection of documents ### Applications of NLP - **Sentiment Analysis**: Analyzing customer reviews, social media, surveys - **Text Classification**: Categorizing documents by topic, intent, or type - **Information Extraction**: Pulling structured data from unstructured text - **Text Summarization**: Creating concise summaries of longer documents - **Language Translation**: Converting text between languages - **Chatbots and Virtual Assistants**: Creating natural language interfaces """)