Ayesha352 commited on
Commit
15634f7
·
verified ·
1 Parent(s): f40f343

Upload 3 files

Browse files
Files changed (3) hide show
  1. README.md +8 -8
  2. app.py +249 -0
  3. requirements.txt +3 -0
README.md CHANGED
@@ -1,15 +1,15 @@
1
  ---
2
  title: NLP Text Lab
3
- emoji: 📈
4
- colorFrom: red
5
- colorTo: red
6
  sdk: gradio
7
- sdk_version: 6.30.0
8
- python_version: '3.12'
9
  app_file: app.py
10
  pinned: false
11
- license: mit
12
- short_description: Tokenization, stop words removal, stemming aur lemmatization
13
  ---
14
 
15
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
1
  ---
2
  title: NLP Text Lab
3
+ emoji: 🧪
4
+ colorFrom: indigo
5
+ colorTo: purple
6
  sdk: gradio
7
+ sdk_version: 5.0.0
 
8
  app_file: app.py
9
  pinned: false
 
 
10
  ---
11
 
12
+ # NLP Text Lab
13
+
14
+ Tokenization, stop-word removal, stemming, lemmatization, POS tagging and a full
15
+ preprocessing pipeline, built with NLTK + spaCy + Gradio.
app.py ADDED
@@ -0,0 +1,249 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ NLP Text Lab - Gradio app for Hugging Face Spaces
3
+ Tasks: Tokenization, Stop-word removal, Stemming, Lemmatization,
4
+ POS tagging + word frequency, Full preprocessing pipeline.
5
+ """
6
+ import string
7
+ from collections import Counter
8
+
9
+ import gradio as gr
10
+ import nltk
11
+ import pandas as pd
12
+ import spacy
13
+ from nltk.corpus import stopwords, wordnet
14
+ from nltk.stem import (LancasterStemmer, PorterStemmer, SnowballStemmer,
15
+ WordNetLemmatizer)
16
+ from nltk.tokenize import sent_tokenize, word_tokenize
17
+
18
+ # ---------------------------------------------------------------- setup ----
19
+ for pkg in ["punkt", "punkt_tab", "stopwords", "wordnet", "omw-1.4",
20
+ "averaged_perceptron_tagger", "averaged_perceptron_tagger_eng"]:
21
+ nltk.download(pkg, quiet=True)
22
+
23
+
24
+ def load_spacy():
25
+ try:
26
+ return spacy.load("en_core_web_sm")
27
+ except OSError:
28
+ from spacy.cli import download
29
+ download("en_core_web_sm")
30
+ return spacy.load("en_core_web_sm")
31
+
32
+
33
+ nlp = load_spacy()
34
+ STOP_WORDS = set(stopwords.words("english"))
35
+ porter, lancaster = PorterStemmer(), LancasterStemmer()
36
+ snowball = SnowballStemmer("english")
37
+ wnl = WordNetLemmatizer()
38
+
39
+ DEFAULT_TEXT = (
40
+ "Consistency is the key to success.\n"
41
+ "Every small step we take today brings us closer to the future we dream of.\n"
42
+ "The journey may be difficult, and sometimes we may feel tired or lost,\n"
43
+ "but we must keep moving forward. Growth takes time, patience, and courage."
44
+ )
45
+
46
+
47
+ # -------------------------------------------------------------- helpers ----
48
+ def _clean(text):
49
+ return (text or "").strip()
50
+
51
+
52
+ def _is_punct(tok):
53
+ return all(ch in string.punctuation for ch in tok)
54
+
55
+
56
+ def _wn_pos(tag):
57
+ """Convert Penn Treebank tag to WordNet POS."""
58
+ if tag.startswith("J"):
59
+ return wordnet.ADJ
60
+ if tag.startswith("V"):
61
+ return wordnet.VERB
62
+ if tag.startswith("R"):
63
+ return wordnet.ADV
64
+ return wordnet.NOUN
65
+
66
+
67
+ # ---------------------------------------------------------------- tasks ----
68
+ def task_tokenize(text):
69
+ text = _clean(text)
70
+ if not text:
71
+ return "Please enter a sentence.", "", "", ""
72
+ words = word_tokenize(text)
73
+ sents = sent_tokenize(text)
74
+ doc = nlp(text)
75
+ spacy_tokens = [t.text for t in doc if not t.is_space]
76
+ spacy_sents = [s.text.strip() for s in doc.sents]
77
+
78
+ nltk_out = (f"WORD TOKENS ({len(words)}):\n{words}\n\n"
79
+ f"SENTENCES ({len(sents)}):\n" +
80
+ "\n".join(f"{i+1}. {s}" for i, s in enumerate(sents)))
81
+ spacy_out = (f"WORD TOKENS ({len(spacy_tokens)}):\n{spacy_tokens}\n\n"
82
+ f"SENTENCES ({len(spacy_sents)}):\n" +
83
+ "\n".join(f"{i+1}. {s}" for i, s in enumerate(spacy_sents)))
84
+ summary = (f"No. of tokens (NLTK): {len(words)} | "
85
+ f"No. of sentences (NLTK): {len(sents)}")
86
+ return summary, nltk_out, spacy_out, ""
87
+
88
+
89
+ def task_stopwords(text, remove_punct, lowercase):
90
+ text = _clean(text)
91
+ if not text:
92
+ return "Please enter a sentence.", "", "", ""
93
+ tokens = word_tokenize(text)
94
+ kept, removed = [], []
95
+ for tok in tokens:
96
+ if remove_punct and _is_punct(tok):
97
+ removed.append(tok)
98
+ continue
99
+ if tok.lower() in STOP_WORDS: # case-insensitive match
100
+ removed.append(tok)
101
+ else:
102
+ kept.append(tok.lower() if lowercase else tok)
103
+ summary = (f"Original tokens: {len(tokens)} | Removed: {len(removed)} | "
104
+ f"Remaining: {len(kept)}")
105
+ return summary, str(tokens), str(removed), " ".join(kept)
106
+
107
+
108
+ def task_stemming(text):
109
+ text = _clean(text)
110
+ if not text:
111
+ return pd.DataFrame(columns=["Word", "Porter", "Lancaster", "Snowball"])
112
+ toks = [t for t in word_tokenize(text) if not _is_punct(t)]
113
+ rows = [[t, porter.stem(t), lancaster.stem(t), snowball.stem(t)] for t in toks]
114
+ return pd.DataFrame(rows, columns=["Word", "Porter", "Lancaster", "Snowball"])
115
+
116
+
117
+ def task_lemmatize(text):
118
+ text = _clean(text)
119
+ empty1 = pd.DataFrame(columns=["Word", "POS tag", "Lemma (default=noun)",
120
+ "Lemma (POS-aware)"])
121
+ empty2 = pd.DataFrame(columns=["Word", "POS", "spaCy Lemma"])
122
+ if not text:
123
+ return empty1, empty2
124
+ toks = [t for t in word_tokenize(text) if not _is_punct(t)]
125
+ rows = []
126
+ for tok, tag in nltk.pos_tag(toks):
127
+ rows.append([tok, tag, wnl.lemmatize(tok.lower()),
128
+ wnl.lemmatize(tok.lower(), _wn_pos(tag))])
129
+ doc = nlp(text)
130
+ rows2 = [[t.text, t.pos_, t.lemma_] for t in doc
131
+ if not t.is_space and not t.is_punct]
132
+ return (pd.DataFrame(rows, columns=empty1.columns),
133
+ pd.DataFrame(rows2, columns=empty2.columns))
134
+
135
+
136
+ def task_pos_freq(text):
137
+ text = _clean(text)
138
+ if not text:
139
+ return (pd.DataFrame(columns=["Word", "POS tag"]),
140
+ pd.DataFrame(columns=["Word", "Count"]))
141
+ toks = [t for t in word_tokenize(text) if not _is_punct(t)]
142
+ pos_df = pd.DataFrame(nltk.pos_tag(toks), columns=["Word", "POS tag"])
143
+ words = [t.lower() for t in toks if t.lower() not in STOP_WORDS]
144
+ freq_df = pd.DataFrame(Counter(words).most_common(),
145
+ columns=["Word", "Count"])
146
+ return pos_df, freq_df
147
+
148
+
149
+ def task_pipeline(text):
150
+ text = _clean(text)
151
+ if not text:
152
+ return ["Please enter a sentence."] + [""] * 5
153
+ tokens = word_tokenize(text.lower())
154
+ no_punct = [t for t in tokens if not _is_punct(t)]
155
+ no_stop = [t for t in no_punct if t not in STOP_WORDS]
156
+ stems = [porter.stem(t) for t in no_stop]
157
+ lemmas = [wnl.lemmatize(t, _wn_pos(tag)) for t, tag in nltk.pos_tag(no_stop)]
158
+ return (f"1. Lowercase + Tokenize\n{tokens}", f"2. Remove punctuation\n{no_punct}",
159
+ f"3. Remove stop words\n{no_stop}", f"4. Stemming (Porter)\n{stems}",
160
+ f"5. Lemmatization (POS-aware)\n{lemmas}",
161
+ " ".join(lemmas))
162
+
163
+
164
+ LEARN_MD = """
165
+ ## Stemming ("trimming") vs Lemmatization
166
+
167
+ **Stemming** chops the ending (suffix) off a word using simple rules, so the
168
+ result may *not* be a real word. It is fast but rough.
169
+
170
+ | Word | Stem (Porter) |
171
+ |---|---|
172
+ | studies | studi |
173
+ | running | run |
174
+ | difficult | difficult |
175
+ | happily | happili |
176
+
177
+ **Lemmatization** uses a dictionary (WordNet) and the word's part of speech to
178
+ return the real base form (the *lemma*). Slower but accurate.
179
+
180
+ | Word | POS | Lemma |
181
+ |---|---|---|
182
+ | studies | noun/verb | study |
183
+ | running | verb | run |
184
+ | better | adjective | good |
185
+ | mice | noun | mouse |
186
+
187
+ **When to use what?** Stemming for search engines/speed, lemmatization when
188
+ meaning matters (chatbots, sentiment analysis, NLP models).
189
+
190
+ ## Stop words
191
+ Very common words (the, is, to, we ...) that carry little meaning. They are
192
+ removed to keep only the useful words.
193
+ """
194
+
195
+ # ------------------------------------------------------------------- UI ----
196
+ with gr.Blocks(title="NLP Text Lab") as demo:
197
+ gr.Markdown("# 🧪 NLP Text Lab\nType a sentence/paragraph, then open any "
198
+ "tab and press the button.")
199
+ text_in = gr.Textbox(label="Your text", lines=5, value=DEFAULT_TEXT)
200
+
201
+ with gr.Tab("1. Tokenization"):
202
+ b1 = gr.Button("Tokenize", variant="primary")
203
+ s1 = gr.Textbox(label="Summary")
204
+ with gr.Row():
205
+ n1 = gr.Textbox(label="NLTK", lines=12)
206
+ sp1 = gr.Textbox(label="spaCy", lines=12)
207
+ hid = gr.Textbox(visible=False)
208
+ b1.click(task_tokenize, text_in, [s1, n1, sp1, hid])
209
+
210
+ with gr.Tab("2. Stop Words Removal"):
211
+ with gr.Row():
212
+ rp = gr.Checkbox(True, label="Also remove punctuation")
213
+ lc = gr.Checkbox(False, label="Lowercase result")
214
+ b2 = gr.Button("Remove stop words", variant="primary")
215
+ s2 = gr.Textbox(label="Summary")
216
+ t2 = gr.Textbox(label="All tokens", lines=3)
217
+ r2 = gr.Textbox(label="Removed (stop words / punctuation)", lines=3)
218
+ f2 = gr.Textbox(label="Filtered sentence", lines=3)
219
+ b2.click(task_stopwords, [text_in, rp, lc], [s2, t2, r2, f2])
220
+
221
+ with gr.Tab("3. Stemming"):
222
+ b3 = gr.Button("Apply stemming", variant="primary")
223
+ o3 = gr.Dataframe(label="Stemming results (Porter / Lancaster / Snowball)")
224
+ b3.click(task_stemming, text_in, o3)
225
+
226
+ with gr.Tab("4. Lemmatization"):
227
+ b4 = gr.Button("Apply lemmatization", variant="primary")
228
+ o4a = gr.Dataframe(label="NLTK WordNet Lemmatizer")
229
+ o4b = gr.Dataframe(label="spaCy Lemmatizer")
230
+ b4.click(task_lemmatize, text_in, [o4a, o4b])
231
+
232
+ with gr.Tab("5. POS Tags + Word Frequency"):
233
+ b5 = gr.Button("Analyse", variant="primary")
234
+ with gr.Row():
235
+ o5a = gr.Dataframe(label="Part-of-speech tags")
236
+ o5b = gr.Dataframe(label="Word frequency (without stop words)")
237
+ b5.click(task_pos_freq, text_in, [o5a, o5b])
238
+
239
+ with gr.Tab("6. Full Pipeline"):
240
+ b6 = gr.Button("Run full pipeline", variant="primary")
241
+ p = [gr.Textbox(lines=3, show_label=False) for _ in range(5)]
242
+ final = gr.Textbox(label="Final cleaned text", lines=2)
243
+ b6.click(task_pipeline, text_in, p + [final])
244
+
245
+ with gr.Tab("📖 Learn"):
246
+ gr.Markdown(LEARN_MD)
247
+
248
+ if __name__ == "__main__":
249
+ demo.launch()
requirements.txt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ nltk>=3.9
2
+ spacy>=3.7
3
+ pandas