Spaces:
Sleeping
Sleeping
Upload 3 files
Browse files- README.md +8 -8
- app.py +249 -0
- requirements.txt +3 -0
README.md
CHANGED
|
@@ -1,15 +1,15 @@
|
|
| 1 |
---
|
| 2 |
title: NLP Text Lab
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
sdk: gradio
|
| 7 |
-
sdk_version:
|
| 8 |
-
python_version: '3.12'
|
| 9 |
app_file: app.py
|
| 10 |
pinned: false
|
| 11 |
-
license: mit
|
| 12 |
-
short_description: Tokenization, stop words removal, stemming aur lemmatization
|
| 13 |
---
|
| 14 |
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
title: NLP Text Lab
|
| 3 |
+
emoji: 🧪
|
| 4 |
+
colorFrom: indigo
|
| 5 |
+
colorTo: purple
|
| 6 |
sdk: gradio
|
| 7 |
+
sdk_version: 5.0.0
|
|
|
|
| 8 |
app_file: app.py
|
| 9 |
pinned: false
|
|
|
|
|
|
|
| 10 |
---
|
| 11 |
|
| 12 |
+
# NLP Text Lab
|
| 13 |
+
|
| 14 |
+
Tokenization, stop-word removal, stemming, lemmatization, POS tagging and a full
|
| 15 |
+
preprocessing pipeline, built with NLTK + spaCy + Gradio.
|
app.py
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
NLP Text Lab - Gradio app for Hugging Face Spaces
|
| 3 |
+
Tasks: Tokenization, Stop-word removal, Stemming, Lemmatization,
|
| 4 |
+
POS tagging + word frequency, Full preprocessing pipeline.
|
| 5 |
+
"""
|
| 6 |
+
import string
|
| 7 |
+
from collections import Counter
|
| 8 |
+
|
| 9 |
+
import gradio as gr
|
| 10 |
+
import nltk
|
| 11 |
+
import pandas as pd
|
| 12 |
+
import spacy
|
| 13 |
+
from nltk.corpus import stopwords, wordnet
|
| 14 |
+
from nltk.stem import (LancasterStemmer, PorterStemmer, SnowballStemmer,
|
| 15 |
+
WordNetLemmatizer)
|
| 16 |
+
from nltk.tokenize import sent_tokenize, word_tokenize
|
| 17 |
+
|
| 18 |
+
# ---------------------------------------------------------------- setup ----
|
| 19 |
+
for pkg in ["punkt", "punkt_tab", "stopwords", "wordnet", "omw-1.4",
|
| 20 |
+
"averaged_perceptron_tagger", "averaged_perceptron_tagger_eng"]:
|
| 21 |
+
nltk.download(pkg, quiet=True)
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
def load_spacy():
|
| 25 |
+
try:
|
| 26 |
+
return spacy.load("en_core_web_sm")
|
| 27 |
+
except OSError:
|
| 28 |
+
from spacy.cli import download
|
| 29 |
+
download("en_core_web_sm")
|
| 30 |
+
return spacy.load("en_core_web_sm")
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
nlp = load_spacy()
|
| 34 |
+
STOP_WORDS = set(stopwords.words("english"))
|
| 35 |
+
porter, lancaster = PorterStemmer(), LancasterStemmer()
|
| 36 |
+
snowball = SnowballStemmer("english")
|
| 37 |
+
wnl = WordNetLemmatizer()
|
| 38 |
+
|
| 39 |
+
DEFAULT_TEXT = (
|
| 40 |
+
"Consistency is the key to success.\n"
|
| 41 |
+
"Every small step we take today brings us closer to the future we dream of.\n"
|
| 42 |
+
"The journey may be difficult, and sometimes we may feel tired or lost,\n"
|
| 43 |
+
"but we must keep moving forward. Growth takes time, patience, and courage."
|
| 44 |
+
)
|
| 45 |
+
|
| 46 |
+
|
| 47 |
+
# -------------------------------------------------------------- helpers ----
|
| 48 |
+
def _clean(text):
|
| 49 |
+
return (text or "").strip()
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def _is_punct(tok):
|
| 53 |
+
return all(ch in string.punctuation for ch in tok)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _wn_pos(tag):
|
| 57 |
+
"""Convert Penn Treebank tag to WordNet POS."""
|
| 58 |
+
if tag.startswith("J"):
|
| 59 |
+
return wordnet.ADJ
|
| 60 |
+
if tag.startswith("V"):
|
| 61 |
+
return wordnet.VERB
|
| 62 |
+
if tag.startswith("R"):
|
| 63 |
+
return wordnet.ADV
|
| 64 |
+
return wordnet.NOUN
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# ---------------------------------------------------------------- tasks ----
|
| 68 |
+
def task_tokenize(text):
|
| 69 |
+
text = _clean(text)
|
| 70 |
+
if not text:
|
| 71 |
+
return "Please enter a sentence.", "", "", ""
|
| 72 |
+
words = word_tokenize(text)
|
| 73 |
+
sents = sent_tokenize(text)
|
| 74 |
+
doc = nlp(text)
|
| 75 |
+
spacy_tokens = [t.text for t in doc if not t.is_space]
|
| 76 |
+
spacy_sents = [s.text.strip() for s in doc.sents]
|
| 77 |
+
|
| 78 |
+
nltk_out = (f"WORD TOKENS ({len(words)}):\n{words}\n\n"
|
| 79 |
+
f"SENTENCES ({len(sents)}):\n" +
|
| 80 |
+
"\n".join(f"{i+1}. {s}" for i, s in enumerate(sents)))
|
| 81 |
+
spacy_out = (f"WORD TOKENS ({len(spacy_tokens)}):\n{spacy_tokens}\n\n"
|
| 82 |
+
f"SENTENCES ({len(spacy_sents)}):\n" +
|
| 83 |
+
"\n".join(f"{i+1}. {s}" for i, s in enumerate(spacy_sents)))
|
| 84 |
+
summary = (f"No. of tokens (NLTK): {len(words)} | "
|
| 85 |
+
f"No. of sentences (NLTK): {len(sents)}")
|
| 86 |
+
return summary, nltk_out, spacy_out, ""
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def task_stopwords(text, remove_punct, lowercase):
|
| 90 |
+
text = _clean(text)
|
| 91 |
+
if not text:
|
| 92 |
+
return "Please enter a sentence.", "", "", ""
|
| 93 |
+
tokens = word_tokenize(text)
|
| 94 |
+
kept, removed = [], []
|
| 95 |
+
for tok in tokens:
|
| 96 |
+
if remove_punct and _is_punct(tok):
|
| 97 |
+
removed.append(tok)
|
| 98 |
+
continue
|
| 99 |
+
if tok.lower() in STOP_WORDS: # case-insensitive match
|
| 100 |
+
removed.append(tok)
|
| 101 |
+
else:
|
| 102 |
+
kept.append(tok.lower() if lowercase else tok)
|
| 103 |
+
summary = (f"Original tokens: {len(tokens)} | Removed: {len(removed)} | "
|
| 104 |
+
f"Remaining: {len(kept)}")
|
| 105 |
+
return summary, str(tokens), str(removed), " ".join(kept)
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def task_stemming(text):
|
| 109 |
+
text = _clean(text)
|
| 110 |
+
if not text:
|
| 111 |
+
return pd.DataFrame(columns=["Word", "Porter", "Lancaster", "Snowball"])
|
| 112 |
+
toks = [t for t in word_tokenize(text) if not _is_punct(t)]
|
| 113 |
+
rows = [[t, porter.stem(t), lancaster.stem(t), snowball.stem(t)] for t in toks]
|
| 114 |
+
return pd.DataFrame(rows, columns=["Word", "Porter", "Lancaster", "Snowball"])
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def task_lemmatize(text):
|
| 118 |
+
text = _clean(text)
|
| 119 |
+
empty1 = pd.DataFrame(columns=["Word", "POS tag", "Lemma (default=noun)",
|
| 120 |
+
"Lemma (POS-aware)"])
|
| 121 |
+
empty2 = pd.DataFrame(columns=["Word", "POS", "spaCy Lemma"])
|
| 122 |
+
if not text:
|
| 123 |
+
return empty1, empty2
|
| 124 |
+
toks = [t for t in word_tokenize(text) if not _is_punct(t)]
|
| 125 |
+
rows = []
|
| 126 |
+
for tok, tag in nltk.pos_tag(toks):
|
| 127 |
+
rows.append([tok, tag, wnl.lemmatize(tok.lower()),
|
| 128 |
+
wnl.lemmatize(tok.lower(), _wn_pos(tag))])
|
| 129 |
+
doc = nlp(text)
|
| 130 |
+
rows2 = [[t.text, t.pos_, t.lemma_] for t in doc
|
| 131 |
+
if not t.is_space and not t.is_punct]
|
| 132 |
+
return (pd.DataFrame(rows, columns=empty1.columns),
|
| 133 |
+
pd.DataFrame(rows2, columns=empty2.columns))
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def task_pos_freq(text):
|
| 137 |
+
text = _clean(text)
|
| 138 |
+
if not text:
|
| 139 |
+
return (pd.DataFrame(columns=["Word", "POS tag"]),
|
| 140 |
+
pd.DataFrame(columns=["Word", "Count"]))
|
| 141 |
+
toks = [t for t in word_tokenize(text) if not _is_punct(t)]
|
| 142 |
+
pos_df = pd.DataFrame(nltk.pos_tag(toks), columns=["Word", "POS tag"])
|
| 143 |
+
words = [t.lower() for t in toks if t.lower() not in STOP_WORDS]
|
| 144 |
+
freq_df = pd.DataFrame(Counter(words).most_common(),
|
| 145 |
+
columns=["Word", "Count"])
|
| 146 |
+
return pos_df, freq_df
|
| 147 |
+
|
| 148 |
+
|
| 149 |
+
def task_pipeline(text):
|
| 150 |
+
text = _clean(text)
|
| 151 |
+
if not text:
|
| 152 |
+
return ["Please enter a sentence."] + [""] * 5
|
| 153 |
+
tokens = word_tokenize(text.lower())
|
| 154 |
+
no_punct = [t for t in tokens if not _is_punct(t)]
|
| 155 |
+
no_stop = [t for t in no_punct if t not in STOP_WORDS]
|
| 156 |
+
stems = [porter.stem(t) for t in no_stop]
|
| 157 |
+
lemmas = [wnl.lemmatize(t, _wn_pos(tag)) for t, tag in nltk.pos_tag(no_stop)]
|
| 158 |
+
return (f"1. Lowercase + Tokenize\n{tokens}", f"2. Remove punctuation\n{no_punct}",
|
| 159 |
+
f"3. Remove stop words\n{no_stop}", f"4. Stemming (Porter)\n{stems}",
|
| 160 |
+
f"5. Lemmatization (POS-aware)\n{lemmas}",
|
| 161 |
+
" ".join(lemmas))
|
| 162 |
+
|
| 163 |
+
|
| 164 |
+
LEARN_MD = """
|
| 165 |
+
## Stemming ("trimming") vs Lemmatization
|
| 166 |
+
|
| 167 |
+
**Stemming** chops the ending (suffix) off a word using simple rules, so the
|
| 168 |
+
result may *not* be a real word. It is fast but rough.
|
| 169 |
+
|
| 170 |
+
| Word | Stem (Porter) |
|
| 171 |
+
|---|---|
|
| 172 |
+
| studies | studi |
|
| 173 |
+
| running | run |
|
| 174 |
+
| difficult | difficult |
|
| 175 |
+
| happily | happili |
|
| 176 |
+
|
| 177 |
+
**Lemmatization** uses a dictionary (WordNet) and the word's part of speech to
|
| 178 |
+
return the real base form (the *lemma*). Slower but accurate.
|
| 179 |
+
|
| 180 |
+
| Word | POS | Lemma |
|
| 181 |
+
|---|---|---|
|
| 182 |
+
| studies | noun/verb | study |
|
| 183 |
+
| running | verb | run |
|
| 184 |
+
| better | adjective | good |
|
| 185 |
+
| mice | noun | mouse |
|
| 186 |
+
|
| 187 |
+
**When to use what?** Stemming for search engines/speed, lemmatization when
|
| 188 |
+
meaning matters (chatbots, sentiment analysis, NLP models).
|
| 189 |
+
|
| 190 |
+
## Stop words
|
| 191 |
+
Very common words (the, is, to, we ...) that carry little meaning. They are
|
| 192 |
+
removed to keep only the useful words.
|
| 193 |
+
"""
|
| 194 |
+
|
| 195 |
+
# ------------------------------------------------------------------- UI ----
|
| 196 |
+
with gr.Blocks(title="NLP Text Lab") as demo:
|
| 197 |
+
gr.Markdown("# 🧪 NLP Text Lab\nType a sentence/paragraph, then open any "
|
| 198 |
+
"tab and press the button.")
|
| 199 |
+
text_in = gr.Textbox(label="Your text", lines=5, value=DEFAULT_TEXT)
|
| 200 |
+
|
| 201 |
+
with gr.Tab("1. Tokenization"):
|
| 202 |
+
b1 = gr.Button("Tokenize", variant="primary")
|
| 203 |
+
s1 = gr.Textbox(label="Summary")
|
| 204 |
+
with gr.Row():
|
| 205 |
+
n1 = gr.Textbox(label="NLTK", lines=12)
|
| 206 |
+
sp1 = gr.Textbox(label="spaCy", lines=12)
|
| 207 |
+
hid = gr.Textbox(visible=False)
|
| 208 |
+
b1.click(task_tokenize, text_in, [s1, n1, sp1, hid])
|
| 209 |
+
|
| 210 |
+
with gr.Tab("2. Stop Words Removal"):
|
| 211 |
+
with gr.Row():
|
| 212 |
+
rp = gr.Checkbox(True, label="Also remove punctuation")
|
| 213 |
+
lc = gr.Checkbox(False, label="Lowercase result")
|
| 214 |
+
b2 = gr.Button("Remove stop words", variant="primary")
|
| 215 |
+
s2 = gr.Textbox(label="Summary")
|
| 216 |
+
t2 = gr.Textbox(label="All tokens", lines=3)
|
| 217 |
+
r2 = gr.Textbox(label="Removed (stop words / punctuation)", lines=3)
|
| 218 |
+
f2 = gr.Textbox(label="Filtered sentence", lines=3)
|
| 219 |
+
b2.click(task_stopwords, [text_in, rp, lc], [s2, t2, r2, f2])
|
| 220 |
+
|
| 221 |
+
with gr.Tab("3. Stemming"):
|
| 222 |
+
b3 = gr.Button("Apply stemming", variant="primary")
|
| 223 |
+
o3 = gr.Dataframe(label="Stemming results (Porter / Lancaster / Snowball)")
|
| 224 |
+
b3.click(task_stemming, text_in, o3)
|
| 225 |
+
|
| 226 |
+
with gr.Tab("4. Lemmatization"):
|
| 227 |
+
b4 = gr.Button("Apply lemmatization", variant="primary")
|
| 228 |
+
o4a = gr.Dataframe(label="NLTK WordNet Lemmatizer")
|
| 229 |
+
o4b = gr.Dataframe(label="spaCy Lemmatizer")
|
| 230 |
+
b4.click(task_lemmatize, text_in, [o4a, o4b])
|
| 231 |
+
|
| 232 |
+
with gr.Tab("5. POS Tags + Word Frequency"):
|
| 233 |
+
b5 = gr.Button("Analyse", variant="primary")
|
| 234 |
+
with gr.Row():
|
| 235 |
+
o5a = gr.Dataframe(label="Part-of-speech tags")
|
| 236 |
+
o5b = gr.Dataframe(label="Word frequency (without stop words)")
|
| 237 |
+
b5.click(task_pos_freq, text_in, [o5a, o5b])
|
| 238 |
+
|
| 239 |
+
with gr.Tab("6. Full Pipeline"):
|
| 240 |
+
b6 = gr.Button("Run full pipeline", variant="primary")
|
| 241 |
+
p = [gr.Textbox(lines=3, show_label=False) for _ in range(5)]
|
| 242 |
+
final = gr.Textbox(label="Final cleaned text", lines=2)
|
| 243 |
+
b6.click(task_pipeline, text_in, p + [final])
|
| 244 |
+
|
| 245 |
+
with gr.Tab("📖 Learn"):
|
| 246 |
+
gr.Markdown(LEARN_MD)
|
| 247 |
+
|
| 248 |
+
if __name__ == "__main__":
|
| 249 |
+
demo.launch()
|
requirements.txt
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
nltk>=3.9
|
| 2 |
+
spacy>=3.7
|
| 3 |
+
pandas
|