Yuvvan Talreja commited on
Commit
75f90c1
·
1 Parent(s): adea0aa
Files changed (2) hide show
  1. Dockerfile +2 -2
  2. topic_segmenter.py +10 -0
Dockerfile CHANGED
@@ -23,8 +23,8 @@ RUN python -c "from sentence_transformers import SentenceTransformer; SentenceTr
23
  # Skip Whisper model preloading - it will be downloaded on first use
24
  # We don't need to preload it during build
25
 
26
- # Run NLTK downloads
27
- RUN python -m nltk.downloader punkt wordnet
28
 
29
  # Cache the torch hub directory to avoid unnecessary downloads
30
  RUN mkdir -p /root/.cache/torch/hub
 
23
  # Skip Whisper model preloading - it will be downloaded on first use
24
  # We don't need to preload it during build
25
 
26
+ # Run NLTK downloads - add all needed NLTK data
27
+ RUN python -c "import nltk; nltk.download('punkt'); nltk.download('wordnet'); nltk.download('omw-1.4')"
28
 
29
  # Cache the torch hub directory to avoid unnecessary downloads
30
  RUN mkdir -p /root/.cache/torch/hub
topic_segmenter.py CHANGED
@@ -7,6 +7,16 @@ from keybert import KeyBERT
7
  import re
8
  from collections import defaultdict
9
  from sklearn.cluster import AgglomerativeClustering
 
 
 
 
 
 
 
 
 
 
10
 
11
  class TopicSegmenter:
12
  def __init__(self, window_size=3, similarity_threshold=0.2, context_size=2, min_segment_size=2, topic_similarity_threshold=0.3,
 
7
  import re
8
  from collections import defaultdict
9
  from sklearn.cluster import AgglomerativeClustering
10
+ import logging
11
+
12
+ # Ensure NLTK resources are available
13
+ try:
14
+ import nltk
15
+ nltk.download('punkt', quiet=True)
16
+ nltk.download('wordnet', quiet=True)
17
+ nltk.download('omw-1.4', quiet=True)
18
+ except Exception as e:
19
+ logging.warning(f"NLTK resource download issue: {str(e)}")
20
 
21
  class TopicSegmenter:
22
  def __init__(self, window_size=3, similarity_threshold=0.2, context_size=2, min_segment_size=2, topic_similarity_threshold=0.3,