Anshul Prasad commited on
Commit
3a08458
·
1 Parent(s): 5708e62

files structure changed.

Browse files

some files are moved.
directory names are changed.

Files changed (1) hide show
  1. main.py +7 -9
main.py CHANGED
@@ -2,15 +2,14 @@ import logging
2
  import pickle
3
  from pathlib import Path
4
 
5
- from utils.download_vtt import download_channel_subtitles
6
- from utils.vtt_to_txt import vtt_to_txt
7
- from utils.preprocess import deduplicate_consecutive_lines, load_text_corpus
8
- from utils.token import count_tokens, trim_to_token_limit
9
- from api.retrieve_context import retrieve_transcripts
10
- from api.generate_response import generate_response
11
- from api.embed_transcripts import embedding
12
  from config import CHANNEL_URLS, VTT_DIR, TXT_DIR, TRANSCRIPT_INDEX, RETRIEVED_TRANSCRIPTS_FILE, RESPONSE_FILE, \
13
- FILE_PATHS, TRANSCRIPTS, MAX_CONTEXT_TOKENS, CHUNKS_PKL
14
 
15
  logger = logging.getLogger(__name__)
16
 
@@ -30,7 +29,6 @@ def stage_download() -> None:
30
 
31
  def stage_preprocess() -> tuple[list[Path], list[str]]:
32
  vtt_to_txt(VTT_DIR, TXT_DIR)
33
- deduplicate_consecutive_lines(TXT_DIR)
34
  return load_text_corpus(TXT_DIR)
35
 
36
 
 
2
  import pickle
3
  from pathlib import Path
4
 
5
+ from src.download_vtt import download_channel_subtitles
6
+ from src.vtt_to_txt import vtt_to_txt
7
+ from src.preprocess import load_text_corpus
8
+ from src.retrieve_context import retrieve_transcripts
9
+ from src.generate_response import generate_response
10
+ from src.embed_transcripts import embedding
 
11
  from config import CHANNEL_URLS, VTT_DIR, TXT_DIR, TRANSCRIPT_INDEX, RETRIEVED_TRANSCRIPTS_FILE, RESPONSE_FILE, \
12
+ FILE_PATHS, TRANSCRIPTS, CHUNKS_PKL
13
 
14
  logger = logging.getLogger(__name__)
15
 
 
29
 
30
  def stage_preprocess() -> tuple[list[Path], list[str]]:
31
  vtt_to_txt(VTT_DIR, TXT_DIR)
 
32
  return load_text_corpus(TXT_DIR)
33
 
34