Anshul Prasad commited on
Commit ·
3a08458
1
Parent(s): 5708e62
files structure changed.
Browse filessome files are moved.
directory names are changed.
main.py
CHANGED
|
@@ -2,15 +2,14 @@ import logging
|
|
| 2 |
import pickle
|
| 3 |
from pathlib import Path
|
| 4 |
|
| 5 |
-
from
|
| 6 |
-
from
|
| 7 |
-
from
|
| 8 |
-
from
|
| 9 |
-
from
|
| 10 |
-
from
|
| 11 |
-
from api.embed_transcripts import embedding
|
| 12 |
from config import CHANNEL_URLS, VTT_DIR, TXT_DIR, TRANSCRIPT_INDEX, RETRIEVED_TRANSCRIPTS_FILE, RESPONSE_FILE, \
|
| 13 |
-
FILE_PATHS, TRANSCRIPTS,
|
| 14 |
|
| 15 |
logger = logging.getLogger(__name__)
|
| 16 |
|
|
@@ -30,7 +29,6 @@ def stage_download() -> None:
|
|
| 30 |
|
| 31 |
def stage_preprocess() -> tuple[list[Path], list[str]]:
|
| 32 |
vtt_to_txt(VTT_DIR, TXT_DIR)
|
| 33 |
-
deduplicate_consecutive_lines(TXT_DIR)
|
| 34 |
return load_text_corpus(TXT_DIR)
|
| 35 |
|
| 36 |
|
|
|
|
| 2 |
import pickle
|
| 3 |
from pathlib import Path
|
| 4 |
|
| 5 |
+
from src.download_vtt import download_channel_subtitles
|
| 6 |
+
from src.vtt_to_txt import vtt_to_txt
|
| 7 |
+
from src.preprocess import load_text_corpus
|
| 8 |
+
from src.retrieve_context import retrieve_transcripts
|
| 9 |
+
from src.generate_response import generate_response
|
| 10 |
+
from src.embed_transcripts import embedding
|
|
|
|
| 11 |
from config import CHANNEL_URLS, VTT_DIR, TXT_DIR, TRANSCRIPT_INDEX, RETRIEVED_TRANSCRIPTS_FILE, RESPONSE_FILE, \
|
| 12 |
+
FILE_PATHS, TRANSCRIPTS, CHUNKS_PKL
|
| 13 |
|
| 14 |
logger = logging.getLogger(__name__)
|
| 15 |
|
|
|
|
| 29 |
|
| 30 |
def stage_preprocess() -> tuple[list[Path], list[str]]:
|
| 31 |
vtt_to_txt(VTT_DIR, TXT_DIR)
|
|
|
|
| 32 |
return load_text_corpus(TXT_DIR)
|
| 33 |
|
| 34 |
|