Spaces:
Sleeping
Sleeping
Fix lazy loading bug
Browse files- summarizer.py +4 -0
summarizer.py
CHANGED
|
@@ -49,9 +49,13 @@ class BookSummarizer:
|
|
| 49 |
logger.info("Model loaded successfully.")
|
| 50 |
|
| 51 |
def tok_len(self, s: str) -> int:
|
|
|
|
|
|
|
| 52 |
return len(self.tokenizer.encode(s, add_special_tokens=False))
|
| 53 |
|
| 54 |
def split_by_tokens(self, s: str, max_len: int, overlap_tokens: int = 64):
|
|
|
|
|
|
|
| 55 |
ids = self.tokenizer.encode(s, add_special_tokens=False)
|
| 56 |
if len(ids) <= max_len:
|
| 57 |
return [s.strip()]
|
|
|
|
| 49 |
logger.info("Model loaded successfully.")
|
| 50 |
|
| 51 |
def tok_len(self, s: str) -> int:
|
| 52 |
+
if not self.tokenizer:
|
| 53 |
+
self.load_model()
|
| 54 |
return len(self.tokenizer.encode(s, add_special_tokens=False))
|
| 55 |
|
| 56 |
def split_by_tokens(self, s: str, max_len: int, overlap_tokens: int = 64):
|
| 57 |
+
if not self.tokenizer:
|
| 58 |
+
self.load_model()
|
| 59 |
ids = self.tokenizer.encode(s, add_special_tokens=False)
|
| 60 |
if len(ids) <= max_len:
|
| 61 |
return [s.strip()]
|