USF00 commited on
Commit
d7630f2
·
1 Parent(s): 62d16a9

Fix lazy loading bug

Browse files
Files changed (1) hide show
  1. summarizer.py +4 -0
summarizer.py CHANGED
@@ -49,9 +49,13 @@ class BookSummarizer:
49
  logger.info("Model loaded successfully.")
50
 
51
  def tok_len(self, s: str) -> int:
 
 
52
  return len(self.tokenizer.encode(s, add_special_tokens=False))
53
 
54
  def split_by_tokens(self, s: str, max_len: int, overlap_tokens: int = 64):
 
 
55
  ids = self.tokenizer.encode(s, add_special_tokens=False)
56
  if len(ids) <= max_len:
57
  return [s.strip()]
 
49
  logger.info("Model loaded successfully.")
50
 
51
  def tok_len(self, s: str) -> int:
52
+ if not self.tokenizer:
53
+ self.load_model()
54
  return len(self.tokenizer.encode(s, add_special_tokens=False))
55
 
56
  def split_by_tokens(self, s: str, max_len: int, overlap_tokens: int = 64):
57
+ if not self.tokenizer:
58
+ self.load_model()
59
  ids = self.tokenizer.encode(s, add_special_tokens=False)
60
  if len(ids) <= max_len:
61
  return [s.strip()]