VictorM-Coder commited on
Commit
f52223f
·
verified ·
1 Parent(s): 6b6ee15

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +12 -14
app.py CHANGED
@@ -11,31 +11,29 @@ def paraphrase_text(text):
11
  input_ids = tokenizer.encode(input_text, return_tensors="pt", truncation=True)
12
  output_ids = model.generate(
13
  input_ids,
14
- max_length=1024, # allow longer output
15
  do_sample=True,
16
  top_k=120,
17
  top_p=0.95,
18
- temperature=1.2, # moderate creativity
19
- early_stopping=True
20
  )
21
  paraphrased = tokenizer.decode(output_ids[0], skip_special_tokens=True)
22
- # Fix spacing around punctuation
23
- paraphrased = re.sub(r'\s+([.,!?;:])', r'\1', paraphrased)
24
- paraphrased = re.sub(r'([(\["])\s+', r'\1', paraphrased)
25
- paraphrased = re.sub(r'\s+([)\]"])', r'\1', paraphrased)
26
- return paraphrased.strip()
27
 
28
- def chunk_text(text, max_sentences=6): # bigger chunks for better flow
 
 
 
 
 
 
 
29
  sentences = re.split(r'(?<=[.!?]) +', text.strip())
30
  return [' '.join(sentences[i:i+max_sentences]) for i in range(0, len(sentences), max_sentences)]
31
 
32
  def full_article_paraphrase(text):
33
  chunks = chunk_text(text)
34
- paraphrased_chunks = []
35
- for chunk in chunks:
36
- if chunk.strip():
37
- paraphrased_chunks.append(paraphrase_text(chunk.strip()))
38
- return "\n\n".join(paraphrased_chunks)
39
 
40
  def extract_text_from_docx(file_obj):
41
  from docx import Document
 
11
  input_ids = tokenizer.encode(input_text, return_tensors="pt", truncation=True)
12
  output_ids = model.generate(
13
  input_ids,
14
+ max_length=256,
15
  do_sample=True,
16
  top_k=120,
17
  top_p=0.95,
18
+ temperature=1.5
 
19
  )
20
  paraphrased = tokenizer.decode(output_ids[0], skip_special_tokens=True)
21
+ return fix_punctuation(paraphrased)
 
 
 
 
22
 
23
+ def fix_punctuation(text):
24
+ # Remove spaces before punctuation and ensure single space after commas/periods
25
+ text = re.sub(r'\s+([,.!?;:])', r'\1', text) # no space before punctuation
26
+ text = re.sub(r'([,.!?;:])([^\s])', r'\1 \2', text) # ensure space after punctuation
27
+ text = re.sub(r'\s+', ' ', text) # normalize multiple spaces
28
+ return text.strip()
29
+
30
+ def chunk_text(text, max_sentences=4):
31
  sentences = re.split(r'(?<=[.!?]) +', text.strip())
32
  return [' '.join(sentences[i:i+max_sentences]) for i in range(0, len(sentences), max_sentences)]
33
 
34
  def full_article_paraphrase(text):
35
  chunks = chunk_text(text)
36
+ return "\n\n".join(paraphrase_text(chunk.strip()) for chunk in chunks if chunk.strip())
 
 
 
 
37
 
38
  def extract_text_from_docx(file_obj):
39
  from docx import Document