EdysorEdutech commited on
Commit
ed55a98
·
verified ·
1 Parent(s): d0a1352

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +74 -121
app.py CHANGED
@@ -352,12 +352,6 @@ class HumanLikeVariations:
352
  return variation(sentence)
353
  except:
354
  return sentence
355
-
356
- def split_into_sentences_advanced(self, text):
357
- """Advanced sentence splitting using spaCy or NLTK"""
358
- # This method is referenced but not defined in the class
359
- # Using basic splitting for now
360
- return text.split('. ')
361
 
362
  class SelectiveGrammarFixer:
363
  """Minimal grammar fixes to maintain human-like quality while fixing critical errors"""
@@ -661,9 +655,8 @@ class EnhancedDipperHumanizer:
661
  sorted_keywords = sorted(keywords, key=len, reverse=True)
662
 
663
  for i, keyword in enumerate(sorted_keywords):
664
- # Use a more unique and robust placeholder that won't get corrupted
665
- # Using double underscores and a unique pattern
666
- placeholder = f"__KEYWORD{i:03d}KEYWORD__"
667
 
668
  # Find all occurrences of the keyword (case-insensitive)
669
  pattern = r'\b' + re.escape(keyword) + r'\b'
@@ -679,7 +672,7 @@ class EnhancedDipperHumanizer:
679
  keyword_map[placeholder] = original_keyword
680
 
681
  return modified_text, keyword_map
682
-
683
  def restore_keywords_robust(self, text, keyword_map):
684
  """Restore keywords with more flexible pattern matching"""
685
  if not keyword_map:
@@ -691,134 +684,94 @@ class EnhancedDipperHumanizer:
691
  print(f"Restoring keywords in text: {restored_text[:100]}...")
692
  print(f"Keyword map: {keyword_map}")
693
 
694
- # Create a reverse map for easier lookup
695
- keyword_values = list(keyword_map.values())
696
 
697
  # First pass: Direct placeholder replacement
698
  for placeholder, keyword in keyword_map.items():
699
  if placeholder in restored_text:
700
  print(f"Found exact placeholder {placeholder}, replacing with {keyword}")
701
  restored_text = restored_text.replace(placeholder, keyword)
 
 
 
702
 
703
- # Second pass: Handle corrupted placeholders
704
- # The model might mangle KEYWORD003KEYWORD into various forms
705
  for placeholder, keyword in keyword_map.items():
706
  # Extract the number from placeholder
707
- match = re.search(r'__KEYWORD(\d+)KEYWORD__', placeholder)
708
  if match:
709
  num = match.group(1)
710
 
711
- # All possible corruptions of the placeholder
712
  patterns = [
713
- # Original patterns
714
- f'__KEYWORD{num}KEYWORD__',
715
- f'__KEYWORD{num}KEYWORD_',
716
- f'_KEYWORD{num}KEYWORD__',
717
- f'_KEYWORD{num}KEYWORD_',
718
- f'__KEYWORD{num}KEYWORD',
719
- f'KEYWORD{num}KEYWORD',
720
- f'__KEYWORD{num}__',
721
- f'__KW{num}__',
722
- f'_KW{num}_',
723
- f'KW{num}',
724
- f'K{num}',
725
- f'_K{num}',
726
- f'__K{num}__',
727
- f'__K{num}_',
728
- f'_K{num}_',
729
- # Lowercase variations
730
- f'__keyword{num}keyword__',
731
- f'__keyword{num}__',
732
- f'__kw{num}__',
733
- f'_kw{num}_',
734
- f'kw{num}',
735
- f'k{num}',
736
- f'_k{num}',
737
- # Mixed case
738
- f'__Keyword{num}Keyword__',
739
- f'__Kw{num}__',
740
- f'_Kw{num}_',
741
- f'Kw{num}',
742
- # With spaces
743
- f'__ KEYWORD{num}KEYWORD __',
744
- f'__ KEYWORD {num} KEYWORD __',
745
- f'__ KW{num} __',
746
- f'__ K{num} __',
747
- # Partial corruptions
748
- f'KEYWORD{num}',
749
- f'keyword{num}',
750
- f'__kword{num}',
751
- f'_word{num}',
752
- # Common OCR-like errors
753
- f'KEYWoRD{num}',
754
- f'KEYW0RD{num}', # 0 instead of O
755
- f'KEYVVORD{num}', # VV instead of W
756
- # Just the number with various prefixes
757
- f'___K{num}',
758
- f'__K{num}',
759
- f'_K{num}',
760
- f'K{num}',
761
- f'k{num}',
762
- # Backtick corruptions
763
- f'``KEYWORD{num}',
764
- f'``K{num}',
765
- # Double character corruptions
766
- f'KKEYWORD{num}',
767
- f'KKW{num}',
768
- f'KK{num}',
769
- # Underscore variations
770
- f'_{num}',
771
- f'__{num}',
772
- f'___{num}',
773
- f'____{num}',
774
  ]
775
 
776
- for pattern in patterns:
777
  if pattern in restored_text:
778
- print(f"Found pattern '{pattern}', replacing with {keyword}")
779
- # Use word boundaries where appropriate
780
- if pattern.startswith('_') or pattern.startswith('K'):
781
- # For patterns that might appear within words, be more careful
782
- restored_text = re.sub(r'\b' + re.escape(pattern) + r'\b', keyword, restored_text)
783
- else:
784
- restored_text = restored_text.replace(pattern, keyword)
785
- break # Move to next placeholder after successful replacement
786
-
787
- # Third pass: Look for any remaining number patterns that might be keywords
788
- # This catches cases where only the number remains
789
- remaining_patterns = re.findall(r'\b[kK]?\d{3}\b|\b_[kK]?\d{1,3}\b|_[kK]\s|[kK]\d+', restored_text)
790
- if remaining_patterns:
791
- print(f"Found remaining patterns that might be corrupted keywords: {remaining_patterns}")
792
- for pattern in remaining_patterns:
793
- # Extract any number from the pattern
794
- num_match = re.search(r'(\d+)', pattern)
795
- if num_match:
796
- num_str = num_match.group(1)
797
- try:
798
- num = int(num_str)
799
- # Check if this number corresponds to a keyword
800
- if num < len(keyword_values):
801
- keyword = keyword_values[num]
802
- print(f"Replacing pattern '{pattern}' with keyword '{keyword}'")
803
- restored_text = restored_text.replace(pattern, keyword, 1)
804
- except ValueError:
805
- pass
806
-
807
- # Fourth pass: Clean up any artifacts
808
- # Remove lone underscores that might remain
809
- restored_text = re.sub(r'\b_+\b', '', restored_text)
810
- # Remove multiple consecutive underscores
811
- restored_text = re.sub(r'_{3,}', '', restored_text)
812
- # Clean up spaces around removed underscores
813
- restored_text = re.sub(r'\s+', ' ', restored_text)
814
-
815
- # Final cleanup: Remove any backticks or quotes that shouldn't be there
816
  restored_text = re.sub(r'``+', '', restored_text)
 
817
  restored_text = re.sub(r"''", '"', restored_text)
818
-
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
819
  print(f"Final restored text: {restored_text[:100]}...")
820
 
821
- return restored_text.strip()
822
 
823
  def should_skip_element(self, element, text):
824
  """Determine if an element should be skipped from paraphrasing"""
@@ -1573,8 +1526,8 @@ class EnhancedDipperHumanizer:
1573
  for p_tag in soup.find_all('p'):
1574
  # Skip paragraphs that are inside special elements
1575
  # Check if paragraph is inside any of these elements
1576
- skip_parents = ['div.author-intro', 'div.testimonial-card',
1577
- 'button', 'a', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
1578
  'div.quiz-container', 'div.question-container', 'div.results']
1579
 
1580
  # Check if this paragraph should be skipped
@@ -1589,7 +1542,7 @@ class EnhancedDipperHumanizer:
1589
  class_str = str(classes)
1590
 
1591
  if any(skip_class in class_str for skip_class in
1592
- ['author-intro', 'testimonial-card',
1593
  'quiz-container', 'question-container', 'results', 'stats-grid',
1594
  'toc-', 'comparison-tables']):
1595
  should_skip = True
 
352
  return variation(sentence)
353
  except:
354
  return sentence
 
 
 
 
 
 
355
 
356
  class SelectiveGrammarFixer:
357
  """Minimal grammar fixes to maintain human-like quality while fixing critical errors"""
 
655
  sorted_keywords = sorted(keywords, key=len, reverse=True)
656
 
657
  for i, keyword in enumerate(sorted_keywords):
658
+ # Use unique markers that won't be confused
659
+ placeholder = f"__KW{i:03d}__" # e.g., __KW001__
 
660
 
661
  # Find all occurrences of the keyword (case-insensitive)
662
  pattern = r'\b' + re.escape(keyword) + r'\b'
 
672
  keyword_map[placeholder] = original_keyword
673
 
674
  return modified_text, keyword_map
675
+
676
  def restore_keywords_robust(self, text, keyword_map):
677
  """Restore keywords with more flexible pattern matching"""
678
  if not keyword_map:
 
684
  print(f"Restoring keywords in text: {restored_text[:100]}...")
685
  print(f"Keyword map: {keyword_map}")
686
 
687
+ # Track which positions have been replaced to avoid double replacement
688
+ replaced_positions = set()
689
 
690
  # First pass: Direct placeholder replacement
691
  for placeholder, keyword in keyword_map.items():
692
  if placeholder in restored_text:
693
  print(f"Found exact placeholder {placeholder}, replacing with {keyword}")
694
  restored_text = restored_text.replace(placeholder, keyword)
695
+ # Mark positions as replaced
696
+ for match in re.finditer(re.escape(keyword), restored_text):
697
+ replaced_positions.update(range(match.start(), match.end()))
698
 
699
+ # Second pass: Handle any mangled placeholders
700
+ # The model might alter placeholders in various ways
701
  for placeholder, keyword in keyword_map.items():
702
  # Extract the number from placeholder
703
+ match = re.search(r'__KW(\d+)__', placeholder)
704
  if match:
705
  num = match.group(1)
706
 
707
+ # Various patterns the model might create
708
  patterns = [
709
+ (f'__KW{num}__', keyword),
710
+ (f'__ KW{num}__', keyword),
711
+ (f'__KW {num}__', keyword),
712
+ (f'__ KW {num} __', keyword),
713
+ (f'_KW{num}_', keyword),
714
+ (f'_kw{num}_', keyword),
715
+ (f'KW{num}', keyword),
716
+ (f'KW {num}', keyword),
717
+ (f'__kw{num}__', keyword),
718
+ (f'__Kw{num}__', keyword),
719
+ (f'__ kw{num}__', keyword),
720
+ (f'__KW{num}_', keyword),
721
+ (f'_KW{num}__', keyword),
722
+ (f'kw{num}', keyword),
723
+ (f'``KW{num}__', keyword), # Handle backtick corruption
724
+ (f'``KKW{num}', keyword), # Handle double K corruption
725
+ (f'KW{num}', keyword), # Simple pattern
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
726
  ]
727
 
728
+ for pattern, replacement in patterns:
729
  if pattern in restored_text:
730
+ # Check if this position has already been replaced
731
+ start_pos = restored_text.find(pattern)
732
+ if start_pos != -1 and not any(pos in replaced_positions for pos in range(start_pos, start_pos + len(pattern))):
733
+ print(f"Found pattern '{pattern}', replacing with {replacement}")
734
+ restored_text = restored_text.replace(pattern, replacement, 1) # Replace only first occurrence
735
+ # Mark new positions as replaced
736
+ for match in re.finditer(re.escape(replacement), restored_text):
737
+ replaced_positions.update(range(match.start(), match.end()))
738
+ break # Move to next placeholder after successful replacement
739
+
740
+ # Third pass: Clean up any backticks or quotes that shouldn't be there
741
+ # Remove double backticks
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
742
  restored_text = re.sub(r'``+', '', restored_text)
743
+ # Fix double quotes
744
  restored_text = re.sub(r"''", '"', restored_text)
745
+ restored_text = re.sub(r'""', '"', restored_text)
746
+
747
+ # Fourth pass: Look for remaining underscore patterns
748
+ # But be more careful about replacement
749
+ if '___' in restored_text and keyword_map:
750
+ # Find all occurrences of multiple underscores
751
+ underscore_matches = list(re.finditer(r'_{3,}', restored_text))
752
+ keyword_values = list(keyword_map.values())
753
+
754
+ # Replace underscores with keywords, but only if not already replaced
755
+ for i, match in enumerate(underscore_matches):
756
+ if i < len(keyword_values):
757
+ start, end = match.span()
758
+ if not any(pos in replaced_positions for pos in range(start, end)):
759
+ before = restored_text[:start]
760
+ after = restored_text[end:]
761
+ restored_text = before + keyword_values[i] + after
762
+ # Update replaced positions
763
+ replaced_positions.update(range(start, start + len(keyword_values[i])))
764
+
765
+ # Final cleanup: Remove any remaining KW patterns that weren't caught
766
+ # But only if they're not part of an already replaced keyword
767
+ remaining_kw_patterns = re.findall(r'\bKW\d{3}\b', restored_text)
768
+ if remaining_kw_patterns:
769
+ print(f"Warning: Found remaining KW patterns: {remaining_kw_patterns}")
770
+
771
+ # Log final result
772
  print(f"Final restored text: {restored_text[:100]}...")
773
 
774
+ return restored_text
775
 
776
  def should_skip_element(self, element, text):
777
  """Determine if an element should be skipped from paraphrasing"""
 
1526
  for p_tag in soup.find_all('p'):
1527
  # Skip paragraphs that are inside special elements
1528
  # Check if paragraph is inside any of these elements
1529
+ skip_parents = ['div.author-intro', 'div.cta-box', 'div.testimonial-card',
1530
+ 'div.news-box', 'button', 'a', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6',
1531
  'div.quiz-container', 'div.question-container', 'div.results']
1532
 
1533
  # Check if this paragraph should be skipped
 
1542
  class_str = str(classes)
1543
 
1544
  if any(skip_class in class_str for skip_class in
1545
+ ['author-intro', 'cta-box', 'testimonial-card', 'news-box',
1546
  'quiz-container', 'question-container', 'results', 'stats-grid',
1547
  'toc-', 'comparison-tables']):
1548
  should_skip = True