File size: 6,824 Bytes
9ba5a9c
 
 
 
 
 
d808060
9ba5a9c
 
 
d808060
9ba5a9c
 
 
 
 
 
 
 
 
d808060
9ba5a9c
d808060
 
 
 
 
 
 
 
 
 
9ba5a9c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
# config.py

########################################
# 1. Corpus / Data configuration
########################################

# Gutenberg classics — always available, broad vocabulary, zero auth required
CORPUS_URLS = [
    # Great Expectations (Dickens) — rich varied prose
    "https://www.gutenberg.org/cache/epub/1400/pg1400.txt",
    # Pride and Prejudice (Austen) — clean conversational English
    "https://www.gutenberg.org/cache/epub/1342/pg1342.txt",
    # The Adventures of Sherlock Holmes (Doyle) — dialog + narrative
    "https://www.gutenberg.org/cache/epub/1661/pg1661.txt",
    # Moby Dick (Melville) — large vocabulary, long sentences
    "https://www.gutenberg.org/cache/epub/2701/pg2701.txt",
    # A Tale of Two Cities (Dickens) — narrative diversity
    "https://www.gutenberg.org/cache/epub/98/pg98.txt",
    # The Picture of Dorian Gray (Wilde) — witty, conversational
    "https://www.gutenberg.org/cache/epub/174/pg174.txt",
    # Frankenstein (Shelley) — letter/diary/first-person style
    "https://www.gutenberg.org/cache/epub/84/pg84.txt",
    # Dracula (Stoker) — epistolary/journal style, mirrors Reddit posts
    "https://www.gutenberg.org/cache/epub/345/pg345.txt",
    # Adventures of Huckleberry Finn (Twain) — informal American English
    "https://www.gutenberg.org/cache/epub/76/pg76.txt",
    # The Adventures of Tom Sawyer (Twain) — informal dialog
    "https://www.gutenberg.org/cache/epub/74/pg74.txt",
    # Grimms Fairy Tales — varied short narratives, simple sentences
    "https://www.gutenberg.org/cache/epub/2591/pg2591.txt",
    # The Yellow Wallpaper (Gilman) — first-person intimate prose
    "https://www.gutenberg.org/cache/epub/1952/pg1952.txt",
]

# Modern corpus (loaded via HuggingFace datasets if available):
#   - Wikipedia 2022 English snapshot (~6M articles, factual modern English)
#   - AG News (120k news articles, contemporary journalism)
# These augment the Gutenberg vocab with internet-era words, slang context,
# abbreviations, and modern proper nouns.
# Set to False to skip (saves startup time if datasets not installed)
USE_MODERN_CORPUS = True

NGRAM_MIN = 2
NGRAM_MAX = 4


########################################
# 2. Model configuration
########################################

# Grammar correction: tried in order, first working model is used.
#
# Why this order:
#   1. vennify/t5-base-grammar-correction — trained on C4 200M (modern web text).
#      BEST for informal/social-media text. Conservative: won't "fix" slang.
#   2. prithivida/grammar_error_correcter_v1 — trained on GEC benchmarks.
#      Good accuracy but known to over-correct pronouns (I→In bug).
#      Kept as fallback since it's usually already cached.
#   3. Rules-only — if both fail, regex + spelling still runs.
GRAMMAR_MODEL_NAMES = [
    "vennify/t5-base-grammar-correction",    # Best for modern informal text
    "prithivida/grammar_error_correcter_v1", # Fallback (may over-correct)
]

# Prefix for the active grammar model
# vennify uses "grammar: " (with space), prithivida uses "gec:"
# The pipeline will auto-detect based on which model loaded.
GRAMMAR_PREFIX = "grammar: "    # for vennify (primary)
GRAMMAR_PREFIX_FALLBACK = "gec:" # for prithivida

# Paraphrase: T5 fine-tuned on PAWS paraphrase pairs
PARAPHRASE_MODEL_NAME = "Vamsi/T5_Paraphrase_Paws"
PARAPHRASE_PREFIX = "paraphrase:"

# Shared generation settings
# T5_MAX_INPUT_LENGTH: per-sentence chunking means each chunk is ~10-30 words
# = ~15-45 tokens. 256 is very safe.
T5_MAX_INPUT_LENGTH  = 256
T5_MAX_OUTPUT_LENGTH = 256
T5_NUM_BEAMS         = 4


########################################
# 3. Word limit
########################################

# 500 words = ~28 sentences = ~20-40s on CPU (acceptable)
# 250 words = ~14 sentences = ~10-20s (faster but cuts off Reddit-length posts)
MAX_WORDS = 500


########################################
# 4. Spelling rules
########################################

CUSTOM_SPELLING_RULES = {
    "tha": "the",
    "adn": "and",
    "teh": "the",
    "recieve": "receive",
    "occured": "occurred",
    "wierd": "weird",
    "completly": "completely",
    "definately": "definitely",
    "seperated": "separated",
    "wich": "which",
    "acommodate": "accommodate",
    "accross": "across",
    "alot": "a lot",
    "alright": "all right",
    "alwasy": "always",
    "arguement": "argument",
    "beggining": "beginning",
    "buisness": "business",
    "calender": "calendar",
    "comfirm": "confirm",
    "comming": "coming",
    "dissapoint": "disappoint",
    "enviroment": "environment",
    "existance": "existence",
    "febuary": "february",
    "humerous": "humorous",
    "independant": "independent",
    "intresting": "interesting",
    "knownledge": "knowledge",
    "maintainance": "maintenance",
    "mispell": "misspell",
    "neccessary": "necessary",
    "peice": "piece",
    "privelege": "privilege",
    "recomend": "recommend",
    "succesful": "successful",
    "tommorow": "tomorrow",
    "unecessary": "unnecessary",
    "untill": "until",
    "vacumn": "vacuum",
    "occurence": "occurrence",
    "appologies": "apologies",
    "possesion": "possession",
    "preceed": "precede",
    "reciept": "receipt",
    "beleive": "believe",
    "definetely": "definitely",
    # Internet / informal
    "ngl": "NGL",
    "irl": "IRL",
    "lol": "LOL",
    "tbh": "TBH",
    "idk": "IDK",
    "imo": "IMO",
    "imho": "IMHO",
    "btw": "BTW",
    "omg": "OMG",
}


########################################
# 5. Grammar rules (regex-based pre-pass)
########################################

CUSTOM_GRAMMAR_RULES = [
    (r"\b[Ii] can able to\b", "I am able to"),
    (r"\b[Ss]he don\'t\b", "She doesn't"),
    (r"\b[Hh]e don\'t\b", "He doesn't"),
    (r"\b[Hh]e can plays\b", "He can play"),
    (r"\b[Mm]e and (\w+)\b", r"\1 and I"),
    (r"\b[Tt]hats\b", "That's"),
    (r"\b[Ww]ould of\b", "would have"),
    (r"\b[Cc]ould of\b", "could have"),
    (r"\b[Ss]hould of\b", "should have"),
    (r"\b[Yy]ou was\b", "You were"),
    (r"\b[Tt]hey is\b", "They are"),
    (r"\b[Tt]his are\b", "These are"),
    (r"\b[Tt]hat are\b", "Those are"),
    (r"\b[Aa]llright\b", "all right"),
    # Tense consistency helpers
    (r"\bhe give\b", "he gives"),
    (r"\bshe give\b", "she gives"),
    (r"\bit give\b", "it gives"),
]


########################################
# 6. Analysis / AI-likeness thresholds
########################################

AI_MAX_LENGTH = 512
AI_LOW_PERPLEXITY_THRESHOLD = 15.0
AI_LOW_BURSTINESS_THRESHOLD = 0.15
AI_HIGH_BURSTINESS_THRESHOLD = 1.5


########################################
# 7. UI options
########################################

MODES = ["Grammar", "Paraphrase", "Both"]
PARAPHRASE_STYLES = ["Neutral", "Formal", "Casual", "Student essay", "Technical"]