File size: 6,824 Bytes
9ba5a9c d808060 9ba5a9c d808060 9ba5a9c d808060 9ba5a9c d808060 9ba5a9c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 | # config.py
########################################
# 1. Corpus / Data configuration
########################################
# Gutenberg classics — always available, broad vocabulary, zero auth required
CORPUS_URLS = [
# Great Expectations (Dickens) — rich varied prose
"https://www.gutenberg.org/cache/epub/1400/pg1400.txt",
# Pride and Prejudice (Austen) — clean conversational English
"https://www.gutenberg.org/cache/epub/1342/pg1342.txt",
# The Adventures of Sherlock Holmes (Doyle) — dialog + narrative
"https://www.gutenberg.org/cache/epub/1661/pg1661.txt",
# Moby Dick (Melville) — large vocabulary, long sentences
"https://www.gutenberg.org/cache/epub/2701/pg2701.txt",
# A Tale of Two Cities (Dickens) — narrative diversity
"https://www.gutenberg.org/cache/epub/98/pg98.txt",
# The Picture of Dorian Gray (Wilde) — witty, conversational
"https://www.gutenberg.org/cache/epub/174/pg174.txt",
# Frankenstein (Shelley) — letter/diary/first-person style
"https://www.gutenberg.org/cache/epub/84/pg84.txt",
# Dracula (Stoker) — epistolary/journal style, mirrors Reddit posts
"https://www.gutenberg.org/cache/epub/345/pg345.txt",
# Adventures of Huckleberry Finn (Twain) — informal American English
"https://www.gutenberg.org/cache/epub/76/pg76.txt",
# The Adventures of Tom Sawyer (Twain) — informal dialog
"https://www.gutenberg.org/cache/epub/74/pg74.txt",
# Grimms Fairy Tales — varied short narratives, simple sentences
"https://www.gutenberg.org/cache/epub/2591/pg2591.txt",
# The Yellow Wallpaper (Gilman) — first-person intimate prose
"https://www.gutenberg.org/cache/epub/1952/pg1952.txt",
]
# Modern corpus (loaded via HuggingFace datasets if available):
# - Wikipedia 2022 English snapshot (~6M articles, factual modern English)
# - AG News (120k news articles, contemporary journalism)
# These augment the Gutenberg vocab with internet-era words, slang context,
# abbreviations, and modern proper nouns.
# Set to False to skip (saves startup time if datasets not installed)
USE_MODERN_CORPUS = True
NGRAM_MIN = 2
NGRAM_MAX = 4
########################################
# 2. Model configuration
########################################
# Grammar correction: tried in order, first working model is used.
#
# Why this order:
# 1. vennify/t5-base-grammar-correction — trained on C4 200M (modern web text).
# BEST for informal/social-media text. Conservative: won't "fix" slang.
# 2. prithivida/grammar_error_correcter_v1 — trained on GEC benchmarks.
# Good accuracy but known to over-correct pronouns (I→In bug).
# Kept as fallback since it's usually already cached.
# 3. Rules-only — if both fail, regex + spelling still runs.
GRAMMAR_MODEL_NAMES = [
"vennify/t5-base-grammar-correction", # Best for modern informal text
"prithivida/grammar_error_correcter_v1", # Fallback (may over-correct)
]
# Prefix for the active grammar model
# vennify uses "grammar: " (with space), prithivida uses "gec:"
# The pipeline will auto-detect based on which model loaded.
GRAMMAR_PREFIX = "grammar: " # for vennify (primary)
GRAMMAR_PREFIX_FALLBACK = "gec:" # for prithivida
# Paraphrase: T5 fine-tuned on PAWS paraphrase pairs
PARAPHRASE_MODEL_NAME = "Vamsi/T5_Paraphrase_Paws"
PARAPHRASE_PREFIX = "paraphrase:"
# Shared generation settings
# T5_MAX_INPUT_LENGTH: per-sentence chunking means each chunk is ~10-30 words
# = ~15-45 tokens. 256 is very safe.
T5_MAX_INPUT_LENGTH = 256
T5_MAX_OUTPUT_LENGTH = 256
T5_NUM_BEAMS = 4
########################################
# 3. Word limit
########################################
# 500 words = ~28 sentences = ~20-40s on CPU (acceptable)
# 250 words = ~14 sentences = ~10-20s (faster but cuts off Reddit-length posts)
MAX_WORDS = 500
########################################
# 4. Spelling rules
########################################
CUSTOM_SPELLING_RULES = {
"tha": "the",
"adn": "and",
"teh": "the",
"recieve": "receive",
"occured": "occurred",
"wierd": "weird",
"completly": "completely",
"definately": "definitely",
"seperated": "separated",
"wich": "which",
"acommodate": "accommodate",
"accross": "across",
"alot": "a lot",
"alright": "all right",
"alwasy": "always",
"arguement": "argument",
"beggining": "beginning",
"buisness": "business",
"calender": "calendar",
"comfirm": "confirm",
"comming": "coming",
"dissapoint": "disappoint",
"enviroment": "environment",
"existance": "existence",
"febuary": "february",
"humerous": "humorous",
"independant": "independent",
"intresting": "interesting",
"knownledge": "knowledge",
"maintainance": "maintenance",
"mispell": "misspell",
"neccessary": "necessary",
"peice": "piece",
"privelege": "privilege",
"recomend": "recommend",
"succesful": "successful",
"tommorow": "tomorrow",
"unecessary": "unnecessary",
"untill": "until",
"vacumn": "vacuum",
"occurence": "occurrence",
"appologies": "apologies",
"possesion": "possession",
"preceed": "precede",
"reciept": "receipt",
"beleive": "believe",
"definetely": "definitely",
# Internet / informal
"ngl": "NGL",
"irl": "IRL",
"lol": "LOL",
"tbh": "TBH",
"idk": "IDK",
"imo": "IMO",
"imho": "IMHO",
"btw": "BTW",
"omg": "OMG",
}
########################################
# 5. Grammar rules (regex-based pre-pass)
########################################
CUSTOM_GRAMMAR_RULES = [
(r"\b[Ii] can able to\b", "I am able to"),
(r"\b[Ss]he don\'t\b", "She doesn't"),
(r"\b[Hh]e don\'t\b", "He doesn't"),
(r"\b[Hh]e can plays\b", "He can play"),
(r"\b[Mm]e and (\w+)\b", r"\1 and I"),
(r"\b[Tt]hats\b", "That's"),
(r"\b[Ww]ould of\b", "would have"),
(r"\b[Cc]ould of\b", "could have"),
(r"\b[Ss]hould of\b", "should have"),
(r"\b[Yy]ou was\b", "You were"),
(r"\b[Tt]hey is\b", "They are"),
(r"\b[Tt]his are\b", "These are"),
(r"\b[Tt]hat are\b", "Those are"),
(r"\b[Aa]llright\b", "all right"),
# Tense consistency helpers
(r"\bhe give\b", "he gives"),
(r"\bshe give\b", "she gives"),
(r"\bit give\b", "it gives"),
]
########################################
# 6. Analysis / AI-likeness thresholds
########################################
AI_MAX_LENGTH = 512
AI_LOW_PERPLEXITY_THRESHOLD = 15.0
AI_LOW_BURSTINESS_THRESHOLD = 0.15
AI_HIGH_BURSTINESS_THRESHOLD = 1.5
########################################
# 7. UI options
########################################
MODES = ["Grammar", "Paraphrase", "Both"]
PARAPHRASE_STYLES = ["Neutral", "Formal", "Casual", "Student essay", "Technical"] |