# config.py ######################################## # 1. Corpus / Data configuration ######################################## # Gutenberg classics — always available, broad vocabulary, zero auth required CORPUS_URLS = [ # Great Expectations (Dickens) — rich varied prose "https://www.gutenberg.org/cache/epub/1400/pg1400.txt", # Pride and Prejudice (Austen) — clean conversational English "https://www.gutenberg.org/cache/epub/1342/pg1342.txt", # The Adventures of Sherlock Holmes (Doyle) — dialog + narrative "https://www.gutenberg.org/cache/epub/1661/pg1661.txt", # Moby Dick (Melville) — large vocabulary, long sentences "https://www.gutenberg.org/cache/epub/2701/pg2701.txt", # A Tale of Two Cities (Dickens) — narrative diversity "https://www.gutenberg.org/cache/epub/98/pg98.txt", # The Picture of Dorian Gray (Wilde) — witty, conversational "https://www.gutenberg.org/cache/epub/174/pg174.txt", # Frankenstein (Shelley) — letter/diary/first-person style "https://www.gutenberg.org/cache/epub/84/pg84.txt", # Dracula (Stoker) — epistolary/journal style, mirrors Reddit posts "https://www.gutenberg.org/cache/epub/345/pg345.txt", # Adventures of Huckleberry Finn (Twain) — informal American English "https://www.gutenberg.org/cache/epub/76/pg76.txt", # The Adventures of Tom Sawyer (Twain) — informal dialog "https://www.gutenberg.org/cache/epub/74/pg74.txt", # Grimms Fairy Tales — varied short narratives, simple sentences "https://www.gutenberg.org/cache/epub/2591/pg2591.txt", # The Yellow Wallpaper (Gilman) — first-person intimate prose "https://www.gutenberg.org/cache/epub/1952/pg1952.txt", ] # Modern corpus (loaded via HuggingFace datasets if available): # - Wikipedia 2022 English snapshot (~6M articles, factual modern English) # - AG News (120k news articles, contemporary journalism) # These augment the Gutenberg vocab with internet-era words, slang context, # abbreviations, and modern proper nouns. # Set to False to skip (saves startup time if datasets not installed) USE_MODERN_CORPUS = True NGRAM_MIN = 2 NGRAM_MAX = 4 ######################################## # 2. Model configuration ######################################## # Grammar correction: tried in order, first working model is used. # # Why this order: # 1. vennify/t5-base-grammar-correction — trained on C4 200M (modern web text). # BEST for informal/social-media text. Conservative: won't "fix" slang. # 2. prithivida/grammar_error_correcter_v1 — trained on GEC benchmarks. # Good accuracy but known to over-correct pronouns (I→In bug). # Kept as fallback since it's usually already cached. # 3. Rules-only — if both fail, regex + spelling still runs. GRAMMAR_MODEL_NAMES = [ "vennify/t5-base-grammar-correction", # Best for modern informal text "prithivida/grammar_error_correcter_v1", # Fallback (may over-correct) ] # Prefix for the active grammar model # vennify uses "grammar: " (with space), prithivida uses "gec:" # The pipeline will auto-detect based on which model loaded. GRAMMAR_PREFIX = "grammar: " # for vennify (primary) GRAMMAR_PREFIX_FALLBACK = "gec:" # for prithivida # Paraphrase: T5 fine-tuned on PAWS paraphrase pairs PARAPHRASE_MODEL_NAME = "Vamsi/T5_Paraphrase_Paws" PARAPHRASE_PREFIX = "paraphrase:" # Shared generation settings # T5_MAX_INPUT_LENGTH: per-sentence chunking means each chunk is ~10-30 words # = ~15-45 tokens. 256 is very safe. T5_MAX_INPUT_LENGTH = 256 T5_MAX_OUTPUT_LENGTH = 256 T5_NUM_BEAMS = 4 ######################################## # 3. Word limit ######################################## # 500 words = ~28 sentences = ~20-40s on CPU (acceptable) # 250 words = ~14 sentences = ~10-20s (faster but cuts off Reddit-length posts) MAX_WORDS = 500 ######################################## # 4. Spelling rules ######################################## CUSTOM_SPELLING_RULES = { "tha": "the", "adn": "and", "teh": "the", "recieve": "receive", "occured": "occurred", "wierd": "weird", "completly": "completely", "definately": "definitely", "seperated": "separated", "wich": "which", "acommodate": "accommodate", "accross": "across", "alot": "a lot", "alright": "all right", "alwasy": "always", "arguement": "argument", "beggining": "beginning", "buisness": "business", "calender": "calendar", "comfirm": "confirm", "comming": "coming", "dissapoint": "disappoint", "enviroment": "environment", "existance": "existence", "febuary": "february", "humerous": "humorous", "independant": "independent", "intresting": "interesting", "knownledge": "knowledge", "maintainance": "maintenance", "mispell": "misspell", "neccessary": "necessary", "peice": "piece", "privelege": "privilege", "recomend": "recommend", "succesful": "successful", "tommorow": "tomorrow", "unecessary": "unnecessary", "untill": "until", "vacumn": "vacuum", "occurence": "occurrence", "appologies": "apologies", "possesion": "possession", "preceed": "precede", "reciept": "receipt", "beleive": "believe", "definetely": "definitely", # Internet / informal "ngl": "NGL", "irl": "IRL", "lol": "LOL", "tbh": "TBH", "idk": "IDK", "imo": "IMO", "imho": "IMHO", "btw": "BTW", "omg": "OMG", } ######################################## # 5. Grammar rules (regex-based pre-pass) ######################################## CUSTOM_GRAMMAR_RULES = [ (r"\b[Ii] can able to\b", "I am able to"), (r"\b[Ss]he don\'t\b", "She doesn't"), (r"\b[Hh]e don\'t\b", "He doesn't"), (r"\b[Hh]e can plays\b", "He can play"), (r"\b[Mm]e and (\w+)\b", r"\1 and I"), (r"\b[Tt]hats\b", "That's"), (r"\b[Ww]ould of\b", "would have"), (r"\b[Cc]ould of\b", "could have"), (r"\b[Ss]hould of\b", "should have"), (r"\b[Yy]ou was\b", "You were"), (r"\b[Tt]hey is\b", "They are"), (r"\b[Tt]his are\b", "These are"), (r"\b[Tt]hat are\b", "Those are"), (r"\b[Aa]llright\b", "all right"), # Tense consistency helpers (r"\bhe give\b", "he gives"), (r"\bshe give\b", "she gives"), (r"\bit give\b", "it gives"), ] ######################################## # 6. Analysis / AI-likeness thresholds ######################################## AI_MAX_LENGTH = 512 AI_LOW_PERPLEXITY_THRESHOLD = 15.0 AI_LOW_BURSTINESS_THRESHOLD = 0.15 AI_HIGH_BURSTINESS_THRESHOLD = 1.5 ######################################## # 7. UI options ######################################## MODES = ["Grammar", "Paraphrase", "Both"] PARAPHRASE_STYLES = ["Neutral", "Formal", "Casual", "Student essay", "Technical"]