Writing_Assistant / config.py
hephaestusonfire's picture
Update config.py
d808060 verified
Raw History Blame Contribute Delete
6.82 kB
# config.py
########################################
# 1. Corpus / Data configuration
########################################
# Gutenberg classics — always available, broad vocabulary, zero auth required
CORPUS_URLS = [
# Great Expectations (Dickens) — rich varied prose
"https://www.gutenberg.org/cache/epub/1400/pg1400.txt",
# Pride and Prejudice (Austen) — clean conversational English
"https://www.gutenberg.org/cache/epub/1342/pg1342.txt",
# The Adventures of Sherlock Holmes (Doyle) — dialog + narrative
"https://www.gutenberg.org/cache/epub/1661/pg1661.txt",
# Moby Dick (Melville) — large vocabulary, long sentences
"https://www.gutenberg.org/cache/epub/2701/pg2701.txt",
# A Tale of Two Cities (Dickens) — narrative diversity
"https://www.gutenberg.org/cache/epub/98/pg98.txt",
# The Picture of Dorian Gray (Wilde) — witty, conversational
"https://www.gutenberg.org/cache/epub/174/pg174.txt",
# Frankenstein (Shelley) — letter/diary/first-person style
"https://www.gutenberg.org/cache/epub/84/pg84.txt",
# Dracula (Stoker) — epistolary/journal style, mirrors Reddit posts
"https://www.gutenberg.org/cache/epub/345/pg345.txt",
# Adventures of Huckleberry Finn (Twain) — informal American English
"https://www.gutenberg.org/cache/epub/76/pg76.txt",
# The Adventures of Tom Sawyer (Twain) — informal dialog
"https://www.gutenberg.org/cache/epub/74/pg74.txt",
# Grimms Fairy Tales — varied short narratives, simple sentences
"https://www.gutenberg.org/cache/epub/2591/pg2591.txt",
# The Yellow Wallpaper (Gilman) — first-person intimate prose
"https://www.gutenberg.org/cache/epub/1952/pg1952.txt",
]
# Modern corpus (loaded via HuggingFace datasets if available):
# - Wikipedia 2022 English snapshot (~6M articles, factual modern English)
# - AG News (120k news articles, contemporary journalism)
# These augment the Gutenberg vocab with internet-era words, slang context,
# abbreviations, and modern proper nouns.
# Set to False to skip (saves startup time if datasets not installed)
USE_MODERN_CORPUS = True
NGRAM_MIN = 2
NGRAM_MAX = 4
########################################
# 2. Model configuration
########################################
# Grammar correction: tried in order, first working model is used.
#
# Why this order:
# 1. vennify/t5-base-grammar-correction — trained on C4 200M (modern web text).
# BEST for informal/social-media text. Conservative: won't "fix" slang.
# 2. prithivida/grammar_error_correcter_v1 — trained on GEC benchmarks.
# Good accuracy but known to over-correct pronouns (I→In bug).
# Kept as fallback since it's usually already cached.
# 3. Rules-only — if both fail, regex + spelling still runs.
GRAMMAR_MODEL_NAMES = [
"vennify/t5-base-grammar-correction", # Best for modern informal text
"prithivida/grammar_error_correcter_v1", # Fallback (may over-correct)
]
# Prefix for the active grammar model
# vennify uses "grammar: " (with space), prithivida uses "gec:"
# The pipeline will auto-detect based on which model loaded.
GRAMMAR_PREFIX = "grammar: " # for vennify (primary)
GRAMMAR_PREFIX_FALLBACK = "gec:" # for prithivida
# Paraphrase: T5 fine-tuned on PAWS paraphrase pairs
PARAPHRASE_MODEL_NAME = "Vamsi/T5_Paraphrase_Paws"
PARAPHRASE_PREFIX = "paraphrase:"
# Shared generation settings
# T5_MAX_INPUT_LENGTH: per-sentence chunking means each chunk is ~10-30 words
# = ~15-45 tokens. 256 is very safe.
T5_MAX_INPUT_LENGTH = 256
T5_MAX_OUTPUT_LENGTH = 256
T5_NUM_BEAMS = 4
########################################
# 3. Word limit
########################################
# 500 words = ~28 sentences = ~20-40s on CPU (acceptable)
# 250 words = ~14 sentences = ~10-20s (faster but cuts off Reddit-length posts)
MAX_WORDS = 500
########################################
# 4. Spelling rules
########################################
CUSTOM_SPELLING_RULES = {
"tha": "the",
"adn": "and",
"teh": "the",
"recieve": "receive",
"occured": "occurred",
"wierd": "weird",
"completly": "completely",
"definately": "definitely",
"seperated": "separated",
"wich": "which",
"acommodate": "accommodate",
"accross": "across",
"alot": "a lot",
"alright": "all right",
"alwasy": "always",
"arguement": "argument",
"beggining": "beginning",
"buisness": "business",
"calender": "calendar",
"comfirm": "confirm",
"comming": "coming",
"dissapoint": "disappoint",
"enviroment": "environment",
"existance": "existence",
"febuary": "february",
"humerous": "humorous",
"independant": "independent",
"intresting": "interesting",
"knownledge": "knowledge",
"maintainance": "maintenance",
"mispell": "misspell",
"neccessary": "necessary",
"peice": "piece",
"privelege": "privilege",
"recomend": "recommend",
"succesful": "successful",
"tommorow": "tomorrow",
"unecessary": "unnecessary",
"untill": "until",
"vacumn": "vacuum",
"occurence": "occurrence",
"appologies": "apologies",
"possesion": "possession",
"preceed": "precede",
"reciept": "receipt",
"beleive": "believe",
"definetely": "definitely",
# Internet / informal
"ngl": "NGL",
"irl": "IRL",
"lol": "LOL",
"tbh": "TBH",
"idk": "IDK",
"imo": "IMO",
"imho": "IMHO",
"btw": "BTW",
"omg": "OMG",
}
########################################
# 5. Grammar rules (regex-based pre-pass)
########################################
CUSTOM_GRAMMAR_RULES = [
(r"\b[Ii] can able to\b", "I am able to"),
(r"\b[Ss]he don\'t\b", "She doesn't"),
(r"\b[Hh]e don\'t\b", "He doesn't"),
(r"\b[Hh]e can plays\b", "He can play"),
(r"\b[Mm]e and (\w+)\b", r"\1 and I"),
(r"\b[Tt]hats\b", "That's"),
(r"\b[Ww]ould of\b", "would have"),
(r"\b[Cc]ould of\b", "could have"),
(r"\b[Ss]hould of\b", "should have"),
(r"\b[Yy]ou was\b", "You were"),
(r"\b[Tt]hey is\b", "They are"),
(r"\b[Tt]his are\b", "These are"),
(r"\b[Tt]hat are\b", "Those are"),
(r"\b[Aa]llright\b", "all right"),
# Tense consistency helpers
(r"\bhe give\b", "he gives"),
(r"\bshe give\b", "she gives"),
(r"\bit give\b", "it gives"),
]
########################################
# 6. Analysis / AI-likeness thresholds
########################################
AI_MAX_LENGTH = 512
AI_LOW_PERPLEXITY_THRESHOLD = 15.0
AI_LOW_BURSTINESS_THRESHOLD = 0.15
AI_HIGH_BURSTINESS_THRESHOLD = 1.5
########################################
# 7. UI options
########################################
MODES = ["Grammar", "Paraphrase", "Both"]
PARAPHRASE_STYLES = ["Neutral", "Formal", "Casual", "Student essay", "Technical"]