code/src/normalize.py

"""Quick cleanup for meme-y dialectal tweets: split hashtags back into words,
squash stretched-out letters, strip URLs. Keeps emojis and dialect intact --
just surfaces the words buried inside campaign hashtags. Used on both the
retrieval pool and the incoming tweets.
"""
import re

_URL = re.compile(r"https?://\S+")
_HASH = re.compile(r"#(\w+)")
_ELONG = re.compile(r"(.)\1{2,}")
_WS = re.compile(r"\s+")


def normalize_text(text):
    t = str(text)
    t = _URL.sub(" ", t)
    # segment hashtags: #a_b_c -> a b c
    t = _HASH.sub(lambda m: " " + m.group(1).replace("_", " ") + " ", t)
    t = t.replace("_", " ")
    # collapse 3+ repeats of any character to two (keeps some emphasis)
    t = _ELONG.sub(r"\1\1", t)
    return _WS.sub(" ", t).strip()


def main():
    import argparse
    import pandas as pd
    ap = argparse.ArgumentParser()
    ap.add_argument("--in", dest="inp", required=True)
    ap.add_argument("--out", required=True)
    ap.add_argument("--cols", default="text,tweet_text",
                    help="comma-separated text columns to normalise")
    args = ap.parse_args()
    df = pd.read_csv(args.inp, keep_default_na=False, encoding="utf-8-sig")
    df.columns = [c.strip() for c in df.columns]
    for c in args.cols.split(","):
        if c in df.columns:
            df[c] = df[c].map(normalize_text)
    df.to_csv(args.out, index=False, encoding="utf-8-sig")
    print(f"[normalize] {len(df)} rows -> {args.out}")


if __name__ == "__main__":
    main()