Download features.py from dmth10/tell: direct link, hf CLI and curl.
- Browser
- Download file 19.2 kB
-
https://huggingface.co/spaces/dmth10/tell/resolve/main/features.py
- Command line
-
hf download hf://spaces/dmth10/tell/features.py
-
curl -L -o features.py https://huggingface.co/spaces/dmth10/tell/resolve/main/features.py
19.2 kB
| """ | |
| The list of everything TELL can do. Order here is the order results show up in. | |
| Adding a feature: | |
| @register | |
| class MyCheck(Feature): | |
| name = "MyCheck" | |
| description = "what it looks for" | |
| kinds = {"photo"} # and/or needs_text = True, or override wants() | |
| def run(self, post, options): | |
| return {"findings": [...], "notes": [...], "confidence": "..."} | |
| Each finding is {"trick": <key in lessons.json>, "quote": <optional>, "why": <one sentence>}. | |
| Set ready = False to list it as coming soon before it works. | |
| """ | |
| import math | |
| import os | |
| import re | |
| from html import escape | |
| from core.features import Feature, register | |
| from tools import chartcheck, langcheck, screenshotcheck, statcheck | |
| HEADERS = {"User-Agent": "TELL media literacy app (Congressional App Challenge student project)"} | |
| class ChartCheck(Feature): | |
| name = "ChartCheck" | |
| description = "misleading axes and chart design" | |
| kinds = {"chart"} | |
| def run(self, post, options): | |
| return chartcheck.analyze(post.image) | |
| class ScreenshotCheck(Feature): | |
| name = "ScreenshotCheck" | |
| description = "fake or edited screenshots" | |
| kinds = {"screenshot"} | |
| def run(self, post, options): | |
| return screenshotcheck.analyze(post.image) | |
| # ---------- quick fact check ---------- | |
| STOP = set("the a an and or but of to in on for with is are was were be been this that it its at by from as " | |
| "has have had will would not no he she they we you i his her their our your who what".split()) | |
| def _words(text): | |
| return {w for w in re.findall(r"[a-z0-9']+", text.lower()) if len(w) > 2 and w not in STOP} | |
| def claim_overlap(post_text, claim_text): | |
| """Share of the fact-checked claim's words that also appear in the post (0 to 1).""" | |
| claim = _words(claim_text) | |
| return len(claim & _words(post_text)) / len(claim) if claim else 0.0 | |
| def rating_to_verdict(rating): | |
| r = rating.lower() | |
| if any(k in r for k in ("missing context", "misleading", "half", "mixture", "partly", "out of context", "exaggerat")): | |
| return "MISSING_CONTEXT" | |
| if any(k in r for k in ("false", "pants on fire", "fake", "incorrect", "fabricated", "not true", "no evidence", | |
| "baseless", "hoax", "scam", "wrong")): | |
| return "CONTRADICTED" | |
| if any(k in r for k in ("true", "correct", "accurate")): | |
| return "SUPPORTED" | |
| return "RATED" | |
| def _queries(text, n=2): | |
| # the longest sentences usually hold the actual claim | |
| sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text) if len(s.strip()) > 25] | |
| return sorted(sentences, key=len, reverse=True)[:n] or [text[:200]] | |
| class FactCheck(Feature): | |
| name = "FactCheck" | |
| description = "published fact-checks of the same claim (Google Fact Check Tools)" | |
| needs_text = True | |
| MATCH = 0.5 # at least half the fact-checked claim's words must be in the post | |
| def wants(self, post): | |
| return super().wants(post) and bool(os.environ.get("GOOGLE_FACTCHECK_KEY")) | |
| def run(self, post, options): | |
| import requests | |
| seen, out = set(), [] | |
| for q in _queries(post.all_text): | |
| r = requests.get("https://factchecktools.googleapis.com/v1alpha1/claims:search", | |
| params={"query": q[:200], "key": os.environ["GOOGLE_FACTCHECK_KEY"], | |
| "languageCode": "en", "pageSize": 5}, timeout=8) | |
| for c in r.json().get("claims", []): | |
| for rev in c.get("claimReview", [])[:1]: | |
| url = rev.get("url", "") | |
| if not url or url in seen: | |
| continue | |
| seen.add(url) | |
| claim = c.get("text", "") | |
| rating = rev.get("textualRating", "") | |
| out.append({"publisher": rev.get("publisher", {}).get("name", "Fact-checker"), | |
| "rating": rating, "title": rev.get("title", claim), "url": url, "claim": claim, | |
| "verdict": rating_to_verdict(rating), | |
| "match": claim_overlap(post.all_text, claim) >= self.MATCH}) | |
| out.sort(key=lambda x: not x["match"]) # same-claim results first | |
| n = sum(x["match"] for x in out) | |
| return {"factchecks": out[:4], "confidence": f"{n} same-claim fact-check(s), {len(out) - n} related"} | |
| class StatCheck(Feature): | |
| name = "StatCheck" | |
| description = "number tricks in the wording" | |
| needs_text = True | |
| def run(self, post, options): | |
| return statcheck.analyze(post.all_text) | |
| class LangCheck(Feature): | |
| name = "LangCheck" | |
| description = "persuasion tricks in the language, plus source tracing" | |
| needs_text = True | |
| def run(self, post, options): | |
| return langcheck.analyze(post.all_text, trace_sources=options.get("trace", True), | |
| place=options.get("place")) | |
| class EmotionCheck(Feature): | |
| name = "EmotionCheck" | |
| description = "how strongly the post pushes anger, fear, or disgust" | |
| needs_text = True | |
| _clf = None | |
| def run(self, post, options): | |
| if EmotionCheck._clf is None: | |
| from transformers import pipeline | |
| EmotionCheck._clf = pipeline("text-classification", | |
| model="j-hartmann/emotion-english-distilroberta-base", top_k=None) | |
| scores = {d["label"]: d["score"] for d in EmotionCheck._clf(post.all_text[:2000], truncation=True)[0]} | |
| push = scores.get("anger", 0) + scores.get("fear", 0) + scores.get("disgust", 0) | |
| top = sorted(scores.items(), key=lambda kv: -kv[1])[:3] | |
| meter = ", ".join(f"{k} {v:.0%}" for k, v in top) | |
| findings = [] | |
| if push >= 0.6: | |
| findings.append({"trick": "emotional_language", | |
| "why": f"The wording leans hard on anger, fear, or disgust ({push:.0%})."}) | |
| return {"findings": findings, "notes": [f"Emotion meter: {meter}"], "confidence": f"emotion push {push:.0%}"} | |
| # ---------- local places (opt-in location) ---------- | |
| # word in the post -> OpenStreetMap tag for that kind of place | |
| PLACE_TYPES = { | |
| "library": ("amenity", "library"), "school": ("amenity", "school"), "hospital": ("amenity", "hospital"), | |
| "post office": ("amenity", "post_office"), "police": ("amenity", "police"), | |
| "fire station": ("amenity", "fire_station"), "city hall": ("amenity", "townhall"), | |
| "town hall": ("amenity", "townhall"), "pharmacy": ("amenity", "pharmacy"), "park": ("leisure", "park"), | |
| "community center": ("amenity", "community_centre"), "dmv": ("amenity", "public_building"), | |
| } | |
| def place_words(text): | |
| t = text.lower() | |
| return [w for w in PLACE_TYPES if re.search(rf"\b{w}s?\b", t)] | |
| def _miles(lat1, lon1, lat2, lon2): | |
| p = math.pi / 180 | |
| a = (math.sin((lat2 - lat1) * p / 2) ** 2 + | |
| math.cos(lat1 * p) * math.cos(lat2 * p) * math.sin((lon2 - lon1) * p / 2) ** 2) | |
| return 7918 * math.asin(math.sqrt(a)) # Earth's diameter in miles | |
| def nearest_place(word, lat, lon, radius_m=10000): | |
| import requests | |
| key, value = PLACE_TYPES[word] | |
| q = (f'[out:json][timeout:10];(node["{key}"="{value}"](around:{radius_m},{lat},{lon});' | |
| f'way["{key}"="{value}"](around:{radius_m},{lat},{lon}););out center tags 15;') | |
| r = requests.post("https://overpass-api.de/api/interpreter", data={"data": q}, headers=HEADERS, timeout=15) | |
| best = None | |
| for el in r.json().get("elements", []): | |
| tags = el.get("tags", {}) | |
| plat, plon = el.get("lat") or el.get("center", {}).get("lat"), el.get("lon") or el.get("center", {}).get("lon") | |
| if not tags.get("name") or plat is None: | |
| continue | |
| d = _miles(lat, lon, plat, plon) | |
| if best is None or d < best[0]: | |
| best = (d, tags, plat, plon) | |
| if best is None: | |
| return None | |
| d, tags, plat, plon = best | |
| maps = f"https://www.google.com/maps/search/?api=1&query={plat},{plon}" | |
| site = tags.get("website") or tags.get("contact:website") | |
| phone = tags.get("phone") or tags.get("contact:phone") | |
| hours = tags.get("opening_hours") | |
| contacts, html_contacts = [f"[Map]({maps})"], [f'<a href="{escape(maps)}" target="_blank">Open in Maps</a>'] | |
| if site: | |
| contacts.append(f"[Website]({site})") | |
| html_contacts.append(f'<a href="{escape(site)}" target="_blank">Official website</a>') | |
| if phone: | |
| contacts.append(f"Phone: {phone}") | |
| html_contacts.append(f"Phone: {escape(phone)}") | |
| if hours: | |
| contacts.append(f"Listed hours: {hours}") | |
| html_contacts.append(f"Listed hours: {escape(hours)}") | |
| return {"name": tags["name"], "distance": f"{d:.1f} miles away", "contacts": contacts, "contacts_html": html_contacts} | |
| def reverse_place(lat, lon): | |
| """City and state for a location, so source tracing can search local news. Rounded for privacy.""" | |
| import requests | |
| try: | |
| r = requests.get("https://nominatim.openstreetmap.org/reverse", | |
| params={"format": "jsonv2", "lat": round(lat, 2), "lon": round(lon, 2), "zoom": 10}, | |
| headers=HEADERS, timeout=8) | |
| a = r.json().get("address", {}) | |
| city = a.get("city") or a.get("town") or a.get("village") or a.get("county") | |
| return ", ".join(x for x in (city, a.get("state")) if x) or None | |
| except Exception: | |
| return None | |
| class LocalCheck(Feature): | |
| name = "LocalCheck" | |
| description = "finds the real local place a post mentions so you can check with them (only with your location)" | |
| needs_text = True | |
| def wants(self, post): | |
| return super().wants(post) and bool(place_words(post.all_text)) | |
| def run(self, post, options): | |
| words = place_words(post.all_text)[:2] | |
| loc = options.get("location") | |
| if not loc: | |
| return {"notes": [f"This post is about a local {words[0]}. Tap \"Use my location\" to find the nearest one " | |
| "and check with them directly."], "confidence": "no location shared"} | |
| lat, lon = round(loc[0], 3), round(loc[1], 3) # about 100 m, plenty for finding a library | |
| places = [p for p in (nearest_place(w, lat, lon) for w in words) if p] | |
| notes = [] if places else [f"Couldn't find a {words[0]} within about 6 miles."] | |
| return {"places": places, "notes": notes, "confidence": f"{len(places)} nearby place(s)"} | |
| # ---------- links ---------- | |
| SHORTENERS = {"bit.ly", "tinyurl.com", "t.co", "goo.gl", "ow.ly", "is.gd", "buff.ly", "rebrand.ly", "cutt.ly", | |
| "shorturl.at", "tiny.cc", "rb.gy", "lnkd.in", "s.id"} | |
| # brand word -> the domains it really uses | |
| BRANDS = {"paypal": {"paypal.com"}, "amazon": {"amazon.com", "amazonaws.com", "amazon.co.uk"}, | |
| "apple": {"apple.com", "icloud.com"}, "google": {"google.com", "youtube.com", "goo.gl"}, | |
| "microsoft": {"microsoft.com", "live.com", "office.com"}, "netflix": {"netflix.com"}, | |
| "facebook": {"facebook.com", "fb.com"}, "instagram": {"instagram.com"}, | |
| "chase": {"chase.com"}, "wellsfargo": {"wellsfargo.com"}, "bankofamerica": {"bankofamerica.com"}, | |
| "usps": {"usps.com"}, "fedex": {"fedex.com"}, "ups": {"ups.com"}, "irs": {"irs.gov"}, | |
| "medicare": {"medicare.gov"}, "socialsecurity": {"ssa.gov"}, "cnn": {"cnn.com"}, | |
| "foxnews": {"foxnews.com"}, "nytimes": {"nytimes.com"}, "bbc": {"bbc.com", "bbc.co.uk"}, | |
| "abcnews": {"abcnews.go.com", "go.com"}, "nbcnews": {"nbcnews.com"}, "reuters": {"reuters.com"}} | |
| TWO_PART_TLDS = {"co.uk", "com.au", "com.co", "co.in", "com.br", "co.jp", "go.com"} | |
| LEET = str.maketrans({"0": "o", "1": "l", "3": "e", "4": "a", "5": "s", "7": "t"}) | |
| def registered_domain(host): | |
| parts = host.lower().strip(".").split(".") | |
| if len(parts) >= 3 and ".".join(parts[-2:]) in TWO_PART_TLDS: | |
| return ".".join(parts[-3:]) | |
| return ".".join(parts[-2:]) | |
| def lookalike_brand(host): | |
| """The brand a domain is pretending to be, or None if it's the real one (or not a brand).""" | |
| reg = registered_domain(host) | |
| name = reg.split(".")[0].translate(LEET) | |
| tokens = set(re.split(r"[-_]", name)) | |
| for brand, official in BRANDS.items(): | |
| if reg in official or any(host.endswith("." + o) or host == o for o in official): | |
| continue | |
| hit = brand in tokens if len(brand) < 5 else brand in name.replace("-", "") | |
| if hit: | |
| return brand | |
| return None | |
| def resolve_short(url): | |
| """Follow a short link to where it really goes, without loading the page.""" | |
| import requests | |
| try: | |
| return requests.head(url, allow_redirects=True, timeout=5, headers=HEADERS).url | |
| except Exception: | |
| return None | |
| def domain_age_days(reg): | |
| """Days since the domain was registered, from public RDAP records.""" | |
| import datetime as dt | |
| import requests | |
| try: | |
| data = requests.get(f"https://rdap.org/domain/{reg}", timeout=6, headers=HEADERS).json() | |
| for ev in data.get("events", []): | |
| if ev.get("eventAction") == "registration": | |
| born = dt.datetime.fromisoformat(ev["eventDate"].replace("Z", "+00:00")) | |
| return (dt.datetime.now(dt.timezone.utc) - born).days | |
| except Exception: | |
| pass | |
| return None | |
| class LinkCheck(Feature): | |
| name = "LinkCheck" | |
| description = "where a link really goes, lookalike sites, and brand-new sites" | |
| NEW_SITE_DAYS = 180 | |
| def wants(self, post): | |
| return bool(post.links) | |
| def run(self, post, options): | |
| from urllib.parse import urlparse | |
| findings, notes = [], [] | |
| for raw in post.links[:3]: | |
| url = raw.rstrip(").,!?;:'\"") | |
| if not url.lower().startswith("http"): | |
| url = "https://" + url | |
| host = (urlparse(url).hostname or "").lower() | |
| if not host: | |
| continue | |
| if registered_domain(host) in SHORTENERS: | |
| real = resolve_short(url) | |
| real_host = (urlparse(real).hostname or "").lower() if real else "" | |
| findings.append({"trick": "hidden_link", "quote": raw, | |
| "why": f"This short link hides where it goes" + | |
| (f". It really goes to {real_host}." if real_host else ".")}) | |
| if real_host: | |
| host = real_host | |
| if re.fullmatch(r"[0-9.]+", host) or "xn--" in host: | |
| findings.append({"trick": "lookalike_link", "quote": raw, | |
| "why": "The link uses a disguised address instead of a normal website name."}) | |
| brand = lookalike_brand(host) | |
| if brand: | |
| findings.append({"trick": "lookalike_link", "quote": raw, | |
| "why": f"{host} looks like {brand}, but it isn't one of {brand}'s real websites."}) | |
| reg = registered_domain(host) | |
| age = domain_age_days(reg) | |
| if age is not None and age < self.NEW_SITE_DAYS: | |
| findings.append({"trick": "new_site", "quote": raw, | |
| "why": f"{reg} was registered only {age} days ago."}) | |
| notes.append(f"Link goes to {host}" + (f" (site registered about {age // 365} years ago)" | |
| if age and age >= 365 else "")) | |
| return {"findings": findings, "notes": notes, "confidence": f"{len(notes)} link(s) checked"} | |
| # ---------- photos and memes ---------- | |
| IMAGE_SCHEMA = { | |
| "type": "object", | |
| "properties": { | |
| "ai_signs": {"type": "array", "items": {"type": "string"}}, | |
| "ai_likelihood": {"type": "string", "enum": ["low", "medium", "high"]}, | |
| "caption_matches": {"type": "string", "enum": ["yes", "no", "unclear", "no caption"]}, | |
| "mismatch_reason": {"type": "string"}, | |
| "search_tip": {"type": "string"}, | |
| }, | |
| "required": ["ai_signs", "ai_likelihood", "caption_matches", "mismatch_reason", "search_tip"], | |
| "additionalProperties": False, | |
| } | |
| IMAGE_PROMPT = """You help people check photos and memes before sharing them. Look at the image and its caption. | |
| 1. ai_signs: specific things you can SEE that suggest AI generation or editing (warped hands or text, | |
| melted details, impossible lighting or reflections, smeared backgrounds). Only list what is visible. | |
| Empty list if none. Never guess from the topic. | |
| 2. ai_likelihood: low, medium, or high, based only on those visible signs. | |
| 3. caption_matches: does the caption describe what the photo actually shows? Use "unclear" if you can't tell, | |
| "no caption" if there is none. Judge only what's visible, not whether the event happened. | |
| 4. mismatch_reason: one plain sentence if it doesn't match, else empty. | |
| 5. search_tip: the best words to search to find where this photo really came from. | |
| Same standard for every topic and political side. | |
| CAPTION: | |
| """ | |
| class ImageCheck(Feature): | |
| name = "ImageCheck" | |
| description = "signs a photo is AI-made, or doesn't match its caption" | |
| kinds = {"photo"} | |
| model = os.environ.get("TELL_VISION_MODEL", "claude-haiku-5-5") | |
| def run(self, post, options): | |
| import base64 | |
| import io | |
| import json | |
| tips = "Reverse-search the photo at images.google.com or tineye.com to find where it first appeared." | |
| client = langcheck._client() | |
| if client is None: | |
| return {"notes": [tips], "confidence": "needs the Claude key for photo checks"} | |
| img = post.image.convert("RGB") | |
| img.thumbnail((1024, 1024)) | |
| buf = io.BytesIO() | |
| img.save(buf, "JPEG", quality=85) | |
| caption = post.all_text or "(none)" | |
| resp = client.messages.create( | |
| model=self.model, max_tokens=800, | |
| messages=[{"role": "user", "content": [ | |
| {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", | |
| "data": base64.b64encode(buf.getvalue()).decode()}}, | |
| {"type": "text", "text": IMAGE_PROMPT + caption[:1500]}]}], | |
| output_config={"format": {"type": "json_schema", "schema": IMAGE_SCHEMA}}, | |
| ) | |
| data = json.loads(next(b.text for b in resp.content if b.type == "text")) | |
| findings = [] | |
| # needs at least two visible signs, so one odd shadow doesn't get a photo called fake | |
| if data["ai_likelihood"] == "high" and len(data["ai_signs"]) >= 2: | |
| findings.append({"trick": "ai_image", "quote": "; ".join(data["ai_signs"][:3]), | |
| "why": "This picture shows several signs of being made or changed by AI."}) | |
| if data["caption_matches"] == "no" and data["mismatch_reason"]: | |
| findings.append({"trick": "out_of_context", "why": data["mismatch_reason"]}) | |
| notes = [tips] | |
| if data["search_tip"]: | |
| notes.append(f"Try searching: {data['search_tip']}") | |
| return {"findings": findings, "notes": notes, | |
| "confidence": f"AI signs {data['ai_likelihood']}, caption match {data['caption_matches']}"} |