Upload 53 files
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitignore +11 -0
- .pytest_cache/.gitignore +2 -0
- .pytest_cache/CACHEDIR.TAG +4 -0
- .pytest_cache/README.md +8 -0
- .pytest_cache/v/cache/lastfailed +1 -0
- .pytest_cache/v/cache/nodeids +17 -0
- AI_USE.md +32 -0
- CLAUDE.md +35 -0
- README.md +85 -15
- __pycache__/features.cpython-314.pyc +0 -0
- app.py +234 -0
- core/__init__.py +0 -0
- core/__pycache__/__init__.cpython-314.pyc +0 -0
- core/__pycache__/classify.cpython-314.pyc +0 -0
- core/__pycache__/features.cpython-314.pyc +0 -0
- core/__pycache__/ocr.cpython-314.pyc +0 -0
- core/__pycache__/pipeline.cpython-314.pyc +0 -0
- core/__pycache__/post.cpython-314.pyc +0 -0
- core/__pycache__/report.cpython-314.pyc +0 -0
- core/classify.py +29 -0
- core/features.py +43 -0
- core/ocr.py +38 -0
- core/pipeline.py +84 -0
- core/post.py +32 -0
- core/report.py +273 -0
- eval/posts.json +102 -0
- features.py +432 -0
- lessons/lessons.json +140 -0
- notebooks/TELL_ChartCheck_Training.ipynb +777 -0
- notebooks/TELL_ScreenshotCheck_Quick.ipynb +787 -0
- notebooks/TELL_ScreenshotCheck_Training.ipynb +676 -0
- requirements-dev.txt +1 -0
- requirements.txt +16 -0
- scripts/run_eval.py +92 -0
- tests/__init__.py +0 -0
- tests/__pycache__/__init__.cpython-314.pyc +0 -0
- tests/__pycache__/test_pipeline.cpython-314-pytest-9.1.1.pyc +0 -0
- tests/test_pipeline.py +210 -0
- tools/__init__.py +0 -0
- tools/__pycache__/__init__.cpython-314.pyc +0 -0
- tools/__pycache__/chartcheck.cpython-314.pyc +0 -0
- tools/__pycache__/langcheck.cpython-314.pyc +0 -0
- tools/__pycache__/screenshotcheck.cpython-314.pyc +0 -0
- tools/__pycache__/statcheck.cpython-314.pyc +0 -0
- tools/chartcheck.py +175 -0
- tools/langcheck.py +287 -0
- tools/screenshotcheck.py +124 -0
- tools/statcheck.py +163 -0
- training/chart_train.py +230 -0
- training/chartgen.py +191 -0
.gitignore
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
.env
|
| 2 |
+
venv/
|
| 3 |
+
.venv/
|
| 4 |
+
__pycache__/
|
| 5 |
+
*.pyc
|
| 6 |
+
.pytest_cache/
|
| 7 |
+
models/
|
| 8 |
+
data/
|
| 9 |
+
screenshots/
|
| 10 |
+
eval/results.csv
|
| 11 |
+
*.zip
|
.pytest_cache/.gitignore
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Created by pytest automatically.
|
| 2 |
+
*
|
.pytest_cache/CACHEDIR.TAG
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Signature: 8a477f597d28d172789f06886806bc55
|
| 2 |
+
# This file is a cache directory tag created by pytest.
|
| 3 |
+
# For information about cache directory tags, see:
|
| 4 |
+
# https://bford.info/cachedir/spec.html
|
.pytest_cache/README.md
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# pytest cache directory #
|
| 2 |
+
|
| 3 |
+
This directory contains data from the pytest's cache plugin,
|
| 4 |
+
which provides the `--lf` and `--ff` options, as well as the `cache` fixture.
|
| 5 |
+
|
| 6 |
+
**Do not** commit this to version control.
|
| 7 |
+
|
| 8 |
+
See [the docs](https://docs.pytest.org/en/stable/how-to/cache.html) for more information.
|
.pytest_cache/v/cache/lastfailed
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
{}
|
.pytest_cache/v/cache/nodeids
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[
|
| 2 |
+
"tests/test_pipeline.py::test_broken_feature_does_not_break_report",
|
| 3 |
+
"tests/test_pipeline.py::test_chart_skips_ocr_and_text_tools",
|
| 4 |
+
"tests/test_pipeline.py::test_chart_with_caption_also_checks_caption",
|
| 5 |
+
"tests/test_pipeline.py::test_empty_post",
|
| 6 |
+
"tests/test_pipeline.py::test_forced_kind_skips_classifier",
|
| 7 |
+
"tests/test_pipeline.py::test_link_in_text_routes_to_linkcheck",
|
| 8 |
+
"tests/test_pipeline.py::test_local_post_with_location_finds_place",
|
| 9 |
+
"tests/test_pipeline.py::test_local_post_without_location_asks_for_it",
|
| 10 |
+
"tests/test_pipeline.py::test_photo_hits_placeholder_without_crashing",
|
| 11 |
+
"tests/test_pipeline.py::test_rating_words_map_to_verdicts",
|
| 12 |
+
"tests/test_pipeline.py::test_same_claim_fact_check_turns_easy_mode_red",
|
| 13 |
+
"tests/test_pipeline.py::test_screenshot_text_gets_read_and_checked",
|
| 14 |
+
"tests/test_pipeline.py::test_text_only_goes_to_text_tools",
|
| 15 |
+
"tests/test_pipeline.py::test_unknown_image_with_no_text_asks_user",
|
| 16 |
+
"tests/test_pipeline.py::test_unrelated_fact_check_is_shown_but_not_trusted"
|
| 17 |
+
]
|
AI_USE.md
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# AI use disclosure
|
| 2 |
+
|
| 3 |
+
The Congressional App Challenge allows AI tools as long as all use is fully disclosed
|
| 4 |
+
and AI is not the entirety of the technical work. This file is the log. Keep it honest
|
| 5 |
+
and keep adding to it as you build.
|
| 6 |
+
|
| 7 |
+
## AI tools used in development
|
| 8 |
+
|
| 9 |
+
| What | Tool | What it did | What I did myself |
|
| 10 |
+
| --- | --- | --- | --- |
|
| 11 |
+
| Starter code scaffold (app.py, tools/*.py, lessons.json) | Claude (Anthropic) | Wrote the first version of the Gradio app, the API endpoint, StatCheck rules, ChartCheck axis rules, and the screenshot heatmap | (fill in: what you changed, tested, rewrote) |
|
| 12 |
+
| Training pipeline (training/*.py, notebooks/*.ipynb, TRAINING.md template) | Claude (Anthropic) | Wrote the synthetic chart generator, the screenshot edit generator, training and evaluation code, and the Colab notebooks | (fill in: data you collected, runs you did, settings you changed, bugs you fixed) |
|
| 13 |
+
| LangCheck (tools/langcheck.py, language lesson cards, source trace display) | Claude (Anthropic) | Wrote the language technique rules, the prompts, the web search source trace, and the safeguards (quote checking, no verdict without a real source) | (fill in: posts you tested, prompt changes, techniques you added or removed) |
|
| 14 |
+
| Routing framework (core/, features.py, app.py rewrite, tests/) | Claude (Anthropic) | Wrote the Post object, classifier and OCR wrappers, feature registry, pipeline that sends each post to the right features, report renderer, and routing tests | (fill in) |
|
| 15 |
+
| Eval set and setup (eval/posts.json, scripts/run_eval.py, CLAUDE.md, .env setup) | Claude (Anthropic) | Collected 18 real posts quoted in PolitiFact and Lead Stories fact-checks plus 6 controls written from their findings, suggested trick labels, wrote the scoring script | (fill in: labels you reviewed, posts you added, rule fixes from the results) |
|
| 16 |
+
| ScreenshotCheck quick training (training/screen_autodata.py, notebooks/TELL_ScreenshotCheck_Quick.ipynb) | Claude (Anthropic) | Wrote the automatic dataset builder (RICO screenshots, synthetic chat renders, automatic edits) and the Colab notebook | (fill in: the run you did, settings you changed) |
|
| 17 |
+
| | | | |
|
| 18 |
+
|
| 19 |
+
## AI models used inside the app
|
| 20 |
+
|
| 21 |
+
| Model | Used for |
|
| 22 |
+
| --- | --- |
|
| 23 |
+
| CLIP (openai/clip-vit-base-patch32) | Router: guesses whether an upload is a chart or a screenshot |
|
| 24 |
+
| Claude Haiku (API) | StatCheck: pulls numerical claims out of text as JSON; rule checks make the final call |
|
| 25 |
+
| Claude Haiku with web search (API) | LangCheck: labels persuasion techniques with exact quotes, pulls out factual claims, searches the web for each one and summarizes only what the cited sources say |
|
| 26 |
+
| Google Fact Check Tools API (optional) | LangCheck: shows published fact-checks from professional fact-checkers |
|
| 27 |
+
| (your ChartCheck model) | Trained by me on the Misviz dataset |
|
| 28 |
+
| (your ScreenshotCheck model) | Trained by me on a dataset I built |
|
| 29 |
+
|
| 30 |
+
## Open-source libraries and datasets
|
| 31 |
+
|
| 32 |
+
Gradio, FastAPI, PyTorch, Hugging Face Transformers, timm, EasyOCR, Pillow, Misviz dataset (Tonglet et al., ACL 2026).
|
CLAUDE.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# TELL
|
| 2 |
+
|
| 3 |
+
Media literacy app for the 2026 Congressional App Challenge (WA-01). TELL shows you the
|
| 4 |
+
manipulation technique in a post, not a true/false verdict. Python, Gradio inside FastAPI,
|
| 5 |
+
hosted on Hugging Face Spaces. Deadline: submit by Sunday Oct 25 night (official cutoff Mon Oct 26, 9:00 AM Pacific).
|
| 6 |
+
|
| 7 |
+
## Layout
|
| 8 |
+
|
| 9 |
+
- `app.py` website + `/api/check` (iPhone Shortcut, HTML) + `/api/check.json`
|
| 10 |
+
- `features.py` every feature and which posts it wants. New features go here.
|
| 11 |
+
- `core/` Post object, CLIP classifier, OCR, routing pipeline, report rendering
|
| 12 |
+
- `tools/` the actual checks: chartcheck, screenshotcheck, statcheck, langcheck
|
| 13 |
+
- `lessons/lessons.json` the "spot it yourself" cards, keyed by trick name
|
| 14 |
+
- `training/`, `notebooks/` model training (runs on Colab, not here)
|
| 15 |
+
- `eval/posts.json` real posts from fact-checks, `scripts/run_eval.py` scores them
|
| 16 |
+
- `tests/` routing tests, no models or keys needed
|
| 17 |
+
|
| 18 |
+
## Commands
|
| 19 |
+
|
| 20 |
+
- Run app: `python app.py` then http://localhost:7860
|
| 21 |
+
- Tests: `python -m pytest tests`
|
| 22 |
+
- Eval (free): `python scripts/run_eval.py`; with source tracing (paid): `--trace`
|
| 23 |
+
|
| 24 |
+
## Rules
|
| 25 |
+
|
| 26 |
+
- API keys live in `.env` only. Never print them, hardcode them, or commit `.env`.
|
| 27 |
+
- AI use must be fully disclosed for the competition. After any change you make, add a
|
| 28 |
+
row to `AI_USE.md` saying what you (Claude Code) did, in one line. Don't skip this.
|
| 29 |
+
- Don't edit `expected_tricks` in `eval/posts.json` and don't loosen tests to make
|
| 30 |
+
something pass. Those are the developer's labels.
|
| 31 |
+
- Labels must be about technique, never about which side is right. If a change makes one
|
| 32 |
+
political side get flagged more on similar wording, that's a bug.
|
| 33 |
+
- Every finding's `trick` must exist in `lessons/lessons.json`.
|
| 34 |
+
- Keep comments short and only where the code isn't obvious. No em dashes in user-facing text.
|
| 35 |
+
- Run `python -m pytest tests` after changes.
|
README.md
CHANGED
|
@@ -1,15 +1,85 @@
|
|
| 1 |
-
|
| 2 |
-
|
| 3 |
-
|
| 4 |
-
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# TELL
|
| 2 |
+
|
| 3 |
+
TELL shows you the trick, not the verdict. Upload a chart or screenshot, or paste a
|
| 4 |
+
headline, and TELL names the manipulation technique and teaches you to spot it.
|
| 5 |
+
|
| 6 |
+
## Run it on your laptop
|
| 7 |
+
|
| 8 |
+
python -m venv venv
|
| 9 |
+
venv\Scripts\activate (Windows) or source venv/bin/activate (Mac)
|
| 10 |
+
pip install -r requirements.txt
|
| 11 |
+
python app.py
|
| 12 |
+
|
| 13 |
+
Open http://localhost:7860. Quick tool tests:
|
| 14 |
+
|
| 15 |
+
python tools/statcheck.py
|
| 16 |
+
python tools/chartcheck.py
|
| 17 |
+
|
| 18 |
+
python tools/langcheck.py
|
| 19 |
+
|
| 20 |
+
Optional keys (the app still runs without them, using rule checks only):
|
| 21 |
+
|
| 22 |
+
ANTHROPIC_API_KEY StatCheck + LangCheck AI layer, and LangCheck source tracing (web search)
|
| 23 |
+
GOOGLE_FACTCHECK_KEY LangCheck: published fact-checks (free, Google Cloud console,
|
| 24 |
+
enable "Fact Check Tools API", create an API key)
|
| 25 |
+
|
| 26 |
+
set ANTHROPIC_API_KEY=your-key (Windows) or export ANTHROPIC_API_KEY=your-key
|
| 27 |
+
|
| 28 |
+
Cost: web search is about $0.01 per search. LangCheck does up to 3 claims x 4 searches,
|
| 29 |
+
so at most about $0.12 per post, usually less. Repeat posts are cached. Set a monthly
|
| 30 |
+
spend limit in the Claude Console so a viral demo can't surprise you.
|
| 31 |
+
|
| 32 |
+
## Put it online (Hugging Face Spaces)
|
| 33 |
+
|
| 34 |
+
1. huggingface.co, New Space, SDK: Gradio, hardware: free CPU.
|
| 35 |
+
2. In the Space's Settings, Secrets: add ANTHROPIC_API_KEY (and GOOGLE_FACTCHECK_KEY if you have one).
|
| 36 |
+
3. Push this folder to the Space's Git repo. It builds and gives you a public link.
|
| 37 |
+
4. The Space needs this block at the TOP of README.md (Hugging Face reads it):
|
| 38 |
+
|
| 39 |
+
---
|
| 40 |
+
title: TELL
|
| 41 |
+
sdk: gradio
|
| 42 |
+
app_file: app.py
|
| 43 |
+
---
|
| 44 |
+
|
| 45 |
+
## Plug in your trained models
|
| 46 |
+
|
| 47 |
+
Train in Colab (notebooks/), push weights to a Hugging Face model repo, then set
|
| 48 |
+
Space secrets CHARTCHECK_MODEL and SCREENSHOT_MODEL to the repo names and fill in
|
| 49 |
+
the TODOs in tools/chartcheck.py and tools/screenshotcheck.py.
|
| 50 |
+
|
| 51 |
+
## Eval
|
| 52 |
+
|
| 53 |
+
python scripts/run_eval.py (free)
|
| 54 |
+
python scripts/run_eval.py --trace (source tracing, costs money)
|
| 55 |
+
|
| 56 |
+
## Tests
|
| 57 |
+
|
| 58 |
+
pip install pytest
|
| 59 |
+
python -m pytest tests
|
| 60 |
+
|
| 61 |
+
## How a post gets routed
|
| 62 |
+
|
| 63 |
+
Post (image and/or text)
|
| 64 |
+
-> core/classify.py CLIP decides: chart, screenshot, photo, or unknown
|
| 65 |
+
-> core/ocr.py reads the words off screenshots and photos
|
| 66 |
+
-> core/features.py every feature that wants this kind of post is picked
|
| 67 |
+
-> core/pipeline.py runs them side by side, one failing doesn't stop the rest
|
| 68 |
+
-> core/report.py one combined result
|
| 69 |
+
|
| 70 |
+
To add a feature, add a class to features.py (instructions at the top of that file).
|
| 71 |
+
|
| 72 |
+
## Structure
|
| 73 |
+
|
| 74 |
+
app.py website + /api/check (iPhone Shortcut) + /api/check.json
|
| 75 |
+
features.py the list of features and what posts each one wants
|
| 76 |
+
core/ post, classifier, OCR, routing pipeline, report
|
| 77 |
+
tests/ routing tests (no models or keys needed)
|
| 78 |
+
tools/statcheck.py number tricks in text (works now)
|
| 79 |
+
tools/langcheck.py language tricks in text + traces claims back to sources
|
| 80 |
+
tools/chartcheck.py axis rule checks (work now) + your trained model
|
| 81 |
+
tools/screenshotcheck.py heatmap (works now) + your trained model
|
| 82 |
+
lessons/lessons.json the "spot it yourself" lesson cards
|
| 83 |
+
training/ dataset generators + training code (tested)
|
| 84 |
+
notebooks/ Colab notebooks that run the training
|
| 85 |
+
TRAINING.md datasets, methods, results, licenses
|
__pycache__/features.cpython-314.pyc
ADDED
|
Binary file (31.7 kB). View file
|
|
|
app.py
ADDED
|
@@ -0,0 +1,234 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
TELL: shows you the trick, not the verdict.
|
| 3 |
+
|
| 4 |
+
One server, two front doors:
|
| 5 |
+
/ the website (Gradio)
|
| 6 |
+
/api/check for the iPhone Shortcut (HTML back), /api/check.json for anything else
|
| 7 |
+
|
| 8 |
+
Run it: python app.py, then open http://localhost:7860
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import io
|
| 12 |
+
import logging
|
| 13 |
+
from html import escape
|
| 14 |
+
|
| 15 |
+
from dotenv import load_dotenv
|
| 16 |
+
|
| 17 |
+
load_dotenv() # reads .env on your laptop; on Hugging Face the Space secrets are already set
|
| 18 |
+
|
| 19 |
+
import gradio as gr # noqa: E402
|
| 20 |
+
import uvicorn # noqa: E402
|
| 21 |
+
from fastapi import FastAPI, File, Form, UploadFile # noqa: E402
|
| 22 |
+
from fastapi.responses import HTMLResponse, JSONResponse # noqa: E402
|
| 23 |
+
from markdown_it import MarkdownIt # noqa: E402
|
| 24 |
+
from PIL import Image # noqa: E402
|
| 25 |
+
|
| 26 |
+
import features # noqa: E402 (importing it registers every feature)
|
| 27 |
+
from core import pipeline # noqa: E402
|
| 28 |
+
from core.features import all_features # noqa: E402
|
| 29 |
+
from core.post import Post # noqa: E402
|
| 30 |
+
from core.report import EASY_CSS, easy_html, render_markdown # noqa: E402
|
| 31 |
+
|
| 32 |
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
| 33 |
+
|
| 34 |
+
CREDIT = "Created by Daksh Mannadath · October 25, 2026"
|
| 35 |
+
|
| 36 |
+
# One tab per checker: (tab label, feature name, forced post kind, input type, what it's for, what it needs)
|
| 37 |
+
TOOLS = [
|
| 38 |
+
("Charts", "ChartCheck", "chart", "image",
|
| 39 |
+
"Use this for a graph or chart. It checks for a chopped axis, an upside-down axis, uneven steps, and two-axis tricks.",
|
| 40 |
+
"Add a picture of the chart."),
|
| 41 |
+
("Screenshots", "ScreenshotCheck", "screenshot", "image",
|
| 42 |
+
"Use this for a screenshot of a text, DM, or post. It looks for text that was edited, and shows where.",
|
| 43 |
+
"Add a screenshot."),
|
| 44 |
+
("Photos", "ImageCheck", "photo", "image+text",
|
| 45 |
+
"Use this for a photo or meme. It looks for visible signs of AI, and a caption that doesn't match the picture.",
|
| 46 |
+
"Add a photo."),
|
| 47 |
+
("Links", "LinkCheck", None, "text",
|
| 48 |
+
"Paste a message or post with a link. It shows where the link really goes and flags lookalike and brand-new sites.",
|
| 49 |
+
"There's no link in this text. Paste a post that includes one."),
|
| 50 |
+
("Fact-checks", "FactCheck", None, "text",
|
| 51 |
+
"Paste a claim. It finds published fact-checks of the same claim from fact-checkers like PolitiFact.",
|
| 52 |
+
"Paste a claim first. (This check also needs the Google Fact Check key.)"),
|
| 53 |
+
("Numbers", "StatCheck", None, "text",
|
| 54 |
+
"Paste a post with numbers or a study. It catches scary percents, missing \"out of how many\", and \"linked\" sold as \"caused\".",
|
| 55 |
+
"Paste some text first."),
|
| 56 |
+
("Language", "LangCheck", None, "text",
|
| 57 |
+
"Paste any post. It names persuasion tricks with exact quotes, then traces the claims back to real sources.",
|
| 58 |
+
"Paste some text first."),
|
| 59 |
+
("Emotion", "EmotionCheck", None, "text",
|
| 60 |
+
"Paste any post. It measures how hard the wording pushes anger, fear, or disgust.",
|
| 61 |
+
"Paste some text first."),
|
| 62 |
+
("Local", "LocalCheck", None, "text+location",
|
| 63 |
+
"Paste a post about a local place, like a library, school, or park. With your permission, it finds the real place so you can check with them.",
|
| 64 |
+
"This post doesn't mention a local place TELL knows, like a library, school, park, or city hall."),
|
| 65 |
+
]
|
| 66 |
+
|
| 67 |
+
# Runs in the browser. Location never leaves it unless the user allows it, and it's only kept for this page.
|
| 68 |
+
GET_LOCATION_JS = """
|
| 69 |
+
async () => {
|
| 70 |
+
if (!navigator.geolocation) return [null, null, "This browser can't share location. TELL still works without it."];
|
| 71 |
+
try {
|
| 72 |
+
const pos = await new Promise((ok, no) =>
|
| 73 |
+
navigator.geolocation.getCurrentPosition(ok, no, {timeout: 10000, maximumAge: 600000}));
|
| 74 |
+
return [pos.coords.latitude, pos.coords.longitude,
|
| 75 |
+
"Location is on for this page. TELL only uses it for posts about local places and never saves it."];
|
| 76 |
+
} catch (e) {
|
| 77 |
+
return [null, null, "Location not shared. That's fine, TELL still works."];
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
"""
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def options_for(location=None, trace=True):
|
| 84 |
+
options = {"trace": trace}
|
| 85 |
+
if location: # only set when the user tapped "Use my location" and allowed it
|
| 86 |
+
options["location"] = location
|
| 87 |
+
options["place"] = features.reverse_place(*location)
|
| 88 |
+
return options
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def check(image, text, trace=True, location=None, kind=None):
|
| 92 |
+
return pipeline.run(Post(image=image, text=text or ""), options_for(location, trace), kind)
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def header_html():
|
| 96 |
+
items = "".join(f"<li><b>{escape(f.name)}</b> {escape(f.description)}</li>" for f in all_features())
|
| 97 |
+
return f"""
|
| 98 |
+
<div id="tell-header">
|
| 99 |
+
<div>
|
| 100 |
+
<div class="wordmark">TELL<span></span></div>
|
| 101 |
+
</div>
|
| 102 |
+
<details class="about">
|
| 103 |
+
<summary>Features</summary>
|
| 104 |
+
<div class="panel">
|
| 105 |
+
<h4>What TELL checks</h4>
|
| 106 |
+
<ul>{items}</ul>
|
| 107 |
+
<p class="credit">{escape(CREDIT)}</p>
|
| 108 |
+
<p class="credit">Nothing you upload is stored.</p>
|
| 109 |
+
</div>
|
| 110 |
+
</details>
|
| 111 |
+
</div>"""
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def location_row():
|
| 115 |
+
with gr.Accordion("Optional: use my location for local posts", open=False):
|
| 116 |
+
btn = gr.Button("Use my location")
|
| 117 |
+
status = gr.Markdown("Only used when a post mentions a local place, like a library or school. Never saved.")
|
| 118 |
+
lat, lon = gr.Number(visible=False), gr.Number(visible=False)
|
| 119 |
+
btn.click(None, None, [lat, lon, status], js=GET_LOCATION_JS)
|
| 120 |
+
return lat, lon
|
| 121 |
+
|
| 122 |
+
|
| 123 |
+
def loc(lat, lon):
|
| 124 |
+
return (lat, lon) if lat is not None and lon is not None else None
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def make_tool_handler(name, kind, need, takes_image, takes_text, takes_location):
|
| 128 |
+
def handler(*values):
|
| 129 |
+
values = list(values)
|
| 130 |
+
image = values.pop(0) if takes_image else None
|
| 131 |
+
text = values.pop(0) if takes_text else ""
|
| 132 |
+
location = loc(values.pop(0), values.pop(0)) if takes_location else None
|
| 133 |
+
report = pipeline.run_one(Post(image=image, text=text or ""), name, options_for(location), kind)
|
| 134 |
+
html = easy_html(report) if report.sections else f'<div class="card"><p>{escape(need)}</p></div>'
|
| 135 |
+
return (html, report.heatmap) if name == "ScreenshotCheck" else html
|
| 136 |
+
return handler
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
APP_CSS = EASY_CSS + """
|
| 140 |
+
#easy { max-width: 1200px; }
|
| 141 |
+
#tell-header { display: flex; justify-content: space-between; align-items: flex-start; gap: 24px; margin: 8px 0 12px; }
|
| 142 |
+
#tell-header .wordmark { font-size: 112px; font-weight: 900; letter-spacing: -0.05em; line-height: 0.9; color: var(--body-text-color); }
|
| 143 |
+
#tell-header .wordmark span { display: inline-block; width: 24px; height: 24px; margin-left: 8px; border-radius: 50%; background: #ea580c; }
|
| 144 |
+
#tell-header .tagline { font-size: 24px !important; color: var(--body-text-color-subdued); margin-top: 10px; }
|
| 145 |
+
#tell-header details.about { position: relative; }
|
| 146 |
+
#tell-header summary { list-style: none; cursor: pointer; font-size: 18px !important; font-weight: 600; padding: 10px 20px;
|
| 147 |
+
border: 2px solid var(--border-color-primary); border-radius: 999px; color: var(--body-text-color); }
|
| 148 |
+
#tell-header summary::-webkit-details-marker { display: none; }
|
| 149 |
+
#tell-header .panel { position: absolute; right: 0; top: 56px; width: 380px; z-index: 50; padding: 20px 22px;
|
| 150 |
+
background: var(--background-fill-primary); border: 1px solid var(--border-color-primary); border-radius: 16px;
|
| 151 |
+
box-shadow: 0 12px 32px rgba(0,0,0,0.25); }
|
| 152 |
+
#tell-header .panel h4 { font-size: 20px !important; margin: 0 0 10px; color: var(--body-text-color); }
|
| 153 |
+
#tell-header .panel ul { padding-left: 18px; margin: 0 0 14px; }
|
| 154 |
+
#tell-header .panel li, #tell-header .panel li b { font-size: 15px !important; line-height: 1.45; color: var(--body-text-color); }
|
| 155 |
+
#tell-header .panel .credit { font-size: 14px !important; color: var(--body-text-color-subdued); margin: 4px 0 0; }
|
| 156 |
+
#easy .tab-wrapper button, #easy .tab-wrapper span, #easy .tab-container button, #easy .tab-container span,
|
| 157 |
+
#easy button[role="tab"], #easy button[role="tab"] span { font-size: 17px !important; font-weight: 600; }
|
| 158 |
+
#easy button[role="tab"] { padding: 8px 10px !important; }
|
| 159 |
+
.intro p { font-size: 20px !important; color: var(--body-text-color-subdued); }
|
| 160 |
+
@media (max-width: 700px) { #tell-header .wordmark { font-size: 72px; } #tell-header { flex-direction: column; } }
|
| 161 |
+
"""
|
| 162 |
+
|
| 163 |
+
# ---------- website ----------
|
| 164 |
+
|
| 165 |
+
with gr.Blocks(title="TELL") as demo:
|
| 166 |
+
with gr.Column(elem_id="easy"):
|
| 167 |
+
gr.HTML(header_html())
|
| 168 |
+
|
| 169 |
+
with gr.Tabs():
|
| 170 |
+
with gr.Tab("Easy mode"):
|
| 171 |
+
gr.Markdown("**Not sure what to check? Start here.** Easy mode runs every check that fits your post "
|
| 172 |
+
"and gives you one clear answer. Checking something specific? Pick its tab.",
|
| 173 |
+
elem_classes="intro")
|
| 174 |
+
easy_img = gr.Image(type="pil", label="Picture of the post", height=340)
|
| 175 |
+
easy_text = gr.Textbox(label="Or paste the words here", lines=4)
|
| 176 |
+
easy_lat, easy_lon = location_row()
|
| 177 |
+
easy_go = gr.Button("Check this post", variant="primary", size="lg")
|
| 178 |
+
easy_out = gr.HTML()
|
| 179 |
+
easy_go.click(lambda i, t, la, lo: easy_html(check(i, t, location=loc(la, lo))),
|
| 180 |
+
[easy_img, easy_text, easy_lat, easy_lon], easy_out)
|
| 181 |
+
|
| 182 |
+
for label, name, kind, takes, about, need in TOOLS:
|
| 183 |
+
with gr.Tab(label):
|
| 184 |
+
gr.Markdown(f"**{name}.** {about}", elem_classes="intro")
|
| 185 |
+
inputs = []
|
| 186 |
+
if "image" in takes:
|
| 187 |
+
inputs.append(gr.Image(type="pil", label="Picture", height=340))
|
| 188 |
+
if "text" in takes:
|
| 189 |
+
inputs.append(gr.Textbox(label="Caption (optional)" if takes == "image+text" else "Paste the post",
|
| 190 |
+
lines=4))
|
| 191 |
+
if "location" in takes:
|
| 192 |
+
inputs += list(location_row())
|
| 193 |
+
go = gr.Button(f"Run {name}", variant="primary", size="lg")
|
| 194 |
+
out = gr.HTML()
|
| 195 |
+
outputs = [out]
|
| 196 |
+
if name == "ScreenshotCheck":
|
| 197 |
+
outputs.append(gr.Image(label="Where it looks edited (red)"))
|
| 198 |
+
go.click(make_tool_handler(name, kind, need, "image" in takes, "text" in takes, "location" in takes),
|
| 199 |
+
inputs, outputs)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
# ---------- API ----------
|
| 203 |
+
|
| 204 |
+
api = FastAPI()
|
| 205 |
+
md = MarkdownIt("commonmark", {"html": True})
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
async def _read_upload(file: UploadFile | None):
|
| 209 |
+
if file is None:
|
| 210 |
+
return None
|
| 211 |
+
return Image.open(io.BytesIO(await file.read())).convert("RGB")
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
@api.post("/api/check", response_class=HTMLResponse)
|
| 215 |
+
async def api_check(file: UploadFile | None = File(None), text: str = Form("")):
|
| 216 |
+
report = check(await _read_upload(file), text)
|
| 217 |
+
body = md.render(render_markdown(report))
|
| 218 |
+
return ("<html><head><meta name='viewport' content='width=device-width, initial-scale=1'></head>"
|
| 219 |
+
f"<body style='font-family:system-ui;padding:16px;line-height:1.5'>{body}</body></html>")
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
@api.post("/api/check.json")
|
| 223 |
+
async def api_check_json(file: UploadFile | None = File(None), text: str = Form(""),
|
| 224 |
+
kind: str = Form(""), trace: bool = Form(True)):
|
| 225 |
+
# kind is optional: chart / screenshot / photo skips the classifier
|
| 226 |
+
report = check(await _read_upload(file), text, trace, kind=kind or None)
|
| 227 |
+
return JSONResponse(report.to_dict())
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
app = gr.mount_gradio_app(api, demo, path="/", css=APP_CSS,
|
| 231 |
+
theme=gr.themes.Soft(primary_hue="orange", font=[gr.themes.GoogleFont("Inter"), "sans-serif"]))
|
| 232 |
+
|
| 233 |
+
if __name__ == "__main__":
|
| 234 |
+
uvicorn.run(app, host="0.0.0.0", port=7860)
|
core/__init__.py
ADDED
|
File without changes
|
core/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (153 Bytes). View file
|
|
|
core/__pycache__/classify.cpython-314.pyc
ADDED
|
Binary file (1.97 kB). View file
|
|
|
core/__pycache__/features.cpython-314.pyc
ADDED
|
Binary file (3.76 kB). View file
|
|
|
core/__pycache__/ocr.cpython-314.pyc
ADDED
|
Binary file (2.43 kB). View file
|
|
|
core/__pycache__/pipeline.cpython-314.pyc
ADDED
|
Binary file (6.15 kB). View file
|
|
|
core/__pycache__/post.cpython-314.pyc
ADDED
|
Binary file (2.91 kB). View file
|
|
|
core/__pycache__/report.cpython-314.pyc
ADDED
|
Binary file (22.4 kB). View file
|
|
|
core/classify.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
|
| 3 |
+
log = logging.getLogger("tell")
|
| 4 |
+
|
| 5 |
+
# CLIP zero-shot prompts. Wording matters a lot here, so test changes on real uploads.
|
| 6 |
+
LABELS = {
|
| 7 |
+
"chart": "a bar chart or line graph with axes",
|
| 8 |
+
"screenshot": "a screenshot of a social media post or a text message conversation",
|
| 9 |
+
"photo": "a photo, painting, meme, or AI-generated picture",
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
_clf = None
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
def classify_image(image, threshold: float = 0.5) -> tuple[str, float]:
|
| 16 |
+
global _clf
|
| 17 |
+
try:
|
| 18 |
+
if _clf is None:
|
| 19 |
+
from transformers import pipeline
|
| 20 |
+
_clf = pipeline("zero-shot-image-classification", model="openai/clip-vit-base-patch32")
|
| 21 |
+
top = _clf(image, candidate_labels=list(LABELS.values()))[0]
|
| 22 |
+
except Exception as e:
|
| 23 |
+
log.warning("Classifier failed: %s", e)
|
| 24 |
+
return "unknown", 0.0
|
| 25 |
+
|
| 26 |
+
if top["score"] < threshold:
|
| 27 |
+
return "unknown", top["score"]
|
| 28 |
+
kind = next(k for k, v in LABELS.items() if v == top["label"])
|
| 29 |
+
return kind, top["score"]
|
core/features.py
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Every TELL tool is a Feature. To add one, subclass Feature in features.py,
|
| 3 |
+
decorate it with @register, and say which posts it wants. Nothing else changes.
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
from .post import Post
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
class Feature:
|
| 10 |
+
name = "Feature"
|
| 11 |
+
description = ""
|
| 12 |
+
kinds: set[str] = set() # post kinds this looks at (empty = any kind)
|
| 13 |
+
needs_text = False # only run if there's readable text
|
| 14 |
+
ready = True # False = placeholder, shows as "coming soon"
|
| 15 |
+
|
| 16 |
+
def wants(self, post: Post) -> bool:
|
| 17 |
+
if self.kinds and post.kind not in self.kinds:
|
| 18 |
+
return False
|
| 19 |
+
if self.needs_text and not post.all_text:
|
| 20 |
+
return False
|
| 21 |
+
# a feature with no rules at all would grab every post, which is never what you want
|
| 22 |
+
return bool(self.kinds or self.needs_text)
|
| 23 |
+
|
| 24 |
+
def run(self, post: Post, options: dict) -> dict:
|
| 25 |
+
raise NotImplementedError(f"{self.name} has no run() yet")
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
_registry: list[Feature] = []
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
def register(cls):
|
| 32 |
+
if any(type(f) is cls for f in _registry):
|
| 33 |
+
return cls # module got imported twice, don't double up
|
| 34 |
+
_registry.append(cls())
|
| 35 |
+
return cls
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
def all_features() -> list[Feature]:
|
| 39 |
+
return list(_registry)
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
def pick(post: Post) -> list[Feature]:
|
| 43 |
+
return [f for f in _registry if f.wants(post)]
|
core/ocr.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import logging
|
| 2 |
+
import re
|
| 3 |
+
|
| 4 |
+
import numpy as np
|
| 5 |
+
|
| 6 |
+
log = logging.getLogger("tell")
|
| 7 |
+
|
| 8 |
+
_reader = None
|
| 9 |
+
|
| 10 |
+
# Bits of app chrome that show up in almost every screenshot and mean nothing.
|
| 11 |
+
UI_NOISE = re.compile(
|
| 12 |
+
r"^(\d+[smhdw]|\d{1,2}:\d{2}(\s?[ap]m)?|reply|like|likes|share|repost|follow|"
|
| 13 |
+
r"following|more|send|message|view|views|\d+(\.\d+)?[km]?)$",
|
| 14 |
+
re.I,
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
def get_reader():
|
| 19 |
+
# EasyOCR takes a few seconds to load, so every tool shares one reader.
|
| 20 |
+
global _reader
|
| 21 |
+
if _reader is None:
|
| 22 |
+
import easyocr
|
| 23 |
+
_reader = easyocr.Reader(["en"], gpu=False)
|
| 24 |
+
return _reader
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
def read_text(image, min_conf: float = 0.3) -> str:
|
| 28 |
+
try:
|
| 29 |
+
results = get_reader().readtext(np.array(image.convert("RGB")))
|
| 30 |
+
except Exception as e:
|
| 31 |
+
log.warning("OCR failed: %s", e)
|
| 32 |
+
return ""
|
| 33 |
+
|
| 34 |
+
# Sort into rough lines (top to bottom, then left to right).
|
| 35 |
+
# 20px buckets keep words on the same line together.
|
| 36 |
+
results.sort(key=lambda r: (round(r[0][0][1] / 20), r[0][0][0]))
|
| 37 |
+
words = [t.strip() for _, t, conf in results if conf >= min_conf]
|
| 38 |
+
return " ".join(w for w in words if w and not UI_NOISE.match(w))
|
core/pipeline.py
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
post in -> figure out what it is -> run every feature that wants it -> one report out
|
| 3 |
+
"""
|
| 4 |
+
|
| 5 |
+
import logging
|
| 6 |
+
import time
|
| 7 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 8 |
+
|
| 9 |
+
from . import features, ocr
|
| 10 |
+
from .classify import classify_image
|
| 11 |
+
from .post import Post
|
| 12 |
+
from .report import Report, Section
|
| 13 |
+
|
| 14 |
+
log = logging.getLogger("tell")
|
| 15 |
+
|
| 16 |
+
# Kinds where the words in the image are worth reading. Charts are skipped on
|
| 17 |
+
# purpose: their text is axis labels, and ChartCheck reads those itself.
|
| 18 |
+
READ_TEXT_FROM = {"screenshot", "photo", "unknown"}
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
def prepare(post: Post, forced_kind: str | None = None) -> Post:
|
| 22 |
+
if post.image is None:
|
| 23 |
+
post.kind = "text" if post.text.strip() else "unknown"
|
| 24 |
+
return post
|
| 25 |
+
|
| 26 |
+
if forced_kind:
|
| 27 |
+
post.kind, post.kind_score = forced_kind, 1.0
|
| 28 |
+
else:
|
| 29 |
+
post.kind, post.kind_score = classify_image(post.image)
|
| 30 |
+
|
| 31 |
+
if post.kind in READ_TEXT_FROM:
|
| 32 |
+
post.ocr_text = ocr.read_text(post.image)
|
| 33 |
+
return post
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
def _run_one(feature, post: Post, options: dict) -> Section:
|
| 37 |
+
if not feature.ready:
|
| 38 |
+
return Section(feature=feature.name, notes=[f"{feature.name} ({feature.description}) is coming soon."])
|
| 39 |
+
|
| 40 |
+
start = time.perf_counter()
|
| 41 |
+
try:
|
| 42 |
+
raw = feature.run(post, options)
|
| 43 |
+
return Section.from_result(feature.name, raw, time.perf_counter() - start)
|
| 44 |
+
except Exception as e:
|
| 45 |
+
# one broken tool shouldn't take down the whole check
|
| 46 |
+
log.exception("%s failed", feature.name)
|
| 47 |
+
return Section(feature=feature.name, error=str(e), seconds=time.perf_counter() - start)
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def run(post: Post, options: dict | None = None, forced_kind: str | None = None) -> Report:
|
| 51 |
+
options = options or {}
|
| 52 |
+
if post.is_empty:
|
| 53 |
+
return Report(post=post, notes=["Upload a screenshot, chart, or photo, or paste a post."])
|
| 54 |
+
|
| 55 |
+
prepare(post, forced_kind)
|
| 56 |
+
chosen = features.pick(post)
|
| 57 |
+
report = Report(post=post, route=[f.name for f in chosen])
|
| 58 |
+
|
| 59 |
+
if not chosen:
|
| 60 |
+
report.notes.append("TELL couldn't tell what this is. Pick Chart, Screenshot, or Photo and try again.")
|
| 61 |
+
return report
|
| 62 |
+
|
| 63 |
+
# Features run at the same time since most of the wait is network calls.
|
| 64 |
+
# pool.map keeps them in registry order so the report reads the same every time.
|
| 65 |
+
with ThreadPoolExecutor(max_workers=len(chosen)) as pool:
|
| 66 |
+
report.sections = list(pool.map(lambda f: _run_one(f, post, options), chosen))
|
| 67 |
+
|
| 68 |
+
log.info("kind=%s route=%s times=%s", post.kind, report.route,
|
| 69 |
+
{s.feature: round(s.seconds, 2) for s in report.sections})
|
| 70 |
+
return report
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def run_one(post: Post, name: str, options: dict | None = None, forced_kind: str | None = None) -> Report:
|
| 74 |
+
"""Run just one check, for the single-tool tabs. Skips the router's choice of tools."""
|
| 75 |
+
options = options or {}
|
| 76 |
+
if post.is_empty:
|
| 77 |
+
return Report(post=post, notes=["Add something to check first."])
|
| 78 |
+
prepare(post, forced_kind)
|
| 79 |
+
feature = next(f for f in features.all_features() if f.name == name)
|
| 80 |
+
report = Report(post=post, route=[feature.name])
|
| 81 |
+
if not feature.wants(post):
|
| 82 |
+
return report # the app explains what this check needs
|
| 83 |
+
report.sections = [_run_one(feature, post, options)]
|
| 84 |
+
return report
|
core/post.py
ADDED
|
@@ -0,0 +1,32 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import re
|
| 2 |
+
from dataclasses import dataclass
|
| 3 |
+
|
| 4 |
+
from PIL import Image
|
| 5 |
+
|
| 6 |
+
# full links, www links, and bare "site.com/path" links (scam posts often skip the https://)
|
| 7 |
+
URL_RE = re.compile(r"(?:https?://|www\.)\S+|\b[a-z0-9-]+(?:\.[a-z0-9-]+)*\.(?:com|net|org|ly|co|info|gov|news|io|me|xyz|site|online)/\S*", re.I)
|
| 8 |
+
|
| 9 |
+
# Everything TELL can decide a post is. "unknown" means the classifier wasn't sure.
|
| 10 |
+
KINDS = ("text", "chart", "screenshot", "photo", "unknown")
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
@dataclass
|
| 14 |
+
class Post:
|
| 15 |
+
image: Image.Image | None = None
|
| 16 |
+
text: str = "" # typed or pasted by the user
|
| 17 |
+
kind: str = "unknown"
|
| 18 |
+
kind_score: float = 0.0 # classifier confidence, 1.0 if the user picked the kind
|
| 19 |
+
ocr_text: str = "" # text read off the image
|
| 20 |
+
|
| 21 |
+
@property
|
| 22 |
+
def all_text(self) -> str:
|
| 23 |
+
parts = [self.text.strip(), self.ocr_text.strip()]
|
| 24 |
+
return "\n".join(p for p in parts if p)
|
| 25 |
+
|
| 26 |
+
@property
|
| 27 |
+
def links(self) -> list[str]:
|
| 28 |
+
return URL_RE.findall(self.all_text)
|
| 29 |
+
|
| 30 |
+
@property
|
| 31 |
+
def is_empty(self) -> bool:
|
| 32 |
+
return self.image is None and not self.text.strip()
|
core/report.py
ADDED
|
@@ -0,0 +1,273 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import json
|
| 2 |
+
from dataclasses import dataclass, field
|
| 3 |
+
from pathlib import Path
|
| 4 |
+
|
| 5 |
+
from .post import Post
|
| 6 |
+
|
| 7 |
+
LESSONS = json.loads((Path(__file__).parent.parent / "lessons" / "lessons.json").read_text())
|
| 8 |
+
|
| 9 |
+
VERDICT_LABEL = {
|
| 10 |
+
"SUPPORTED": "\U0001F7E2 Sources support this",
|
| 11 |
+
"CONTRADICTED": "\U0001F534 Sources say otherwise",
|
| 12 |
+
"MISSING_CONTEXT": "\U0001F7E1 Missing context",
|
| 13 |
+
"DISPUTED": "\U0001F7E0 Sources disagree",
|
| 14 |
+
"NO_SOURCE": "⚪ No reliable source found",
|
| 15 |
+
"RATED": "\U0001F535 Rated by a fact-checker",
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
KIND_LABEL = {
|
| 19 |
+
"text": "text", "chart": "chart", "screenshot": "screenshot",
|
| 20 |
+
"photo": "photo or meme", "unknown": "not sure",
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
|
| 24 |
+
@dataclass
|
| 25 |
+
class Section:
|
| 26 |
+
"""What one feature found."""
|
| 27 |
+
feature: str
|
| 28 |
+
findings: list = field(default_factory=list)
|
| 29 |
+
notes: list = field(default_factory=list)
|
| 30 |
+
claim_checks: list = field(default_factory=list)
|
| 31 |
+
factchecks: list = field(default_factory=list) # published fact-checks (FactCheck)
|
| 32 |
+
places: list = field(default_factory=list) # nearby places to confirm with (LocalCheck)
|
| 33 |
+
heatmap: object = None
|
| 34 |
+
confidence: str = ""
|
| 35 |
+
seconds: float = 0.0
|
| 36 |
+
error: str = ""
|
| 37 |
+
|
| 38 |
+
@classmethod
|
| 39 |
+
def from_result(cls, feature: str, raw: dict, seconds: float = 0.0):
|
| 40 |
+
# tools return plain dicts, this keeps the rest of the app from caring about their shape
|
| 41 |
+
return cls(
|
| 42 |
+
feature=feature,
|
| 43 |
+
findings=raw.get("findings", []),
|
| 44 |
+
notes=raw.get("notes", []),
|
| 45 |
+
claim_checks=raw.get("claim_checks", []),
|
| 46 |
+
factchecks=raw.get("factchecks", []),
|
| 47 |
+
places=raw.get("places", []),
|
| 48 |
+
heatmap=raw.get("heatmap"),
|
| 49 |
+
confidence=str(raw.get("confidence", "")),
|
| 50 |
+
seconds=seconds,
|
| 51 |
+
)
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
@dataclass
|
| 55 |
+
class Report:
|
| 56 |
+
post: Post
|
| 57 |
+
route: list = field(default_factory=list)
|
| 58 |
+
sections: list = field(default_factory=list)
|
| 59 |
+
notes: list = field(default_factory=list)
|
| 60 |
+
|
| 61 |
+
@property
|
| 62 |
+
def findings(self) -> list:
|
| 63 |
+
return [f for s in self.sections for f in s.findings]
|
| 64 |
+
|
| 65 |
+
@property
|
| 66 |
+
def heatmap(self):
|
| 67 |
+
return next((s.heatmap for s in self.sections if s.heatmap is not None), None)
|
| 68 |
+
|
| 69 |
+
def to_dict(self) -> dict:
|
| 70 |
+
return {
|
| 71 |
+
"kind": self.post.kind,
|
| 72 |
+
"kind_score": round(self.post.kind_score, 3),
|
| 73 |
+
"route": self.route,
|
| 74 |
+
"ocr_text": self.post.ocr_text,
|
| 75 |
+
"sections": [
|
| 76 |
+
{k: v for k, v in s.__dict__.items() if k != "heatmap"} for s in self.sections
|
| 77 |
+
],
|
| 78 |
+
"notes": self.notes,
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
def _trick_cards(findings) -> list[str]:
|
| 83 |
+
lines = []
|
| 84 |
+
grouped = {}
|
| 85 |
+
for f in findings:
|
| 86 |
+
grouped.setdefault(f["trick"], []).append(f)
|
| 87 |
+
|
| 88 |
+
for trick, fs in grouped.items():
|
| 89 |
+
lesson = LESSONS.get(trick, {})
|
| 90 |
+
lines.append(f"### Trick found: {lesson.get('title', trick)}")
|
| 91 |
+
quotes = [f"> {f['quote']}" for f in fs if f.get("quote")]
|
| 92 |
+
if quotes:
|
| 93 |
+
lines.append("\n>\n".join(quotes))
|
| 94 |
+
lines.append(fs[0]["why"])
|
| 95 |
+
if lesson:
|
| 96 |
+
lines.append(
|
| 97 |
+
f"**Why it works on us:** {lesson['why_it_works']} \n"
|
| 98 |
+
f"**Spot it yourself next time:** {lesson['spot_it']}"
|
| 99 |
+
)
|
| 100 |
+
return lines
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _source_trace(checks) -> list[str]:
|
| 104 |
+
lines = ["## Source trace",
|
| 105 |
+
"TELL searched for each factual claim. Here's what the sources it found say. Read them yourself."]
|
| 106 |
+
for c in checks:
|
| 107 |
+
verdict = VERDICT_LABEL.get(c["verdict"], c["verdict"])
|
| 108 |
+
lines.append(f"**Claim:** {c['claim']} \n**{verdict}** \n{c['summary']}")
|
| 109 |
+
for src in c["sources"]:
|
| 110 |
+
quote = f": \"{src['quote'][:150]}\"" if src.get("quote") else ""
|
| 111 |
+
lines.append(f"- [{src['title']}]({src['url']}){quote}")
|
| 112 |
+
for fc in c.get("factchecks", []):
|
| 113 |
+
lines.append(f"- Fact-check by {fc['publisher']}: **{fc['rating']}**, [{fc['title']}]({fc['url']})")
|
| 114 |
+
return lines
|
| 115 |
+
|
| 116 |
+
|
| 117 |
+
def render_markdown(report: Report) -> str:
|
| 118 |
+
post = report.post
|
| 119 |
+
kind = KIND_LABEL.get(post.kind, post.kind)
|
| 120 |
+
route = ", ".join(report.route) if report.route else "nothing"
|
| 121 |
+
lines = [f"<sub>Looks like: **{kind}**. Checked with: {route}.</sub>"]
|
| 122 |
+
|
| 123 |
+
if report.findings:
|
| 124 |
+
lines += _trick_cards(report.findings)
|
| 125 |
+
elif report.sections:
|
| 126 |
+
lines.append("## No tricks found\nThat doesn't mean it's true, just that TELL didn't spot a technique it knows.")
|
| 127 |
+
|
| 128 |
+
fcs = [f for s in report.sections for f in s.factchecks]
|
| 129 |
+
if fcs:
|
| 130 |
+
lines.append("## Published fact-checks")
|
| 131 |
+
for f in fcs:
|
| 132 |
+
tag = "Same claim" if f["match"] else "Related claim, may not be the same"
|
| 133 |
+
lines.append(f"- **{f['publisher']}: {f['rating']}** ({tag}): [{f['title']}]({f['url']})")
|
| 134 |
+
|
| 135 |
+
checks = [c for s in report.sections for c in s.claim_checks]
|
| 136 |
+
if checks:
|
| 137 |
+
lines += _source_trace(checks)
|
| 138 |
+
|
| 139 |
+
places = [p for s in report.sections for p in s.places]
|
| 140 |
+
if places:
|
| 141 |
+
lines.append("## Check with them directly")
|
| 142 |
+
for p in places:
|
| 143 |
+
lines.append(f"- **{p['name']}**, {p['distance']}. " + " · ".join(p["contacts"]))
|
| 144 |
+
|
| 145 |
+
for s in report.sections:
|
| 146 |
+
for n in s.notes:
|
| 147 |
+
lines.append(f"*{n}*")
|
| 148 |
+
if s.error:
|
| 149 |
+
lines.append(f"*{s.feature} hit an error and was skipped.*")
|
| 150 |
+
for n in report.notes:
|
| 151 |
+
lines.append(f"*{n}*")
|
| 152 |
+
|
| 153 |
+
if post.ocr_text:
|
| 154 |
+
lines.append(f"<details><summary>Text TELL read off the image</summary>\n\n{post.ocr_text}\n\n</details>")
|
| 155 |
+
|
| 156 |
+
conf = "; ".join(f"{s.feature}: {s.confidence}" for s in report.sections if s.confidence)
|
| 157 |
+
lines.append(f"\n<sub>{conf}. Nothing you upload is stored.</sub>")
|
| 158 |
+
return "\n\n".join(lines)
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
# ---------- Easy mode (big, plain-language view) ----------
|
| 162 |
+
|
| 163 |
+
from html import escape
|
| 164 |
+
|
| 165 |
+
EASY_CSS = """
|
| 166 |
+
#easy { max-width: 900px; margin: 0 auto; }
|
| 167 |
+
#easy p, #easy label, #easy textarea, #easy span { font-size: 22px !important; line-height: 1.5; }
|
| 168 |
+
#easy h1 { font-size: 40px !important; }
|
| 169 |
+
#easy h2 { font-size: 30px !important; margin-top: 28px; }
|
| 170 |
+
#easy button { font-size: 28px !important; padding: 18px !important; }
|
| 171 |
+
.verdict { border-radius: 16px; padding: 24px 28px; margin: 16px 0; color: #111; }
|
| 172 |
+
.verdict h2 { font-size: 36px !important; margin: 0 0 8px !important; color: #111; }
|
| 173 |
+
.v-red { background: #ffd9d9; border: 5px solid #c62828; }
|
| 174 |
+
.v-orange { background: #ffe8cc; border: 5px solid #e65100; }
|
| 175 |
+
.v-yellow { background: #fff6c4; border: 5px solid #a68400; }
|
| 176 |
+
.v-green { background: #dcf5e0; border: 5px solid #2e7d32; }
|
| 177 |
+
.card { background: #ffffff; color: #111; border: 2px solid #d0d0d0; border-radius: 12px; padding: 16px 22px; margin: 12px 0; }
|
| 178 |
+
.card h3 { font-size: 26px !important; margin: 0 0 6px; color: #111; }
|
| 179 |
+
.card a { color: #0b57d0; text-decoration: underline; }
|
| 180 |
+
.quote { font-style: italic; color: #444 !important; }
|
| 181 |
+
.card p, .card b, .card span, .card li, .verdict p { color: #111 !important; }
|
| 182 |
+
.card .quote { color: #444 !important; }
|
| 183 |
+
.tag { display: inline-block; font-size: 16px !important; padding: 2px 10px; border-radius: 999px; background: #eee; color: #333 !important; }
|
| 184 |
+
"""
|
| 185 |
+
|
| 186 |
+
RED_FLAGS = {"screenshot_edited", "screenshot_generated"}
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def _verdict(report):
|
| 190 |
+
tricks = {f["trick"] for f in report.findings}
|
| 191 |
+
verdicts = {c["verdict"] for s in report.sections for c in s.claim_checks}
|
| 192 |
+
verdicts |= {f["verdict"] for s in report.sections for f in s.factchecks if f["match"]}
|
| 193 |
+
n = len(tricks)
|
| 194 |
+
if "CONTRADICTED" in verdicts or tricks & RED_FLAGS:
|
| 195 |
+
return "v-red", "Stop. Don't share this yet.", "Something in this post looks false or edited. See why below."
|
| 196 |
+
if n >= 2 or verdicts & {"MISSING_CONTEXT", "DISPUTED"}:
|
| 197 |
+
signs = f"{n} warning signs" if n >= 2 else "warning signs"
|
| 198 |
+
return "v-orange", f"Be careful: {signs}", "This post uses tricks to make you react before you think."
|
| 199 |
+
if n == 1:
|
| 200 |
+
return "v-yellow", "One warning sign", "Read it closely before you believe it or share it."
|
| 201 |
+
return "v-green", "No warning signs found", "TELL didn't spot any tricks it knows. That doesn't prove it's true."
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
def easy_html(report: Report) -> str:
|
| 205 |
+
if not report.sections:
|
| 206 |
+
notes = " ".join(report.notes) or "Add a picture of the post or paste its words, then press Check."
|
| 207 |
+
return f'<div class="card"><p>{escape(notes)}</p></div>'
|
| 208 |
+
|
| 209 |
+
if all(s.error for s in report.sections):
|
| 210 |
+
return '<div class="card"><p>That check hit a problem and couldn\'t finish. Try again in a moment.</p></div>'
|
| 211 |
+
|
| 212 |
+
css_class, title, sub = _verdict(report)
|
| 213 |
+
out = [f'<div class="verdict {css_class}"><h2>{escape(title)}</h2><p>{escape(sub)}</p></div>']
|
| 214 |
+
|
| 215 |
+
fcs = [f for s in report.sections for f in s.factchecks]
|
| 216 |
+
if fcs:
|
| 217 |
+
out.append("<h2>Quick fact check</h2>")
|
| 218 |
+
for f in fcs:
|
| 219 |
+
tag = "Same claim" if f["match"] else "Related claim, may not be the same"
|
| 220 |
+
out.append(f'<div class="card"><span class="tag">{escape(tag)}</span>'
|
| 221 |
+
f'<h3>{escape(f["publisher"])} rated it: {escape(f["rating"])}</h3>'
|
| 222 |
+
f'<p class="quote">{escape(f["claim"])}</p>'
|
| 223 |
+
f'<p><a href="{escape(f["url"])}" target="_blank">Read the fact-check</a></p></div>')
|
| 224 |
+
|
| 225 |
+
first = {} # one card per trick
|
| 226 |
+
for f in report.findings:
|
| 227 |
+
first.setdefault(f["trick"], f)
|
| 228 |
+
if first:
|
| 229 |
+
out.append("<h2>The tricks it uses</h2>")
|
| 230 |
+
for trick, f in first.items():
|
| 231 |
+
lesson = LESSONS.get(trick, {})
|
| 232 |
+
quote = f.get("quote", "")
|
| 233 |
+
quote_html = f'<p class="quote">"{escape(quote)}"</p>' if quote and len(quote) < 200 else ""
|
| 234 |
+
out.append(
|
| 235 |
+
f'<div class="card"><h3>{escape(lesson.get("title", trick))}</h3>{quote_html}'
|
| 236 |
+
f'<p>{escape(lesson.get("what", f["why"]))}</p>'
|
| 237 |
+
f'<p><b>Next time:</b> {escape(lesson.get("spot_it", ""))}</p></div>')
|
| 238 |
+
|
| 239 |
+
extra = [n for s in report.sections if s.feature in ("LinkCheck", "ImageCheck") for n in s.notes]
|
| 240 |
+
if extra:
|
| 241 |
+
out.append("<h2>More to check</h2>")
|
| 242 |
+
out += [f'<div class="card"><p>{escape(n)}</p></div>' for n in extra]
|
| 243 |
+
|
| 244 |
+
feelings = [n for s in report.sections if s.feature == "EmotionCheck" for n in s.notes]
|
| 245 |
+
if feelings:
|
| 246 |
+
out.append("<h2>How it wants you to feel</h2>")
|
| 247 |
+
out += [f'<div class="card"><p>{escape(n)}</p></div>' for n in feelings]
|
| 248 |
+
|
| 249 |
+
checks = [c for s in report.sections for c in s.claim_checks]
|
| 250 |
+
if checks:
|
| 251 |
+
out.append("<h2>What other sources say</h2>")
|
| 252 |
+
for c in checks:
|
| 253 |
+
links = "".join(f'<p>Source: <a href="{escape(s["url"])}" target="_blank">{escape(s["title"])}</a></p>'
|
| 254 |
+
for s in c["sources"][:2])
|
| 255 |
+
fcs = "".join(f'<p>Fact-checker {escape(x["publisher"])} rated a similar claim: <b>{escape(x["rating"])}</b> '
|
| 256 |
+
f'(<a href="{escape(x["url"])}" target="_blank">read it</a>)</p>' for x in c.get("factchecks", []))
|
| 257 |
+
out.append(f'<div class="card"><h3>{escape(VERDICT_LABEL.get(c["verdict"], c["verdict"]))}</h3>'
|
| 258 |
+
f'<p class="quote">{escape(c["claim"])}</p><p>{escape(c["summary"])}</p>{links}{fcs}</div>')
|
| 259 |
+
|
| 260 |
+
places = [p for s in report.sections for p in s.places]
|
| 261 |
+
local_notes = [n for s in report.sections if s.feature == "LocalCheck" for n in s.notes]
|
| 262 |
+
if places or local_notes:
|
| 263 |
+
out.append("<h2>Check with them directly</h2>")
|
| 264 |
+
out += [f'<div class="card"><p>{escape(n)}</p></div>' for n in local_notes]
|
| 265 |
+
for p in places:
|
| 266 |
+
contacts = "".join(f"<p>{c}</p>" for c in p["contacts_html"])
|
| 267 |
+
out.append(f'<div class="card"><h3>{escape(p["name"])}</h3><p>{escape(p["distance"])}</p>{contacts}</div>')
|
| 268 |
+
|
| 269 |
+
out.append('<h2>What to do</h2><div class="card">'
|
| 270 |
+
"<p>1. Don't share it yet.</p>"
|
| 271 |
+
"<p>2. Look for the same story on a news site you already trust.</p>"
|
| 272 |
+
"<p>3. If it asks for money, passwords, or to act fast, ask someone you trust first.</p></div>")
|
| 273 |
+
return "\n".join(out)
|
eval/posts.json
ADDED
|
@@ -0,0 +1,102 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"about": "Real posts quoted in published fact-checks, plus controls. 'text_note' says whether the text is verbatim or stitched from the quoted pieces the fact-check printed. 'suggested_tricks' are a first guess: review every one and copy the ones you agree with into 'expected_tricks'. The eval script scores against expected_tricks only.",
|
| 3 |
+
"posts": [
|
| 4 |
+
{"id": "cleveland-clinic", "favors": "neither (health)", "text_note": "assembled from quoted fragments",
|
| 5 |
+
"text": "Cleveland Clinic: Millions of COVID-vaxxed will die within '5 years.' The renowned Cleveland Clinic has dropped a bone-chilling warning that millions of people who received COVID mRNA 'vaccines' are facing sudden death within the next 'five years.' According to the Ohio-based academic medical center, large numbers of the Covid-vaccinated 'may need a heart transplant' to avoid a sudden cardiac arrest.",
|
| 6 |
+
"rating": "PolitiFact: False", "source": "https://politifact.com/factchecks/2025/apr/04/facebook-posts/cleveland-clinic-didnt-say-millions-of-people-with/",
|
| 7 |
+
"suggested_tricks": ["emotional_language"], "expected_trace": "CONTRADICTED", "expected_tricks": []},
|
| 8 |
+
{"id": "aoc-social-security", "favors": "Republicans", "text_note": "verbatim quoted lines",
|
| 9 |
+
"text": "AOC has no excuse for why her family has been cashing her dead grandmother's Social Security checks for a decade and a half. We must have just overlooked it.",
|
| 10 |
+
"rating": "PolitiFact: Pants on Fire", "source": "https://politifact.com/factchecks/2025/apr/04/viral-image/no-aoc-is-not-cashing-grandmothers-social-security/",
|
| 11 |
+
"suggested_tricks": ["ad_hominem"], "expected_trace": "NO_SOURCE or CONTRADICTED", "expected_tricks": []},
|
| 12 |
+
{"id": "musk-golf-fake", "favors": "Democrats", "text_note": "verbatim (fabricated screenshot text)",
|
| 13 |
+
"text": "Unbelievable. The DOGE team has discovered TENS OF MILLIONS OF DOLLARS spent on GOLF by a SINGLE GOVERNMENT OFFICIAL. Stay tuned for when we figure out who it is.",
|
| 14 |
+
"rating": "PolitiFact: False", "source": "https://politifact.com/factchecks/2025/apr/04/viral-image/elon-musk-didnt-target-trumps-golf-trips-in-an-x-p/",
|
| 15 |
+
"suggested_tricks": ["emotional_language"], "expected_trace": "CONTRADICTED or NO_SOURCE", "expected_tricks": []},
|
| 16 |
+
{"id": "trump-dual-citizenship-fake", "favors": "Democrats", "text_note": "verbatim quoted pieces joined",
|
| 17 |
+
"text": "Starting on Monday, I am ending dual citizenship under the Espionage Act and all these traitors will be 'denaturalized and deported' to the country they came from. No exceptions!!!!! America is for Americans, not all these foreigners who hate our great country and don't even pay taxes.",
|
| 18 |
+
"rating": "PolitiFact: False", "source": "https://politifact.com/factchecks/2025/apr/04/viral-image/this-isnt-a-real-truth-social-post-from-trump-abou/",
|
| 19 |
+
"suggested_tricks": ["us_vs_them", "emotional_language", "absolute_claims", "scapegoating"], "expected_trace": "CONTRADICTED or NO_SOURCE", "expected_tricks": []},
|
| 20 |
+
{"id": "missing-officer-scam", "favors": "neither (scam)", "text_note": "verbatim quoted pieces joined",
|
| 21 |
+
"text": "FEMALE POLICE OFFICER MISSING in Winnemucca. Be on the look out, the police need help in locating female police officer KATIE CUNNINGHAM (28) who went missing yesterday morning. Her car was found last night with her daughter inside but unfortunately, there is still no sign of her. She is 5'9\" and 163 lbs. A silver alert has been issued for Katie who is considered to possibly be in EXTREME danger and in need of medical assistance. We are asking for the community's help. IT ONLY TAKES 2 SECONDS to share...",
|
| 22 |
+
"rating": "PolitiFact: Pants on Fire", "source": "https://politifact.com/factchecks/2025/apr/04/facebook-posts/this-police-officer-isnt-in-danger-but-your-person/",
|
| 23 |
+
"suggested_tricks": ["false_urgency", "emotional_language"], "expected_trace": "CONTRADICTED", "expected_tricks": []},
|
| 24 |
+
{"id": "dolly-parton-pride", "favors": "Republicans", "text_note": "verbatim quoted pieces joined",
|
| 25 |
+
"text": "BREAKING NEWS: Dolly Parton announces her refusal to celebrate Pride Month in June. She says 'WOKE' doesn't deserve to be commemorated.",
|
| 26 |
+
"rating": "PolitiFact: Pants on Fire", "source": "https://politifact.com/factchecks/2025/jun/25/facebook-posts/dolly-parton-pride-month-lgbtq/",
|
| 27 |
+
"suggested_tricks": ["false_urgency"], "expected_trace": "CONTRADICTED or NO_SOURCE", "expected_tricks": []},
|
| 28 |
+
{"id": "pratt-zero-votes", "favors": "Republicans", "text_note": "verbatim quoted lines",
|
| 29 |
+
"text": "Spencer Pratt got 0 out of 24,000 votes in a late night LA ballot drop. 0/24,000. A guy getting around 30% support got 0 out of 24,000. Astronomically small probability of happening. Impossible. California no longer even hides it. Doors need to be kicked in.",
|
| 30 |
+
"rating": "Lead Stories: False", "source": "https://leadstories.com/hoax-alert/2026/06/fact-check-ballot-drop-did-not-give-spencer-pratt-0-votes-in-la-mayor-race.html",
|
| 31 |
+
"suggested_tricks": ["absolute_claims", "emotional_language", "scapegoating"], "expected_trace": "CONTRADICTED", "expected_tricks": []},
|
| 32 |
+
{"id": "musk-the-view", "favors": "Democrats", "text_note": "verbatim",
|
| 33 |
+
"text": "THRILLING. Elon Musk called for a boycott of 'The View,' labeling it 'a meeting place for ignorant women.' The entire cast responded fiercely immediately after...",
|
| 34 |
+
"rating": "PolitiFact: False", "source": "https://politifact.com/factchecks/2025/apr/02/facebook-posts/dont-fall-for-unfounded-claims-about-elon-musk-urg/",
|
| 35 |
+
"suggested_tricks": ["emotional_language"], "expected_trace": "NO_SOURCE or CONTRADICTED", "expected_tricks": []},
|
| 36 |
+
{"id": "stein-ballot-rejections", "favors": "Democrats", "text_note": "assembled from quoted fragments (full post not printed)",
|
| 37 |
+
"text": "Thousands more absentee ballots being rejected this election, at a rejection rate twice as high as in 2022.",
|
| 38 |
+
"rating": "PolitiFact: Mostly False", "source": "https://politifact.com/factchecks/2026/oct/23/josh-stein/fact-checking-steins-claims-about-mail-in-ballot-rejections-in-north-carolina/",
|
| 39 |
+
"suggested_tricks": ["relative_risk_no_baseline"], "expected_trace": "CONTRADICTED or MISSING_CONTEXT", "expected_tricks": []},
|
| 40 |
+
{"id": "hinson-8-million", "favors": "Democrats", "text_note": "verbatim ad lines",
|
| 41 |
+
"text": "After just five years in Washington, she's $8 million richer. Ashley Hinson got rich. Iowa got left behind.",
|
| 42 |
+
"rating": "PolitiFact: Mostly False", "source": "https://politifact.com/factchecks/2026/oct/23/winsenate/ashley-hinson-8-million-richer-iowa-senate/",
|
| 43 |
+
"suggested_tricks": ["us_vs_them"], "expected_trace": "MISSING_CONTEXT", "expected_tricks": []},
|
| 44 |
+
{"id": "castor-insider-trading", "favors": "Republicans", "text_note": "assembled from quoted fragments",
|
| 45 |
+
"text": "Kathy Castor worked the system, repeatedly violating a law aimed at stopping members of Congress from insider trading, thanks to two decades of shady trading.",
|
| 46 |
+
"rating": "PolitiFact: Mostly False", "source": "https://politifact.com/factchecks/2026/oct/22/club-growth-action/kathy-castor-insider-trading/",
|
| 47 |
+
"suggested_tricks": ["ad_hominem", "emotional_language"], "expected_trace": "MISSING_CONTEXT or CONTRADICTED", "expected_tricks": []},
|
| 48 |
+
{"id": "warmington-purdue-TRUE", "favors": "Republicans", "text_note": "verbatim ad lines. Rated TRUE: tests that TELL doesn't call true claims false",
|
| 49 |
+
"text": "Cinde Warmington was a Purdue Pharma lobbyist. She said that OxyContin was a miracle drug.",
|
| 50 |
+
"rating": "PolitiFact: True", "source": "https://politifact.com/factchecks/2026/oct/22/kelly-ayotte/warmington-opioid-purdue-pharma-new-hampshire/",
|
| 51 |
+
"suggested_tricks": [], "expected_trace": "SUPPORTED", "expected_tricks": []},
|
| 52 |
+
{"id": "vance-gas-refining", "favors": "Republicans", "text_note": "partial quote (fact-check paraphrases the rest: about 20% of refining went offline under Biden)",
|
| 53 |
+
"text": "The biggest reason why gas prices are so high is because when Joe Biden was president, we had about 20% of American refining capacity go offline.",
|
| 54 |
+
"rating": "PolitiFact: False", "source": "https://politifact.com/factchecks/2026/oct/19/jd-vance/biden-gasoline-prices-refineries-cbs-town-hall/",
|
| 55 |
+
"suggested_tricks": ["scapegoating"], "expected_trace": "CONTRADICTED", "expected_tricks": []},
|
| 56 |
+
{"id": "moody-4-trillion", "favors": "Republicans", "text_note": "verbatim ad lines",
|
| 57 |
+
"text": "She works nine-hour shifts, mostly on her feet. They spent 40 years paying in to Social Security. Every dollar in this diner was earned. I voted to stop taxing tips and Social Security while my opponent backed a $4 trillion tax hike on everyone in this diner.",
|
| 58 |
+
"rating": "PolitiFact: Half True", "source": "https://politifact.com/factchecks/2026/oct/20/ashley-moody/moodys-fl-diner-ad-mischaracterizes-nixons-position-on-obbba-as-support-for-4-trillion-tax-hike/",
|
| 59 |
+
"suggested_tricks": ["emotional_language"], "expected_trace": "MISSING_CONTEXT", "expected_tricks": []},
|
| 60 |
+
{"id": "ballot-written-on", "favors": "neither (election process)", "text_note": "verbatim Facebook line",
|
| 61 |
+
"text": "Your ballot will be disqualified if it is written on.",
|
| 62 |
+
"rating": "PolitiFact: False", "source": "https://politifact.com/factchecks/2026/oct/16/social-media/in-north-carolina-and-other-states-a-ballot-isnt-disqualified-if-a-poll-worker-marks-it/",
|
| 63 |
+
"suggested_tricks": [], "expected_trace": "CONTRADICTED", "expected_tricks": []},
|
| 64 |
+
{"id": "census-24000-noncitizens", "favors": "Republicans", "text_note": "verbatim quoted lines",
|
| 65 |
+
"text": "The Census Bureau has begun checking the Voter Records from 2020 against their Citizenship Records. On the first 128,000,000 Voters, the Census Bureau proves that over 24,000 Noncitizens voted illegally! The Census Bureau is going to analyze the next 32,000,000 Voters, and this number will explode. I WON THE ELECTION! We must pass THE SAVE AMERICA ACT. Thank you for your attention to this matter!",
|
| 66 |
+
"rating": "Lead Stories: Overstated", "source": "https://leadstories.com/hoax-alert/2026/08/fact-check-census-bureau-did-not-find-over-24000-noncitizens-voted-illegally-in-2020-election.html",
|
| 67 |
+
"suggested_tricks": ["missing_denominator", "emotional_language", "false_urgency"], "expected_trace": "MISSING_CONTEXT", "expected_tricks": []},
|
| 68 |
+
{"id": "trump-wave-clip", "favors": "Democrats", "text_note": "verbatim quoted lines",
|
| 69 |
+
"text": "I now have absolutely no doubt that Donald Trump has major mental decline. Just think for one minute if Joe Biden had waved into the abyss like Trump just did here. Imagine what Fox News would be saying. There is absolutely no rational explanation for this that I can see. How about you?",
|
| 70 |
+
"rating": "Lead Stories: False (missing context)", "source": "https://leadstories.com/hoax-alert/2026/10/fact-check-video-does-not-prove-trump-waved-to-nobody-before-boarding-marine-one-people-are-outside-camera-view.html",
|
| 71 |
+
"suggested_tricks": ["absolute_claims", "ad_hominem", "us_vs_them"], "expected_trace": "CONTRADICTED or MISSING_CONTEXT", "expected_tricks": []},
|
| 72 |
+
{"id": "marshall-beef", "favors": "Republicans", "text_note": "verbatim",
|
| 73 |
+
"text": "In a state with more cattle than people, here's the guy that wants to shut down the beef industry.",
|
| 74 |
+
"rating": "PolitiFact: False", "source": "https://politifact.com/factchecks/2026/oct/20/roger-marshall/adam-hamilton-kansas-beef-climate-change-cows/",
|
| 75 |
+
"suggested_tricks": ["ad_hominem"], "expected_trace": "CONTRADICTED", "expected_tricks": []},
|
| 76 |
+
|
| 77 |
+
{"id": "control-ballot-rejections", "control": true, "favors": "neither", "text_note": "control, written from the PolitiFact findings",
|
| 78 |
+
"text": "Through Oct. 20, North Carolina had rejected 1,349 mail-in ballots, or 3.1% of those cast, compared with 2.6% at the same point in 2022. Fewer people are voting by mail this year, so the total number of rejections is down.",
|
| 79 |
+
"source": "https://politifact.com/factchecks/2026/oct/23/josh-stein/fact-checking-steins-claims-about-mail-in-ballot-rejections-in-north-carolina/",
|
| 80 |
+
"suggested_tricks": [], "expected_trace": "SUPPORTED", "expected_tricks": []},
|
| 81 |
+
{"id": "control-ballot-marks", "control": true, "favors": "neither", "text_note": "control, written from the PolitiFact findings",
|
| 82 |
+
"text": "Election officials in North Carolina say poll workers can legally write on a ballot, for example to sort it by precinct. Those marks don't disqualify your vote.",
|
| 83 |
+
"source": "https://politifact.com/factchecks/2026/oct/16/social-media/in-north-carolina-and-other-states-a-ballot-isnt-disqualified-if-a-poll-worker-marks-it/",
|
| 84 |
+
"suggested_tricks": [], "expected_trace": "SUPPORTED", "expected_tricks": []},
|
| 85 |
+
{"id": "control-census", "control": true, "favors": "neither", "text_note": "control, written from the Lead Stories findings",
|
| 86 |
+
"text": "A Census Bureau analysis flagged about 24,000 voter records that matched immigration or visa records with no later citizenship record. The report says this suggests possible noncitizenship but doesn't prove anyone voted illegally.",
|
| 87 |
+
"source": "https://leadstories.com/hoax-alert/2026/08/fact-check-census-bureau-did-not-find-over-24000-noncitizens-voted-illegally-in-2020-election.html",
|
| 88 |
+
"suggested_tricks": [], "expected_trace": "SUPPORTED", "expected_tricks": []},
|
| 89 |
+
{"id": "control-pratt-feed", "control": true, "favors": "neither", "text_note": "control, written from the Lead Stories findings",
|
| 90 |
+
"text": "During the LA mayor count, an AP data feed briefly showed Spencer Pratt with 0 votes in one update because his numbers posted about a minute after the other candidates'. County records show every candidate got votes in every update.",
|
| 91 |
+
"source": "https://leadstories.com/hoax-alert/2026/06/fact-check-ballot-drop-did-not-give-spencer-pratt-0-votes-in-la-mayor-race.html",
|
| 92 |
+
"suggested_tricks": [], "expected_trace": "SUPPORTED", "expected_tricks": []},
|
| 93 |
+
{"id": "control-trump-wave", "control": true, "favors": "neither", "text_note": "control, written from the Lead Stories findings",
|
| 94 |
+
"text": "A viral clip seemed to show Trump waving at no one before boarding Marine One. Longer footage shows he was waving at construction workers who were outside the frame.",
|
| 95 |
+
"source": "https://leadstories.com/hoax-alert/2026/10/fact-check-video-does-not-prove-trump-waved-to-nobody-before-boarding-marine-one-people-are-outside-camera-view.html",
|
| 96 |
+
"suggested_tricks": [], "expected_trace": "SUPPORTED", "expected_tricks": []},
|
| 97 |
+
{"id": "control-hinson", "control": true, "favors": "neither", "text_note": "control, written from the PolitiFact findings",
|
| 98 |
+
"text": "Comparing average estimates from her financial disclosures, Rep. Ashley Hinson's reported assets grew by roughly $4 million since 2020, much of it tied to her husband's stake in an insurance firm.",
|
| 99 |
+
"source": "https://politifact.com/factchecks/2026/oct/23/winsenate/ashley-hinson-8-million-richer-iowa-senate/",
|
| 100 |
+
"suggested_tricks": [], "expected_trace": "SUPPORTED", "expected_tricks": []}
|
| 101 |
+
]
|
| 102 |
+
}
|
features.py
ADDED
|
@@ -0,0 +1,432 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
The list of everything TELL can do. Order here is the order results show up in.
|
| 3 |
+
|
| 4 |
+
Adding a feature:
|
| 5 |
+
@register
|
| 6 |
+
class MyCheck(Feature):
|
| 7 |
+
name = "MyCheck"
|
| 8 |
+
description = "what it looks for"
|
| 9 |
+
kinds = {"photo"} # and/or needs_text = True, or override wants()
|
| 10 |
+
def run(self, post, options):
|
| 11 |
+
return {"findings": [...], "notes": [...], "confidence": "..."}
|
| 12 |
+
|
| 13 |
+
Each finding is {"trick": <key in lessons.json>, "quote": <optional>, "why": <one sentence>}.
|
| 14 |
+
Set ready = False to list it as coming soon before it works.
|
| 15 |
+
"""
|
| 16 |
+
|
| 17 |
+
import math
|
| 18 |
+
import os
|
| 19 |
+
import re
|
| 20 |
+
from html import escape
|
| 21 |
+
|
| 22 |
+
from core.features import Feature, register
|
| 23 |
+
from tools import chartcheck, langcheck, screenshotcheck, statcheck
|
| 24 |
+
|
| 25 |
+
HEADERS = {"User-Agent": "TELL media literacy app (Congressional App Challenge student project)"}
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
@register
|
| 29 |
+
class ChartCheck(Feature):
|
| 30 |
+
name = "ChartCheck"
|
| 31 |
+
description = "misleading axes and chart design"
|
| 32 |
+
kinds = {"chart"}
|
| 33 |
+
|
| 34 |
+
def run(self, post, options):
|
| 35 |
+
return chartcheck.analyze(post.image)
|
| 36 |
+
|
| 37 |
+
|
| 38 |
+
@register
|
| 39 |
+
class ScreenshotCheck(Feature):
|
| 40 |
+
name = "ScreenshotCheck"
|
| 41 |
+
description = "fake or edited screenshots"
|
| 42 |
+
kinds = {"screenshot"}
|
| 43 |
+
|
| 44 |
+
def run(self, post, options):
|
| 45 |
+
return screenshotcheck.analyze(post.image)
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# ---------- quick fact check ----------
|
| 49 |
+
|
| 50 |
+
STOP = set("the a an and or but of to in on for with is are was were be been this that it its at by from as "
|
| 51 |
+
"has have had will would not no he she they we you i his her their our your who what".split())
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _words(text):
|
| 55 |
+
return {w for w in re.findall(r"[a-z0-9']+", text.lower()) if len(w) > 2 and w not in STOP}
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def claim_overlap(post_text, claim_text):
|
| 59 |
+
"""Share of the fact-checked claim's words that also appear in the post (0 to 1)."""
|
| 60 |
+
claim = _words(claim_text)
|
| 61 |
+
return len(claim & _words(post_text)) / len(claim) if claim else 0.0
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def rating_to_verdict(rating):
|
| 65 |
+
r = rating.lower()
|
| 66 |
+
if any(k in r for k in ("missing context", "misleading", "half", "mixture", "partly", "out of context", "exaggerat")):
|
| 67 |
+
return "MISSING_CONTEXT"
|
| 68 |
+
if any(k in r for k in ("false", "pants on fire", "fake", "incorrect", "fabricated", "not true", "no evidence",
|
| 69 |
+
"baseless", "hoax", "scam", "wrong")):
|
| 70 |
+
return "CONTRADICTED"
|
| 71 |
+
if any(k in r for k in ("true", "correct", "accurate")):
|
| 72 |
+
return "SUPPORTED"
|
| 73 |
+
return "RATED"
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
def _queries(text, n=2):
|
| 77 |
+
# the longest sentences usually hold the actual claim
|
| 78 |
+
sentences = [s.strip() for s in re.split(r"(?<=[.!?])\s+", text) if len(s.strip()) > 25]
|
| 79 |
+
return sorted(sentences, key=len, reverse=True)[:n] or [text[:200]]
|
| 80 |
+
|
| 81 |
+
|
| 82 |
+
@register
|
| 83 |
+
class FactCheck(Feature):
|
| 84 |
+
name = "FactCheck"
|
| 85 |
+
description = "published fact-checks of the same claim (Google Fact Check Tools)"
|
| 86 |
+
needs_text = True
|
| 87 |
+
MATCH = 0.5 # at least half the fact-checked claim's words must be in the post
|
| 88 |
+
|
| 89 |
+
def wants(self, post):
|
| 90 |
+
return super().wants(post) and bool(os.environ.get("GOOGLE_FACTCHECK_KEY"))
|
| 91 |
+
|
| 92 |
+
def run(self, post, options):
|
| 93 |
+
import requests
|
| 94 |
+
seen, out = set(), []
|
| 95 |
+
for q in _queries(post.all_text):
|
| 96 |
+
r = requests.get("https://factchecktools.googleapis.com/v1alpha1/claims:search",
|
| 97 |
+
params={"query": q[:200], "key": os.environ["GOOGLE_FACTCHECK_KEY"],
|
| 98 |
+
"languageCode": "en", "pageSize": 5}, timeout=8)
|
| 99 |
+
for c in r.json().get("claims", []):
|
| 100 |
+
for rev in c.get("claimReview", [])[:1]:
|
| 101 |
+
url = rev.get("url", "")
|
| 102 |
+
if not url or url in seen:
|
| 103 |
+
continue
|
| 104 |
+
seen.add(url)
|
| 105 |
+
claim = c.get("text", "")
|
| 106 |
+
rating = rev.get("textualRating", "")
|
| 107 |
+
out.append({"publisher": rev.get("publisher", {}).get("name", "Fact-checker"),
|
| 108 |
+
"rating": rating, "title": rev.get("title", claim), "url": url, "claim": claim,
|
| 109 |
+
"verdict": rating_to_verdict(rating),
|
| 110 |
+
"match": claim_overlap(post.all_text, claim) >= self.MATCH})
|
| 111 |
+
out.sort(key=lambda x: not x["match"]) # same-claim results first
|
| 112 |
+
n = sum(x["match"] for x in out)
|
| 113 |
+
return {"factchecks": out[:4], "confidence": f"{n} same-claim fact-check(s), {len(out) - n} related"}
|
| 114 |
+
|
| 115 |
+
|
| 116 |
+
@register
|
| 117 |
+
class StatCheck(Feature):
|
| 118 |
+
name = "StatCheck"
|
| 119 |
+
description = "number tricks in the wording"
|
| 120 |
+
needs_text = True
|
| 121 |
+
|
| 122 |
+
def run(self, post, options):
|
| 123 |
+
return statcheck.analyze(post.all_text)
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
@register
|
| 127 |
+
class LangCheck(Feature):
|
| 128 |
+
name = "LangCheck"
|
| 129 |
+
description = "persuasion tricks in the language, plus source tracing"
|
| 130 |
+
needs_text = True
|
| 131 |
+
|
| 132 |
+
def run(self, post, options):
|
| 133 |
+
return langcheck.analyze(post.all_text, trace_sources=options.get("trace", True),
|
| 134 |
+
place=options.get("place"))
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
@register
|
| 138 |
+
class EmotionCheck(Feature):
|
| 139 |
+
name = "EmotionCheck"
|
| 140 |
+
description = "how strongly the post pushes anger, fear, or disgust"
|
| 141 |
+
needs_text = True
|
| 142 |
+
_clf = None
|
| 143 |
+
|
| 144 |
+
def run(self, post, options):
|
| 145 |
+
if EmotionCheck._clf is None:
|
| 146 |
+
from transformers import pipeline
|
| 147 |
+
EmotionCheck._clf = pipeline("text-classification",
|
| 148 |
+
model="j-hartmann/emotion-english-distilroberta-base", top_k=None)
|
| 149 |
+
scores = {d["label"]: d["score"] for d in EmotionCheck._clf(post.all_text[:2000], truncation=True)[0]}
|
| 150 |
+
push = scores.get("anger", 0) + scores.get("fear", 0) + scores.get("disgust", 0)
|
| 151 |
+
top = sorted(scores.items(), key=lambda kv: -kv[1])[:3]
|
| 152 |
+
meter = ", ".join(f"{k} {v:.0%}" for k, v in top)
|
| 153 |
+
findings = []
|
| 154 |
+
if push >= 0.6:
|
| 155 |
+
findings.append({"trick": "emotional_language",
|
| 156 |
+
"why": f"The wording leans hard on anger, fear, or disgust ({push:.0%})."})
|
| 157 |
+
return {"findings": findings, "notes": [f"Emotion meter: {meter}"], "confidence": f"emotion push {push:.0%}"}
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
# ---------- local places (opt-in location) ----------
|
| 161 |
+
|
| 162 |
+
# word in the post -> OpenStreetMap tag for that kind of place
|
| 163 |
+
PLACE_TYPES = {
|
| 164 |
+
"library": ("amenity", "library"), "school": ("amenity", "school"), "hospital": ("amenity", "hospital"),
|
| 165 |
+
"post office": ("amenity", "post_office"), "police": ("amenity", "police"),
|
| 166 |
+
"fire station": ("amenity", "fire_station"), "city hall": ("amenity", "townhall"),
|
| 167 |
+
"town hall": ("amenity", "townhall"), "pharmacy": ("amenity", "pharmacy"), "park": ("leisure", "park"),
|
| 168 |
+
"community center": ("amenity", "community_centre"), "dmv": ("amenity", "public_building"),
|
| 169 |
+
}
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
def place_words(text):
|
| 173 |
+
t = text.lower()
|
| 174 |
+
return [w for w in PLACE_TYPES if re.search(rf"\b{w}s?\b", t)]
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
def _miles(lat1, lon1, lat2, lon2):
|
| 178 |
+
p = math.pi / 180
|
| 179 |
+
a = (math.sin((lat2 - lat1) * p / 2) ** 2 +
|
| 180 |
+
math.cos(lat1 * p) * math.cos(lat2 * p) * math.sin((lon2 - lon1) * p / 2) ** 2)
|
| 181 |
+
return 7918 * math.asin(math.sqrt(a)) # Earth's diameter in miles
|
| 182 |
+
|
| 183 |
+
|
| 184 |
+
def nearest_place(word, lat, lon, radius_m=10000):
|
| 185 |
+
import requests
|
| 186 |
+
key, value = PLACE_TYPES[word]
|
| 187 |
+
q = (f'[out:json][timeout:10];(node["{key}"="{value}"](around:{radius_m},{lat},{lon});'
|
| 188 |
+
f'way["{key}"="{value}"](around:{radius_m},{lat},{lon}););out center tags 15;')
|
| 189 |
+
r = requests.post("https://overpass-api.de/api/interpreter", data={"data": q}, headers=HEADERS, timeout=15)
|
| 190 |
+
best = None
|
| 191 |
+
for el in r.json().get("elements", []):
|
| 192 |
+
tags = el.get("tags", {})
|
| 193 |
+
plat, plon = el.get("lat") or el.get("center", {}).get("lat"), el.get("lon") or el.get("center", {}).get("lon")
|
| 194 |
+
if not tags.get("name") or plat is None:
|
| 195 |
+
continue
|
| 196 |
+
d = _miles(lat, lon, plat, plon)
|
| 197 |
+
if best is None or d < best[0]:
|
| 198 |
+
best = (d, tags, plat, plon)
|
| 199 |
+
if best is None:
|
| 200 |
+
return None
|
| 201 |
+
d, tags, plat, plon = best
|
| 202 |
+
maps = f"https://www.google.com/maps/search/?api=1&query={plat},{plon}"
|
| 203 |
+
site = tags.get("website") or tags.get("contact:website")
|
| 204 |
+
phone = tags.get("phone") or tags.get("contact:phone")
|
| 205 |
+
hours = tags.get("opening_hours")
|
| 206 |
+
contacts, html_contacts = [f"[Map]({maps})"], [f'<a href="{escape(maps)}" target="_blank">Open in Maps</a>']
|
| 207 |
+
if site:
|
| 208 |
+
contacts.append(f"[Website]({site})")
|
| 209 |
+
html_contacts.append(f'<a href="{escape(site)}" target="_blank">Official website</a>')
|
| 210 |
+
if phone:
|
| 211 |
+
contacts.append(f"Phone: {phone}")
|
| 212 |
+
html_contacts.append(f"Phone: {escape(phone)}")
|
| 213 |
+
if hours:
|
| 214 |
+
contacts.append(f"Listed hours: {hours}")
|
| 215 |
+
html_contacts.append(f"Listed hours: {escape(hours)}")
|
| 216 |
+
return {"name": tags["name"], "distance": f"{d:.1f} miles away", "contacts": contacts, "contacts_html": html_contacts}
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def reverse_place(lat, lon):
|
| 220 |
+
"""City and state for a location, so source tracing can search local news. Rounded for privacy."""
|
| 221 |
+
import requests
|
| 222 |
+
try:
|
| 223 |
+
r = requests.get("https://nominatim.openstreetmap.org/reverse",
|
| 224 |
+
params={"format": "jsonv2", "lat": round(lat, 2), "lon": round(lon, 2), "zoom": 10},
|
| 225 |
+
headers=HEADERS, timeout=8)
|
| 226 |
+
a = r.json().get("address", {})
|
| 227 |
+
city = a.get("city") or a.get("town") or a.get("village") or a.get("county")
|
| 228 |
+
return ", ".join(x for x in (city, a.get("state")) if x) or None
|
| 229 |
+
except Exception:
|
| 230 |
+
return None
|
| 231 |
+
|
| 232 |
+
|
| 233 |
+
@register
|
| 234 |
+
class LocalCheck(Feature):
|
| 235 |
+
name = "LocalCheck"
|
| 236 |
+
description = "finds the real local place a post mentions so you can check with them (only with your location)"
|
| 237 |
+
needs_text = True
|
| 238 |
+
|
| 239 |
+
def wants(self, post):
|
| 240 |
+
return super().wants(post) and bool(place_words(post.all_text))
|
| 241 |
+
|
| 242 |
+
def run(self, post, options):
|
| 243 |
+
words = place_words(post.all_text)[:2]
|
| 244 |
+
loc = options.get("location")
|
| 245 |
+
if not loc:
|
| 246 |
+
return {"notes": [f"This post is about a local {words[0]}. Tap \"Use my location\" to find the nearest one "
|
| 247 |
+
"and check with them directly."], "confidence": "no location shared"}
|
| 248 |
+
lat, lon = round(loc[0], 3), round(loc[1], 3) # about 100 m, plenty for finding a library
|
| 249 |
+
places = [p for p in (nearest_place(w, lat, lon) for w in words) if p]
|
| 250 |
+
notes = [] if places else [f"Couldn't find a {words[0]} within about 6 miles."]
|
| 251 |
+
return {"places": places, "notes": notes, "confidence": f"{len(places)} nearby place(s)"}
|
| 252 |
+
|
| 253 |
+
|
| 254 |
+
# ---------- links ----------
|
| 255 |
+
|
| 256 |
+
SHORTENERS = {"bit.ly", "tinyurl.com", "t.co", "goo.gl", "ow.ly", "is.gd", "buff.ly", "rebrand.ly", "cutt.ly",
|
| 257 |
+
"shorturl.at", "tiny.cc", "rb.gy", "lnkd.in", "s.id"}
|
| 258 |
+
# brand word -> the domains it really uses
|
| 259 |
+
BRANDS = {"paypal": {"paypal.com"}, "amazon": {"amazon.com", "amazonaws.com", "amazon.co.uk"},
|
| 260 |
+
"apple": {"apple.com", "icloud.com"}, "google": {"google.com", "youtube.com", "goo.gl"},
|
| 261 |
+
"microsoft": {"microsoft.com", "live.com", "office.com"}, "netflix": {"netflix.com"},
|
| 262 |
+
"facebook": {"facebook.com", "fb.com"}, "instagram": {"instagram.com"},
|
| 263 |
+
"chase": {"chase.com"}, "wellsfargo": {"wellsfargo.com"}, "bankofamerica": {"bankofamerica.com"},
|
| 264 |
+
"usps": {"usps.com"}, "fedex": {"fedex.com"}, "ups": {"ups.com"}, "irs": {"irs.gov"},
|
| 265 |
+
"medicare": {"medicare.gov"}, "socialsecurity": {"ssa.gov"}, "cnn": {"cnn.com"},
|
| 266 |
+
"foxnews": {"foxnews.com"}, "nytimes": {"nytimes.com"}, "bbc": {"bbc.com", "bbc.co.uk"},
|
| 267 |
+
"abcnews": {"abcnews.go.com", "go.com"}, "nbcnews": {"nbcnews.com"}, "reuters": {"reuters.com"}}
|
| 268 |
+
TWO_PART_TLDS = {"co.uk", "com.au", "com.co", "co.in", "com.br", "co.jp", "go.com"}
|
| 269 |
+
LEET = str.maketrans({"0": "o", "1": "l", "3": "e", "4": "a", "5": "s", "7": "t"})
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
def registered_domain(host):
|
| 273 |
+
parts = host.lower().strip(".").split(".")
|
| 274 |
+
if len(parts) >= 3 and ".".join(parts[-2:]) in TWO_PART_TLDS:
|
| 275 |
+
return ".".join(parts[-3:])
|
| 276 |
+
return ".".join(parts[-2:])
|
| 277 |
+
|
| 278 |
+
|
| 279 |
+
def lookalike_brand(host):
|
| 280 |
+
"""The brand a domain is pretending to be, or None if it's the real one (or not a brand)."""
|
| 281 |
+
reg = registered_domain(host)
|
| 282 |
+
name = reg.split(".")[0].translate(LEET)
|
| 283 |
+
tokens = set(re.split(r"[-_]", name))
|
| 284 |
+
for brand, official in BRANDS.items():
|
| 285 |
+
if reg in official or any(host.endswith("." + o) or host == o for o in official):
|
| 286 |
+
continue
|
| 287 |
+
hit = brand in tokens if len(brand) < 5 else brand in name.replace("-", "")
|
| 288 |
+
if hit:
|
| 289 |
+
return brand
|
| 290 |
+
return None
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
def resolve_short(url):
|
| 294 |
+
"""Follow a short link to where it really goes, without loading the page."""
|
| 295 |
+
import requests
|
| 296 |
+
try:
|
| 297 |
+
return requests.head(url, allow_redirects=True, timeout=5, headers=HEADERS).url
|
| 298 |
+
except Exception:
|
| 299 |
+
return None
|
| 300 |
+
|
| 301 |
+
|
| 302 |
+
def domain_age_days(reg):
|
| 303 |
+
"""Days since the domain was registered, from public RDAP records."""
|
| 304 |
+
import datetime as dt
|
| 305 |
+
import requests
|
| 306 |
+
try:
|
| 307 |
+
data = requests.get(f"https://rdap.org/domain/{reg}", timeout=6, headers=HEADERS).json()
|
| 308 |
+
for ev in data.get("events", []):
|
| 309 |
+
if ev.get("eventAction") == "registration":
|
| 310 |
+
born = dt.datetime.fromisoformat(ev["eventDate"].replace("Z", "+00:00"))
|
| 311 |
+
return (dt.datetime.now(dt.timezone.utc) - born).days
|
| 312 |
+
except Exception:
|
| 313 |
+
pass
|
| 314 |
+
return None
|
| 315 |
+
|
| 316 |
+
|
| 317 |
+
@register
|
| 318 |
+
class LinkCheck(Feature):
|
| 319 |
+
name = "LinkCheck"
|
| 320 |
+
description = "where a link really goes, lookalike sites, and brand-new sites"
|
| 321 |
+
NEW_SITE_DAYS = 180
|
| 322 |
+
|
| 323 |
+
def wants(self, post):
|
| 324 |
+
return bool(post.links)
|
| 325 |
+
|
| 326 |
+
def run(self, post, options):
|
| 327 |
+
from urllib.parse import urlparse
|
| 328 |
+
findings, notes = [], []
|
| 329 |
+
for raw in post.links[:3]:
|
| 330 |
+
url = raw.rstrip(").,!?;:'\"")
|
| 331 |
+
if not url.lower().startswith("http"):
|
| 332 |
+
url = "https://" + url
|
| 333 |
+
host = (urlparse(url).hostname or "").lower()
|
| 334 |
+
if not host:
|
| 335 |
+
continue
|
| 336 |
+
if registered_domain(host) in SHORTENERS:
|
| 337 |
+
real = resolve_short(url)
|
| 338 |
+
real_host = (urlparse(real).hostname or "").lower() if real else ""
|
| 339 |
+
findings.append({"trick": "hidden_link", "quote": raw,
|
| 340 |
+
"why": f"This short link hides where it goes" +
|
| 341 |
+
(f". It really goes to {real_host}." if real_host else ".")})
|
| 342 |
+
if real_host:
|
| 343 |
+
host = real_host
|
| 344 |
+
if re.fullmatch(r"[0-9.]+", host) or "xn--" in host:
|
| 345 |
+
findings.append({"trick": "lookalike_link", "quote": raw,
|
| 346 |
+
"why": "The link uses a disguised address instead of a normal website name."})
|
| 347 |
+
brand = lookalike_brand(host)
|
| 348 |
+
if brand:
|
| 349 |
+
findings.append({"trick": "lookalike_link", "quote": raw,
|
| 350 |
+
"why": f"{host} looks like {brand}, but it isn't one of {brand}'s real websites."})
|
| 351 |
+
reg = registered_domain(host)
|
| 352 |
+
age = domain_age_days(reg)
|
| 353 |
+
if age is not None and age < self.NEW_SITE_DAYS:
|
| 354 |
+
findings.append({"trick": "new_site", "quote": raw,
|
| 355 |
+
"why": f"{reg} was registered only {age} days ago."})
|
| 356 |
+
notes.append(f"Link goes to {host}" + (f" (site registered about {age // 365} years ago)"
|
| 357 |
+
if age and age >= 365 else ""))
|
| 358 |
+
return {"findings": findings, "notes": notes, "confidence": f"{len(notes)} link(s) checked"}
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
# ---------- photos and memes ----------
|
| 362 |
+
|
| 363 |
+
IMAGE_SCHEMA = {
|
| 364 |
+
"type": "object",
|
| 365 |
+
"properties": {
|
| 366 |
+
"ai_signs": {"type": "array", "items": {"type": "string"}},
|
| 367 |
+
"ai_likelihood": {"type": "string", "enum": ["low", "medium", "high"]},
|
| 368 |
+
"caption_matches": {"type": "string", "enum": ["yes", "no", "unclear", "no caption"]},
|
| 369 |
+
"mismatch_reason": {"type": "string"},
|
| 370 |
+
"search_tip": {"type": "string"},
|
| 371 |
+
},
|
| 372 |
+
"required": ["ai_signs", "ai_likelihood", "caption_matches", "mismatch_reason", "search_tip"],
|
| 373 |
+
"additionalProperties": False,
|
| 374 |
+
}
|
| 375 |
+
|
| 376 |
+
IMAGE_PROMPT = """You help people check photos and memes before sharing them. Look at the image and its caption.
|
| 377 |
+
|
| 378 |
+
1. ai_signs: specific things you can SEE that suggest AI generation or editing (warped hands or text,
|
| 379 |
+
melted details, impossible lighting or reflections, smeared backgrounds). Only list what is visible.
|
| 380 |
+
Empty list if none. Never guess from the topic.
|
| 381 |
+
2. ai_likelihood: low, medium, or high, based only on those visible signs.
|
| 382 |
+
3. caption_matches: does the caption describe what the photo actually shows? Use "unclear" if you can't tell,
|
| 383 |
+
"no caption" if there is none. Judge only what's visible, not whether the event happened.
|
| 384 |
+
4. mismatch_reason: one plain sentence if it doesn't match, else empty.
|
| 385 |
+
5. search_tip: the best words to search to find where this photo really came from.
|
| 386 |
+
Same standard for every topic and political side.
|
| 387 |
+
|
| 388 |
+
CAPTION:
|
| 389 |
+
"""
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
@register
|
| 393 |
+
class ImageCheck(Feature):
|
| 394 |
+
name = "ImageCheck"
|
| 395 |
+
description = "signs a photo is AI-made, or doesn't match its caption"
|
| 396 |
+
kinds = {"photo"}
|
| 397 |
+
model = os.environ.get("TELL_VISION_MODEL", "claude-haiku-5-5")
|
| 398 |
+
|
| 399 |
+
def run(self, post, options):
|
| 400 |
+
import base64
|
| 401 |
+
import io
|
| 402 |
+
import json
|
| 403 |
+
tips = "Reverse-search the photo at images.google.com or tineye.com to find where it first appeared."
|
| 404 |
+
client = langcheck._client()
|
| 405 |
+
if client is None:
|
| 406 |
+
return {"notes": [tips], "confidence": "needs the Claude key for photo checks"}
|
| 407 |
+
img = post.image.convert("RGB")
|
| 408 |
+
img.thumbnail((1024, 1024))
|
| 409 |
+
buf = io.BytesIO()
|
| 410 |
+
img.save(buf, "JPEG", quality=85)
|
| 411 |
+
caption = post.all_text or "(none)"
|
| 412 |
+
resp = client.messages.create(
|
| 413 |
+
model=self.model, max_tokens=800,
|
| 414 |
+
messages=[{"role": "user", "content": [
|
| 415 |
+
{"type": "image", "source": {"type": "base64", "media_type": "image/jpeg",
|
| 416 |
+
"data": base64.b64encode(buf.getvalue()).decode()}},
|
| 417 |
+
{"type": "text", "text": IMAGE_PROMPT + caption[:1500]}]}],
|
| 418 |
+
output_config={"format": {"type": "json_schema", "schema": IMAGE_SCHEMA}},
|
| 419 |
+
)
|
| 420 |
+
data = json.loads(next(b.text for b in resp.content if b.type == "text"))
|
| 421 |
+
findings = []
|
| 422 |
+
# needs at least two visible signs, so one odd shadow doesn't get a photo called fake
|
| 423 |
+
if data["ai_likelihood"] == "high" and len(data["ai_signs"]) >= 2:
|
| 424 |
+
findings.append({"trick": "ai_image", "quote": "; ".join(data["ai_signs"][:3]),
|
| 425 |
+
"why": "This picture shows several signs of being made or changed by AI."})
|
| 426 |
+
if data["caption_matches"] == "no" and data["mismatch_reason"]:
|
| 427 |
+
findings.append({"trick": "out_of_context", "why": data["mismatch_reason"]})
|
| 428 |
+
notes = [tips]
|
| 429 |
+
if data["search_tip"]:
|
| 430 |
+
notes.append(f"Try searching: {data['search_tip']}")
|
| 431 |
+
return {"findings": findings, "notes": notes,
|
| 432 |
+
"confidence": f"AI signs {data['ai_likelihood']}, caption match {data['caption_matches']}"}
|
lessons/lessons.json
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"truncated_axis": {
|
| 3 |
+
"title": "The chopped axis",
|
| 4 |
+
"what": "The chart's y-axis starts way above zero, so a tiny gap looks like a cliff.",
|
| 5 |
+
"why_it_works": "Our eyes compare bar heights, not the numbers printed beside them.",
|
| 6 |
+
"spot_it": "Look at the bottom number on the y-axis. On a bar chart, it should be 0."
|
| 7 |
+
},
|
| 8 |
+
"inverted_axis": {
|
| 9 |
+
"title": "The upside-down axis",
|
| 10 |
+
"what": "The numbers grow going down, so a line that rises is really falling.",
|
| 11 |
+
"why_it_works": "We assume up means more without reading the labels.",
|
| 12 |
+
"spot_it": "Read the top and bottom axis numbers before reading the shape."
|
| 13 |
+
},
|
| 14 |
+
"uneven_ticks": {
|
| 15 |
+
"title": "The stretchy scale",
|
| 16 |
+
"what": "The steps on the axis aren't equal, so some parts of the chart are stretched.",
|
| 17 |
+
"why_it_works": "We assume every grid line means the same jump.",
|
| 18 |
+
"spot_it": "Check that the gaps between axis numbers are all the same size."
|
| 19 |
+
},
|
| 20 |
+
"screenshot_generated": {
|
| 21 |
+
"title": "The fake-chat generator",
|
| 22 |
+
"what": "Free websites can make a perfect-looking text or DM screenshot that never happened.",
|
| 23 |
+
"why_it_works": "Screenshots feel like proof because they look like real apps.",
|
| 24 |
+
"spot_it": "Ask for the original message or link, not a picture of it."
|
| 25 |
+
},
|
| 26 |
+
"screenshot_edited": {
|
| 27 |
+
"title": "The edited screenshot",
|
| 28 |
+
"what": "A real screenshot with one message or name changed.",
|
| 29 |
+
"why_it_works": "Most of it is real, so the one fake part borrows trust.",
|
| 30 |
+
"spot_it": "Zoom in: edited text often has slightly different sharpness, spacing, or alignment."
|
| 31 |
+
},
|
| 32 |
+
"relative_risk_no_baseline": {
|
| 33 |
+
"title": "\"Doubles your risk\"",
|
| 34 |
+
"what": "It tells you how much a risk changed, but not how big it was to start.",
|
| 35 |
+
"why_it_works": "Doubling sounds scary even when 1 in 10,000 becomes 2 in 10,000.",
|
| 36 |
+
"spot_it": "Ask: double of what? Look for the starting number."
|
| 37 |
+
},
|
| 38 |
+
"big_percent_small_base": {
|
| 39 |
+
"title": "The giant percent",
|
| 40 |
+
"what": "A huge percent increase from a tiny starting number.",
|
| 41 |
+
"why_it_works": "Going from 2 to 10 is a 400% jump, and 400% sounds like an emergency.",
|
| 42 |
+
"spot_it": "Find the actual before and after numbers."
|
| 43 |
+
},
|
| 44 |
+
"missing_denominator": {
|
| 45 |
+
"title": "The missing \"out of\"",
|
| 46 |
+
"what": "A raw count with no idea of how many people there were in total.",
|
| 47 |
+
"why_it_works": "Big numbers feel common, even when they're a tiny share.",
|
| 48 |
+
"spot_it": "Ask: out of how many?"
|
| 49 |
+
},
|
| 50 |
+
"correlation_as_cause": {
|
| 51 |
+
"title": "Linked is not caused",
|
| 52 |
+
"what": "A study found two things happen together, and the post says one causes the other.",
|
| 53 |
+
"why_it_works": "Our brains love simple cause-and-effect stories.",
|
| 54 |
+
"spot_it": "Look for words like 'causes' or 'makes you' next to the word 'study'."
|
| 55 |
+
},
|
| 56 |
+
"dual_axis": {
|
| 57 |
+
"title": "The two-ruler chart",
|
| 58 |
+
"what": "Two lines share one chart but use two different y-axes, so they look connected when they might not be.",
|
| 59 |
+
"why_it_works": "When two lines rise together on screen, we assume they move together in real life.",
|
| 60 |
+
"spot_it": "Check for numbers on BOTH sides of the chart. If there are two scales, the lines' crossing points mean nothing."
|
| 61 |
+
},
|
| 62 |
+
"emotional_language": {
|
| 63 |
+
"title": "The feelings button",
|
| 64 |
+
"what": "Words picked to make you scared, angry, or outraged.",
|
| 65 |
+
"why_it_works": "Strong emotions make us share fast and think later.",
|
| 66 |
+
"spot_it": "Notice how the post makes you feel. Then ask what the actual facts are without the loud words."
|
| 67 |
+
},
|
| 68 |
+
"false_dichotomy": {
|
| 69 |
+
"title": "Only two choices",
|
| 70 |
+
"what": "The post acts like there are only two options, so you have to pick its side.",
|
| 71 |
+
"why_it_works": "Two choices feel simple, and the other one is made to look terrible.",
|
| 72 |
+
"spot_it": "Ask: is there a third option, or a middle ground, that the post skipped?"
|
| 73 |
+
},
|
| 74 |
+
"scapegoating": {
|
| 75 |
+
"title": "The blame target",
|
| 76 |
+
"what": "One person or group gets blamed for a big, complicated problem.",
|
| 77 |
+
"why_it_works": "A clear villain is easier to understand than a messy real cause.",
|
| 78 |
+
"spot_it": "Ask: could one group really cause all of this? What else could be going on?"
|
| 79 |
+
},
|
| 80 |
+
"ad_hominem": {
|
| 81 |
+
"title": "Attack the person",
|
| 82 |
+
"what": "The post goes after who someone is instead of what they said.",
|
| 83 |
+
"why_it_works": "If we dislike the person, we stop checking whether their point is right.",
|
| 84 |
+
"spot_it": "Strip out the insult. Is there an argument left? Is it answered?"
|
| 85 |
+
},
|
| 86 |
+
"vague_sourcing": {
|
| 87 |
+
"title": "Who are 'experts'?",
|
| 88 |
+
"what": "The post leans on unnamed experts, studies, or 'people' to sound credible.",
|
| 89 |
+
"why_it_works": "'Experts say' sounds official even when no expert is named.",
|
| 90 |
+
"spot_it": "Look for a name, a study title, or a link. No name means you can't check it."
|
| 91 |
+
},
|
| 92 |
+
"false_urgency": {
|
| 93 |
+
"title": "Share before it's gone",
|
| 94 |
+
"what": "The post pressures you to act or share right now.",
|
| 95 |
+
"why_it_works": "Rushing stops you from checking, which is the point.",
|
| 96 |
+
"spot_it": "Real news doesn't vanish. If it says 'share before it's deleted', slow down."
|
| 97 |
+
},
|
| 98 |
+
"absolute_claims": {
|
| 99 |
+
"title": "Always and never",
|
| 100 |
+
"what": "Words like always, never, everyone, or 100% that are almost never literally true.",
|
| 101 |
+
"why_it_works": "Absolute words sound confident, and confidence sounds like truth.",
|
| 102 |
+
"spot_it": "Swap 'always' for 'sometimes' and see if the post still makes its point."
|
| 103 |
+
},
|
| 104 |
+
"us_vs_them": {
|
| 105 |
+
"title": "Us vs them",
|
| 106 |
+
"what": "The post splits people into a good 'us' and a bad 'them'.",
|
| 107 |
+
"why_it_works": "We trust our own group and distrust the other, so we stop judging the claim itself.",
|
| 108 |
+
"spot_it": "Ask who 'they' actually are, and whether every one of them really fits the label."
|
| 109 |
+
},
|
| 110 |
+
"hidden_link": {
|
| 111 |
+
"title": "The hidden link",
|
| 112 |
+
"what": "A short link hides which website it really goes to.",
|
| 113 |
+
"why_it_works": "You can't judge a site you can't see, so you click on trust.",
|
| 114 |
+
"spot_it": "Hold your finger on the link (or hover) to see where it goes before you tap."
|
| 115 |
+
},
|
| 116 |
+
"lookalike_link": {
|
| 117 |
+
"title": "The lookalike website",
|
| 118 |
+
"what": "The link looks like a real brand or news site, but the address is slightly off.",
|
| 119 |
+
"why_it_works": "We read the brand name and skip the rest of the address.",
|
| 120 |
+
"spot_it": "Read the part right before .com. paypa1-secure.com is not paypal.com."
|
| 121 |
+
},
|
| 122 |
+
"new_site": {
|
| 123 |
+
"title": "The brand-new website",
|
| 124 |
+
"what": "The website was created very recently.",
|
| 125 |
+
"why_it_works": "A nice-looking site feels established, even if it was made last week.",
|
| 126 |
+
"spot_it": "Be extra careful with sites you've never heard of, especially ones selling, scaring, or asking for info."
|
| 127 |
+
},
|
| 128 |
+
"ai_image": {
|
| 129 |
+
"title": "The AI-made picture",
|
| 130 |
+
"what": "The picture shows signs it was made or changed by AI.",
|
| 131 |
+
"why_it_works": "Photos feel like proof, and AI pictures now look real at a glance.",
|
| 132 |
+
"spot_it": "Zoom in on hands, text, ears, and backgrounds. Then reverse-search the image."
|
| 133 |
+
},
|
| 134 |
+
"out_of_context": {
|
| 135 |
+
"title": "The wrong caption",
|
| 136 |
+
"what": "The picture is real, but the caption says it shows something else.",
|
| 137 |
+
"why_it_works": "A real photo makes a false story feel true.",
|
| 138 |
+
"spot_it": "Reverse-search the photo to see when and where it first appeared."
|
| 139 |
+
}
|
| 140 |
+
}
|
notebooks/TELL_ChartCheck_Training.ipynb
ADDED
|
@@ -0,0 +1,777 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"metadata": {},
|
| 6 |
+
"source": [
|
| 7 |
+
"# TELL: ChartCheck training\n",
|
| 8 |
+
"\n",
|
| 9 |
+
"Trains the model that names misleading chart tricks.\n",
|
| 10 |
+
"\n",
|
| 11 |
+
"| Step | What happens | Time on a free T4 |\n",
|
| 12 |
+
"| --- | --- | --- |\n",
|
| 13 |
+
"| 1 | Generate 10,000 labeled synthetic charts (2,000 per class) | about 10 to 15 min |\n",
|
| 14 |
+
"| 2 | Fine-tune EfficientNet-B0 (pretrained on ImageNet) | about 25 to 40 min |\n",
|
| 15 |
+
"| 3 | Test on held-out synthetic charts | 1 min |\n",
|
| 16 |
+
"| 4 | Test on REAL charts from the Misviz benchmark | about 5 min |\n",
|
| 17 |
+
"| 5 | Upload the model to Hugging Face for the app | 1 min |\n",
|
| 18 |
+
"\n",
|
| 19 |
+
"**Before you start:** Runtime > Change runtime type > **T4 GPU**. Then request access to\n",
|
| 20 |
+
"[UKPLab/misviz](https://huggingface.co/datasets/UKPLab/misviz) on Hugging Face (step 4 needs it),\n",
|
| 21 |
+
"and make a Hugging Face **write** token at huggingface.co/settings/tokens.\n",
|
| 22 |
+
"\n",
|
| 23 |
+
"Classes: `none`, `truncated_axis`, `inverted_axis`, `uneven_ticks`, `dual_axis`."
|
| 24 |
+
]
|
| 25 |
+
},
|
| 26 |
+
{
|
| 27 |
+
"cell_type": "markdown",
|
| 28 |
+
"metadata": {},
|
| 29 |
+
"source": [
|
| 30 |
+
"## 0. Check the GPU and install libraries"
|
| 31 |
+
]
|
| 32 |
+
},
|
| 33 |
+
{
|
| 34 |
+
"cell_type": "code",
|
| 35 |
+
"metadata": {},
|
| 36 |
+
"execution_count": null,
|
| 37 |
+
"outputs": [],
|
| 38 |
+
"source": [
|
| 39 |
+
"!nvidia-smi --query-gpu=name,memory.total --format=csv\n",
|
| 40 |
+
"!pip -q install timm safetensors scikit-learn datasets huggingface_hub"
|
| 41 |
+
]
|
| 42 |
+
},
|
| 43 |
+
{
|
| 44 |
+
"cell_type": "markdown",
|
| 45 |
+
"metadata": {},
|
| 46 |
+
"source": [
|
| 47 |
+
"## 1. Save everything to Google Drive (Colab deletes local files when it disconnects)"
|
| 48 |
+
]
|
| 49 |
+
},
|
| 50 |
+
{
|
| 51 |
+
"cell_type": "code",
|
| 52 |
+
"metadata": {},
|
| 53 |
+
"execution_count": null,
|
| 54 |
+
"outputs": [],
|
| 55 |
+
"source": [
|
| 56 |
+
"from google.colab import drive\n",
|
| 57 |
+
"drive.mount('/content/drive')\n",
|
| 58 |
+
"import os\n",
|
| 59 |
+
"WORK = '/content/drive/MyDrive/TELL/chartcheck'\n",
|
| 60 |
+
"DATA = '/content/synth_charts' # images stay local (fast); results go to Drive\n",
|
| 61 |
+
"RUN = f'{WORK}/run1'\n",
|
| 62 |
+
"os.makedirs(WORK, exist_ok=True)\n",
|
| 63 |
+
"print('results will be saved in', RUN)"
|
| 64 |
+
]
|
| 65 |
+
},
|
| 66 |
+
{
|
| 67 |
+
"cell_type": "markdown",
|
| 68 |
+
"metadata": {},
|
| 69 |
+
"source": [
|
| 70 |
+
"## 2. Write the training code (same files as `training/` in the repo)"
|
| 71 |
+
]
|
| 72 |
+
},
|
| 73 |
+
{
|
| 74 |
+
"cell_type": "code",
|
| 75 |
+
"metadata": {},
|
| 76 |
+
"execution_count": null,
|
| 77 |
+
"outputs": [],
|
| 78 |
+
"source": [
|
| 79 |
+
"%%writefile chartgen.py\n",
|
| 80 |
+
"\"\"\"\n",
|
| 81 |
+
"TELL ChartCheck synthetic chart generator.\n",
|
| 82 |
+
"\n",
|
| 83 |
+
"Makes labeled charts with matplotlib so every label is known for sure.\n",
|
| 84 |
+
"Classes (names match the app's lesson keys):\n",
|
| 85 |
+
" none honest chart\n",
|
| 86 |
+
" truncated_axis BAR chart whose y-axis starts well above zero\n",
|
| 87 |
+
" inverted_axis y-axis flipped so bigger values sit lower\n",
|
| 88 |
+
" uneven_ticks equal visual spacing between ticks, unequal values (0,10,20,50,100)\n",
|
| 89 |
+
" dual_axis two series on two different y-axes\n",
|
| 90 |
+
"\n",
|
| 91 |
+
"Design notes (worth putting in your documentation):\n",
|
| 92 |
+
" - Truncated axis only applies to BAR charts. A line chart that doesn't start\n",
|
| 93 |
+
" at zero is normal and honest, so \"none\" includes zoomed-in line charts.\n",
|
| 94 |
+
" Otherwise the model would wrongly learn \"any axis not at zero is a trick\".\n",
|
| 95 |
+
" - Every chart is randomized (style, colors, size, fonts, titles, gridlines,\n",
|
| 96 |
+
" number of categories) so the model learns the trick, not one look.\n",
|
| 97 |
+
"\"\"\"\n",
|
| 98 |
+
"\n",
|
| 99 |
+
"import json\n",
|
| 100 |
+
"import random\n",
|
| 101 |
+
"from pathlib import Path\n",
|
| 102 |
+
"\n",
|
| 103 |
+
"import matplotlib\n",
|
| 104 |
+
"matplotlib.use(\"Agg\")\n",
|
| 105 |
+
"import matplotlib.pyplot as plt\n",
|
| 106 |
+
"import numpy as np\n",
|
| 107 |
+
"\n",
|
| 108 |
+
"CLASSES = [\"none\", \"truncated_axis\", \"inverted_axis\", \"uneven_ticks\", \"dual_axis\"]\n",
|
| 109 |
+
"\n",
|
| 110 |
+
"TITLES = [\"Monthly club members\", \"Cafeteria pizza sales\", \"Average commute time\",\n",
|
| 111 |
+
" \"Bike rentals per week\", \"Library visits\", \"Concert ticket prices\",\n",
|
| 112 |
+
" \"Rainfall by month\", \"Game downloads\", \"Team points per season\",\n",
|
| 113 |
+
" \"Snack prices\", \"Gym attendance\", \"Bus riders per day\", \"App ratings\",\n",
|
| 114 |
+
" \"Plant growth\", \"Website visitors\", \"Volunteer hours\"]\n",
|
| 115 |
+
"CAT_GROUPS = [\n",
|
| 116 |
+
" [\"Jan\", \"Feb\", \"Mar\", \"Apr\", \"May\", \"Jun\", \"Jul\", \"Aug\", \"Sep\", \"Oct\", \"Nov\", \"Dec\"],\n",
|
| 117 |
+
" [\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\"],\n",
|
| 118 |
+
" [\"Q1\", \"Q2\", \"Q3\", \"Q4\"],\n",
|
| 119 |
+
" [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\"],\n",
|
| 120 |
+
" [\"North\", \"South\", \"East\", \"West\", \"Central\"],\n",
|
| 121 |
+
" [\"2018\", \"2019\", \"2020\", \"2021\", \"2022\", \"2023\", \"2024\", \"2025\"],\n",
|
| 122 |
+
" [\"Team 1\", \"Team 2\", \"Team 3\", \"Team 4\", \"Team 5\", \"Team 6\"],\n",
|
| 123 |
+
"]\n",
|
| 124 |
+
"STYLES = [\"default\", \"ggplot\", \"seaborn-v0_8\", \"seaborn-v0_8-whitegrid\", \"bmh\",\n",
|
| 125 |
+
" \"fivethirtyeight\", \"classic\", \"grayscale\", \"seaborn-v0_8-pastel\"]\n",
|
| 126 |
+
"UNITS = [\"\", \"$\", \"%\", \" min\", \" k\"]\n",
|
| 127 |
+
"\n",
|
| 128 |
+
"\n",
|
| 129 |
+
"def _setup(rng):\n",
|
| 130 |
+
" plt.style.use(rng.choice(STYLES))\n",
|
| 131 |
+
" w = rng.uniform(4.5, 8.0)\n",
|
| 132 |
+
" h = rng.uniform(3.2, 6.0)\n",
|
| 133 |
+
" fig, ax = plt.subplots(figsize=(w, h), dpi=rng.choice([80, 100, 120]))\n",
|
| 134 |
+
" plt.rcParams[\"font.size\"] = rng.uniform(8, 13)\n",
|
| 135 |
+
" return fig, ax\n",
|
| 136 |
+
"\n",
|
| 137 |
+
"\n",
|
| 138 |
+
"def _categories(rng):\n",
|
| 139 |
+
" group = rng.choice(CAT_GROUPS)\n",
|
| 140 |
+
" n = rng.randint(3, min(8, len(group)))\n",
|
| 141 |
+
" start = rng.randint(0, len(group) - n)\n",
|
| 142 |
+
" return group[start:start + n]\n",
|
| 143 |
+
"\n",
|
| 144 |
+
"\n",
|
| 145 |
+
"def _decorate(ax, rng):\n",
|
| 146 |
+
" ax.set_title(rng.choice(TITLES), fontsize=rng.uniform(10, 15))\n",
|
| 147 |
+
" if rng.random() < 0.5:\n",
|
| 148 |
+
" ax.grid(axis=\"y\", alpha=rng.uniform(0.2, 0.7))\n",
|
| 149 |
+
" if rng.random() < 0.3:\n",
|
| 150 |
+
" ax.set_ylabel(rng.choice([\"Count\", \"Amount\", \"Total\", \"Score\", \"Value\"]))\n",
|
| 151 |
+
"\n",
|
| 152 |
+
"\n",
|
| 153 |
+
"def honest(rng, ax):\n",
|
| 154 |
+
" cats = _categories(rng)\n",
|
| 155 |
+
" if rng.random() < 0.55: # bar chart from zero\n",
|
| 156 |
+
" vals = [rng.uniform(5, 100) for _ in cats]\n",
|
| 157 |
+
" ax.bar(cats, vals, color=rng.choice([\"C0\", \"C1\", \"C2\", \"C3\", \"C4\"]))\n",
|
| 158 |
+
" ax.set_ylim(0, max(vals) * rng.uniform(1.05, 1.3))\n",
|
| 159 |
+
" else: # line chart: zoomed-in range is fine and honest\n",
|
| 160 |
+
" base = rng.uniform(20, 200)\n",
|
| 161 |
+
" vals = base + np.cumsum([rng.uniform(-5, 5) for _ in cats])\n",
|
| 162 |
+
" ax.plot(cats, vals, marker=rng.choice([\"o\", \"s\", None]), linewidth=rng.uniform(1.2, 3))\n",
|
| 163 |
+
" if rng.random() < 0.5:\n",
|
| 164 |
+
" ax.set_ylim(0, max(vals) * 1.2)\n",
|
| 165 |
+
"\n",
|
| 166 |
+
"\n",
|
| 167 |
+
"def truncated(rng, ax):\n",
|
| 168 |
+
" cats = _categories(rng)\n",
|
| 169 |
+
" base = rng.uniform(50, 500)\n",
|
| 170 |
+
" vals = [base * rng.uniform(0.92, 1.08) for _ in cats]\n",
|
| 171 |
+
" ax.bar(cats, vals, color=rng.choice([\"C0\", \"C1\", \"C2\", \"C3\", \"C4\"]))\n",
|
| 172 |
+
" low = min(vals) * rng.uniform(0.85, 0.98) # starts far above zero\n",
|
| 173 |
+
" ax.set_ylim(low, max(vals) * rng.uniform(1.005, 1.03))\n",
|
| 174 |
+
"\n",
|
| 175 |
+
"\n",
|
| 176 |
+
"def inverted(rng, ax):\n",
|
| 177 |
+
" if rng.random() < 0.5:\n",
|
| 178 |
+
" cats = _categories(rng)\n",
|
| 179 |
+
" vals = [rng.uniform(5, 100) for _ in cats]\n",
|
| 180 |
+
" ax.bar(cats, vals)\n",
|
| 181 |
+
" ax.set_ylim(0, max(vals) * 1.15)\n",
|
| 182 |
+
" else:\n",
|
| 183 |
+
" cats = _categories(rng)\n",
|
| 184 |
+
" vals = rng.uniform(20, 200) + np.cumsum([rng.uniform(-5, 8) for _ in cats])\n",
|
| 185 |
+
" ax.plot(cats, vals, marker=\"o\")\n",
|
| 186 |
+
" ax.invert_yaxis()\n",
|
| 187 |
+
"\n",
|
| 188 |
+
"\n",
|
| 189 |
+
"def uneven(rng, ax):\n",
|
| 190 |
+
" \"\"\"Ticks equally spaced on screen but with unequal value gaps.\"\"\"\n",
|
| 191 |
+
" tick_vals = sorted(rng.choice([[0, 10, 20, 50, 100], [0, 5, 10, 50, 200],\n",
|
| 192 |
+
" [0, 1, 5, 20, 100], [0, 25, 50, 100, 500],\n",
|
| 193 |
+
" [0, 2, 4, 10, 40, 100]]))\n",
|
| 194 |
+
" pos = list(range(len(tick_vals)))\n",
|
| 195 |
+
" to_pos = lambda v: np.interp(v, tick_vals, pos) # nonlinear squish\n",
|
| 196 |
+
" cats = _categories(rng)\n",
|
| 197 |
+
" vals = [rng.uniform(tick_vals[1], tick_vals[-1]) for _ in cats]\n",
|
| 198 |
+
" if rng.random() < 0.6:\n",
|
| 199 |
+
" ax.bar(cats, [to_pos(v) for v in vals])\n",
|
| 200 |
+
" else:\n",
|
| 201 |
+
" ax.plot(cats, [to_pos(v) for v in vals], marker=\"o\")\n",
|
| 202 |
+
" ax.set_yticks(pos)\n",
|
| 203 |
+
" unit = rng.choice(UNITS)\n",
|
| 204 |
+
" ax.set_yticklabels([f\"{unit}{v}\" if unit == \"$\" else f\"{v}{unit}\" for v in tick_vals])\n",
|
| 205 |
+
" ax.set_ylim(0, pos[-1] * 1.05)\n",
|
| 206 |
+
"\n",
|
| 207 |
+
"\n",
|
| 208 |
+
"def dual(rng, ax):\n",
|
| 209 |
+
" cats = _categories(rng)\n",
|
| 210 |
+
" a = [rng.uniform(10, 100) for _ in cats]\n",
|
| 211 |
+
" b = [rng.uniform(1000, 9000) for _ in cats]\n",
|
| 212 |
+
" if rng.random() < 0.5:\n",
|
| 213 |
+
" ax.bar(cats, a, color=\"C0\", alpha=0.8)\n",
|
| 214 |
+
" else:\n",
|
| 215 |
+
" ax.plot(cats, a, color=\"C0\", marker=\"o\")\n",
|
| 216 |
+
" ax2 = ax.twinx()\n",
|
| 217 |
+
" ax2.plot(cats, b, color=\"C3\", marker=\"s\")\n",
|
| 218 |
+
" ax.tick_params(axis=\"y\", colors=\"C0\")\n",
|
| 219 |
+
" ax2.tick_params(axis=\"y\", colors=\"C3\")\n",
|
| 220 |
+
"\n",
|
| 221 |
+
"\n",
|
| 222 |
+
"MAKERS = {\"none\": honest, \"truncated_axis\": truncated, \"inverted_axis\": inverted,\n",
|
| 223 |
+
" \"uneven_ticks\": uneven, \"dual_axis\": dual}\n",
|
| 224 |
+
"\n",
|
| 225 |
+
"\n",
|
| 226 |
+
"def make_chart(label, seed, out_path):\n",
|
| 227 |
+
" rng = random.Random(seed)\n",
|
| 228 |
+
" np.random.seed(seed % (2**32))\n",
|
| 229 |
+
" fig, ax = _setup(rng)\n",
|
| 230 |
+
" MAKERS[label](rng, ax)\n",
|
| 231 |
+
" _decorate(ax, rng)\n",
|
| 232 |
+
" fig.tight_layout()\n",
|
| 233 |
+
" fig.savefig(out_path)\n",
|
| 234 |
+
" plt.close(fig)\n",
|
| 235 |
+
" plt.style.use(\"default\")\n",
|
| 236 |
+
"\n",
|
| 237 |
+
"\n",
|
| 238 |
+
"def _job(args):\n",
|
| 239 |
+
" label, seed, out_path = args\n",
|
| 240 |
+
" try:\n",
|
| 241 |
+
" make_chart(label, seed, out_path)\n",
|
| 242 |
+
" return {\"path\": out_path.name, \"label\": label, \"seed\": seed}\n",
|
| 243 |
+
" except Exception as e:\n",
|
| 244 |
+
" return {\"error\": str(e), \"label\": label, \"seed\": seed}\n",
|
| 245 |
+
"\n",
|
| 246 |
+
"\n",
|
| 247 |
+
"def generate(out_dir, per_class=2000, start_seed=0, workers=4):\n",
|
| 248 |
+
" \"\"\"Make per_class images per class and a labels.json. Seeds make it reproducible.\"\"\"\n",
|
| 249 |
+
" from multiprocessing import Pool\n",
|
| 250 |
+
" out_dir = Path(out_dir)\n",
|
| 251 |
+
" (out_dir / \"img\").mkdir(parents=True, exist_ok=True)\n",
|
| 252 |
+
" jobs = []\n",
|
| 253 |
+
" seed = start_seed\n",
|
| 254 |
+
" for label in CLASSES:\n",
|
| 255 |
+
" for _ in range(per_class):\n",
|
| 256 |
+
" jobs.append((label, seed, out_dir / \"img\" / f\"{label}_{seed:06d}.png\"))\n",
|
| 257 |
+
" seed += 1\n",
|
| 258 |
+
" with Pool(workers) as pool:\n",
|
| 259 |
+
" rows = pool.map(_job, jobs, chunksize=32)\n",
|
| 260 |
+
" good = [r for r in rows if \"error\" not in r]\n",
|
| 261 |
+
" bad = [r for r in rows if \"error\" in r]\n",
|
| 262 |
+
" (out_dir / \"labels.json\").write_text(json.dumps(good))\n",
|
| 263 |
+
" print(f\"made {len(good)} charts, {len(bad)} failed\")\n",
|
| 264 |
+
" return good\n",
|
| 265 |
+
"\n",
|
| 266 |
+
"\n",
|
| 267 |
+
"if __name__ == \"__main__\":\n",
|
| 268 |
+
" import sys\n",
|
| 269 |
+
" generate(sys.argv[1] if len(sys.argv) > 1 else \"synth_charts\",\n",
|
| 270 |
+
" per_class=int(sys.argv[2]) if len(sys.argv) > 2 else 20)"
|
| 271 |
+
]
|
| 272 |
+
},
|
| 273 |
+
{
|
| 274 |
+
"cell_type": "code",
|
| 275 |
+
"metadata": {},
|
| 276 |
+
"execution_count": null,
|
| 277 |
+
"outputs": [],
|
| 278 |
+
"source": [
|
| 279 |
+
"%%writefile chart_train.py\n",
|
| 280 |
+
"\"\"\"\n",
|
| 281 |
+
"TELL ChartCheck training: fine-tunes a pretrained image model to name chart tricks.\n",
|
| 282 |
+
"\n",
|
| 283 |
+
"Pipeline:\n",
|
| 284 |
+
" synthetic charts (chartgen.py) -> split -> train EfficientNet-B0 -> pick best epoch\n",
|
| 285 |
+
" on validation macro-F1 -> test on held-out synthetic AND on real Misviz charts.\n",
|
| 286 |
+
"\n",
|
| 287 |
+
"Key decisions (document these):\n",
|
| 288 |
+
" - NO horizontal or vertical flips. A vertical flip literally creates an\n",
|
| 289 |
+
" \"inverted axis\", and a horizontal flip moves the y-axis to the wrong side.\n",
|
| 290 |
+
" Normal image augmentation would teach the model wrong labels.\n",
|
| 291 |
+
" - Pad to square, then resize. Squashing a wide chart distorts text and bars.\n",
|
| 292 |
+
" - Mild augmentations that match real life: JPEG compression (screenshots get\n",
|
| 293 |
+
" recompressed on social media), small scale changes, color jitter, blur.\n",
|
| 294 |
+
" - Model picked by validation macro-F1 (treats every trick equally), not accuracy.\n",
|
| 295 |
+
"\"\"\"\n",
|
| 296 |
+
"\n",
|
| 297 |
+
"import io\n",
|
| 298 |
+
"import json\n",
|
| 299 |
+
"import random\n",
|
| 300 |
+
"from pathlib import Path\n",
|
| 301 |
+
"\n",
|
| 302 |
+
"import numpy as np\n",
|
| 303 |
+
"import torch\n",
|
| 304 |
+
"import torch.nn as nn\n",
|
| 305 |
+
"from PIL import Image, ImageFilter\n",
|
| 306 |
+
"from sklearn.metrics import classification_report, confusion_matrix, f1_score\n",
|
| 307 |
+
"\n",
|
| 308 |
+
"CLASSES = [\"none\", \"truncated_axis\", \"inverted_axis\", \"uneven_ticks\", \"dual_axis\"]\n",
|
| 309 |
+
"IMG_SIZE = 384\n",
|
| 310 |
+
"MEAN = (0.485, 0.456, 0.406)\n",
|
| 311 |
+
"STD = (0.229, 0.224, 0.225)\n",
|
| 312 |
+
"\n",
|
| 313 |
+
"\n",
|
| 314 |
+
"# ---------- Preprocessing (MUST match tools/chartcheck.py in the app) ----------\n",
|
| 315 |
+
"\n",
|
| 316 |
+
"def pad_resize(img, size=IMG_SIZE):\n",
|
| 317 |
+
" img = img.convert(\"RGB\")\n",
|
| 318 |
+
" w, h = img.size\n",
|
| 319 |
+
" s = max(w, h)\n",
|
| 320 |
+
" canvas = Image.new(\"RGB\", (s, s), (255, 255, 255))\n",
|
| 321 |
+
" canvas.paste(img, ((s - w) // 2, (s - h) // 2))\n",
|
| 322 |
+
" return canvas.resize((size, size), Image.BILINEAR)\n",
|
| 323 |
+
"\n",
|
| 324 |
+
"\n",
|
| 325 |
+
"def to_tensor(img):\n",
|
| 326 |
+
" a = np.asarray(img, dtype=np.float32) / 255.0\n",
|
| 327 |
+
" a = (a - MEAN) / STD\n",
|
| 328 |
+
" return torch.from_numpy(a.transpose(2, 0, 1)).float()\n",
|
| 329 |
+
"\n",
|
| 330 |
+
"\n",
|
| 331 |
+
"def augment(img, rng):\n",
|
| 332 |
+
" \"\"\"Realistic, label-safe augmentations. No flips, no rotations.\"\"\"\n",
|
| 333 |
+
" if rng.random() < 0.5: # social-media JPEG recompression\n",
|
| 334 |
+
" buf = io.BytesIO()\n",
|
| 335 |
+
" img.convert(\"RGB\").save(buf, \"JPEG\", quality=rng.randint(45, 95))\n",
|
| 336 |
+
" buf.seek(0)\n",
|
| 337 |
+
" img = Image.open(buf).convert(\"RGB\")\n",
|
| 338 |
+
" if rng.random() < 0.4: # screenshot at a different size\n",
|
| 339 |
+
" f = rng.uniform(0.6, 1.0)\n",
|
| 340 |
+
" w, h = img.size\n",
|
| 341 |
+
" img = img.resize((max(64, int(w * f)), max(64, int(h * f)))).resize((w, h))\n",
|
| 342 |
+
" if rng.random() < 0.2:\n",
|
| 343 |
+
" img = img.filter(ImageFilter.GaussianBlur(rng.uniform(0.3, 1.0)))\n",
|
| 344 |
+
" if rng.random() < 0.3: # slight brightness/contrast change\n",
|
| 345 |
+
" a = np.asarray(img).astype(np.float32)\n",
|
| 346 |
+
" a = np.clip(a * rng.uniform(0.85, 1.15) + rng.uniform(-15, 15), 0, 255)\n",
|
| 347 |
+
" img = Image.fromarray(a.astype(np.uint8))\n",
|
| 348 |
+
" return img\n",
|
| 349 |
+
"\n",
|
| 350 |
+
"\n",
|
| 351 |
+
"# ---------- Data ----------\n",
|
| 352 |
+
"\n",
|
| 353 |
+
"class ChartDataset(torch.utils.data.Dataset):\n",
|
| 354 |
+
" def __init__(self, rows, img_dir, train=False):\n",
|
| 355 |
+
" self.rows, self.img_dir, self.train = rows, Path(img_dir), train\n",
|
| 356 |
+
"\n",
|
| 357 |
+
" def __len__(self):\n",
|
| 358 |
+
" return len(self.rows)\n",
|
| 359 |
+
"\n",
|
| 360 |
+
" def __getitem__(self, i):\n",
|
| 361 |
+
" r = self.rows[i]\n",
|
| 362 |
+
" img = Image.open(self.img_dir / r[\"path\"]).convert(\"RGB\")\n",
|
| 363 |
+
" if self.train:\n",
|
| 364 |
+
" img = augment(img, random.Random())\n",
|
| 365 |
+
" return to_tensor(pad_resize(img)), CLASSES.index(r[\"label\"])\n",
|
| 366 |
+
"\n",
|
| 367 |
+
"\n",
|
| 368 |
+
"def split_rows(rows, seed=42, val=0.1, test=0.1):\n",
|
| 369 |
+
" \"\"\"Stratified split: same class mix in train, val, and test.\"\"\"\n",
|
| 370 |
+
" rng = random.Random(seed)\n",
|
| 371 |
+
" out = {\"train\": [], \"val\": [], \"test\": []}\n",
|
| 372 |
+
" for c in CLASSES:\n",
|
| 373 |
+
" cr = [r for r in rows if r[\"label\"] == c]\n",
|
| 374 |
+
" rng.shuffle(cr)\n",
|
| 375 |
+
" nv, nt = int(len(cr) * val), int(len(cr) * test)\n",
|
| 376 |
+
" out[\"val\"] += cr[:nv]\n",
|
| 377 |
+
" out[\"test\"] += cr[nv:nv + nt]\n",
|
| 378 |
+
" out[\"train\"] += cr[nv + nt:]\n",
|
| 379 |
+
" return out\n",
|
| 380 |
+
"\n",
|
| 381 |
+
"\n",
|
| 382 |
+
"# ---------- Model ----------\n",
|
| 383 |
+
"\n",
|
| 384 |
+
"def build_model(arch=\"efficientnet_b0\", pretrained=True):\n",
|
| 385 |
+
" import timm\n",
|
| 386 |
+
" return timm.create_model(arch, pretrained=pretrained, num_classes=len(CLASSES))\n",
|
| 387 |
+
"\n",
|
| 388 |
+
"\n",
|
| 389 |
+
"@torch.no_grad()\n",
|
| 390 |
+
"def predict(model, loader, device):\n",
|
| 391 |
+
" model.eval()\n",
|
| 392 |
+
" ys, ps = [], []\n",
|
| 393 |
+
" for x, y in loader:\n",
|
| 394 |
+
" logits = model(x.to(device))\n",
|
| 395 |
+
" ps.append(torch.softmax(logits.float(), 1).cpu())\n",
|
| 396 |
+
" ys.append(y)\n",
|
| 397 |
+
" return torch.cat(ys).numpy(), torch.cat(ps).numpy()\n",
|
| 398 |
+
"\n",
|
| 399 |
+
"\n",
|
| 400 |
+
"def train(data_dir, out_dir, arch=\"efficientnet_b0\", epochs=6, batch=32, lr=3e-4,\n",
|
| 401 |
+
" workers=2, limit=None, seed=42, pretrained=True):\n",
|
| 402 |
+
" torch.manual_seed(seed)\n",
|
| 403 |
+
" random.seed(seed)\n",
|
| 404 |
+
" data_dir, out_dir = Path(data_dir), Path(out_dir)\n",
|
| 405 |
+
" out_dir.mkdir(parents=True, exist_ok=True)\n",
|
| 406 |
+
" rows = json.loads((data_dir / \"labels.json\").read_text())\n",
|
| 407 |
+
" if limit:\n",
|
| 408 |
+
" rows = random.Random(seed).sample(rows, min(limit, len(rows)))\n",
|
| 409 |
+
" splits = split_rows(rows, seed)\n",
|
| 410 |
+
" (out_dir / \"splits.json\").write_text(json.dumps(splits))\n",
|
| 411 |
+
" print({k: len(v) for k, v in splits.items()})\n",
|
| 412 |
+
"\n",
|
| 413 |
+
" device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
|
| 414 |
+
" mk = lambda part, tr: torch.utils.data.DataLoader(\n",
|
| 415 |
+
" ChartDataset(splits[part], data_dir / \"img\", train=tr),\n",
|
| 416 |
+
" batch_size=batch, shuffle=tr, num_workers=workers)\n",
|
| 417 |
+
" dl_train, dl_val, dl_test = mk(\"train\", True), mk(\"val\", False), mk(\"test\", False)\n",
|
| 418 |
+
"\n",
|
| 419 |
+
" model = build_model(arch, pretrained=pretrained).to(device)\n",
|
| 420 |
+
" opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)\n",
|
| 421 |
+
" steps = epochs * len(dl_train)\n",
|
| 422 |
+
" sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.1)\n",
|
| 423 |
+
" scaler = torch.amp.GradScaler(enabled=device == \"cuda\")\n",
|
| 424 |
+
" loss_fn = nn.CrossEntropyLoss(label_smoothing=0.05)\n",
|
| 425 |
+
"\n",
|
| 426 |
+
" best_f1, history = -1, []\n",
|
| 427 |
+
" for ep in range(epochs):\n",
|
| 428 |
+
" model.train()\n",
|
| 429 |
+
" total = 0.0\n",
|
| 430 |
+
" for x, y in dl_train:\n",
|
| 431 |
+
" x, y = x.to(device), y.to(device)\n",
|
| 432 |
+
" with torch.autocast(device_type=device, enabled=device == \"cuda\"):\n",
|
| 433 |
+
" loss = loss_fn(model(x), y)\n",
|
| 434 |
+
" opt.zero_grad()\n",
|
| 435 |
+
" scaler.scale(loss).backward()\n",
|
| 436 |
+
" scaler.step(opt)\n",
|
| 437 |
+
" scaler.update()\n",
|
| 438 |
+
" sched.step()\n",
|
| 439 |
+
" total += loss.item() * len(y)\n",
|
| 440 |
+
" yv, pv = predict(model, dl_val, device)\n",
|
| 441 |
+
" f1 = f1_score(yv, pv.argmax(1), average=\"macro\")\n",
|
| 442 |
+
" history.append({\"epoch\": ep + 1, \"train_loss\": total / len(dl_train.dataset), \"val_macro_f1\": f1})\n",
|
| 443 |
+
" print(history[-1])\n",
|
| 444 |
+
" if f1 > best_f1:\n",
|
| 445 |
+
" best_f1 = f1\n",
|
| 446 |
+
" torch.save(model.state_dict(), out_dir / \"best.pt\")\n",
|
| 447 |
+
"\n",
|
| 448 |
+
" model.load_state_dict(torch.load(out_dir / \"best.pt\", map_location=device))\n",
|
| 449 |
+
" yt, pt = predict(model, dl_test, device)\n",
|
| 450 |
+
" report = classification_report(yt, pt.argmax(1), labels=list(range(len(CLASSES))),\n",
|
| 451 |
+
" target_names=CLASSES, output_dict=True, zero_division=0)\n",
|
| 452 |
+
" cm = confusion_matrix(yt, pt.argmax(1), labels=list(range(len(CLASSES)))).tolist()\n",
|
| 453 |
+
" results = {\"history\": history, \"best_val_macro_f1\": best_f1,\n",
|
| 454 |
+
" \"synthetic_test\": report, \"synthetic_confusion\": cm}\n",
|
| 455 |
+
" (out_dir / \"results.json\").write_text(json.dumps(results, indent=2))\n",
|
| 456 |
+
" print(classification_report(yt, pt.argmax(1), labels=list(range(len(CLASSES))),\n",
|
| 457 |
+
" target_names=CLASSES, zero_division=0))\n",
|
| 458 |
+
" save_for_app(model, out_dir, arch)\n",
|
| 459 |
+
" return model, results\n",
|
| 460 |
+
"\n",
|
| 461 |
+
"\n",
|
| 462 |
+
"def save_for_app(model, out_dir, arch):\n",
|
| 463 |
+
" \"\"\"Writes the two files the app downloads: model.safetensors + config.json.\"\"\"\n",
|
| 464 |
+
" from safetensors.torch import save_file\n",
|
| 465 |
+
" out_dir = Path(out_dir)\n",
|
| 466 |
+
" save_file({k: v.contiguous().cpu() for k, v in model.state_dict().items()},\n",
|
| 467 |
+
" out_dir / \"model.safetensors\")\n",
|
| 468 |
+
" (out_dir / \"config.json\").write_text(json.dumps({\n",
|
| 469 |
+
" \"arch\": arch, \"classes\": CLASSES, \"img_size\": IMG_SIZE,\n",
|
| 470 |
+
" \"mean\": MEAN, \"std\": STD, \"threshold\": 0.5}, indent=2))\n",
|
| 471 |
+
"\n",
|
| 472 |
+
"\n",
|
| 473 |
+
"# ---------- Real-world test on Misviz ----------\n",
|
| 474 |
+
"\n",
|
| 475 |
+
"MISVIZ_MAP = {\"truncated axis\": \"truncated_axis\", \"inverted axis\": \"inverted_axis\",\n",
|
| 476 |
+
" \"inconsistent tick intervals\": \"uneven_ticks\", \"dual axis\": \"dual_axis\"}\n",
|
| 477 |
+
"\n",
|
| 478 |
+
"\n",
|
| 479 |
+
"def eval_misviz(model, hf_rows, device=None):\n",
|
| 480 |
+
" \"\"\"\n",
|
| 481 |
+
" hf_rows: Misviz rows with 'image' (PIL) and 'misleader' (list of names).\n",
|
| 482 |
+
" Only keeps charts whose tricks are ones TELL knows, or that have none.\n",
|
| 483 |
+
" A chart counts as caught if TELL's top prediction is one of its true tricks.\n",
|
| 484 |
+
" \"\"\"\n",
|
| 485 |
+
" device = device or (\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
|
| 486 |
+
" model.eval().to(device)\n",
|
| 487 |
+
" kept, correct, per = 0, 0, {c: [0, 0] for c in CLASSES}\n",
|
| 488 |
+
" preds_true = []\n",
|
| 489 |
+
" for r in hf_rows:\n",
|
| 490 |
+
" mis = r[\"misleader\"] or []\n",
|
| 491 |
+
" mapped = {MISVIZ_MAP[m] for m in mis if m in MISVIZ_MAP}\n",
|
| 492 |
+
" if mis and len(mapped) < len(mis):\n",
|
| 493 |
+
" continue # has a trick TELL doesn't cover yet; skip, don't count\n",
|
| 494 |
+
" truth = mapped or {\"none\"}\n",
|
| 495 |
+
" x = to_tensor(pad_resize(r[\"image\"])).unsqueeze(0).to(device)\n",
|
| 496 |
+
" with torch.no_grad():\n",
|
| 497 |
+
" pred = CLASSES[model(x).argmax(1).item()]\n",
|
| 498 |
+
" kept += 1\n",
|
| 499 |
+
" hit = pred in truth\n",
|
| 500 |
+
" correct += hit\n",
|
| 501 |
+
" for t in truth:\n",
|
| 502 |
+
" per[t][0] += hit\n",
|
| 503 |
+
" per[t][1] += 1\n",
|
| 504 |
+
" preds_true.append((pred, sorted(truth)))\n",
|
| 505 |
+
" return {\"charts_scored\": kept,\n",
|
| 506 |
+
" \"accuracy\": correct / max(kept, 1),\n",
|
| 507 |
+
" \"per_class_recall\": {c: (h / n if n else None) for c, (h, n) in per.items()},\n",
|
| 508 |
+
" \"per_class_count\": {c: n for c, (h, n) in per.items()},\n",
|
| 509 |
+
" \"examples\": preds_true[:50]}"
|
| 510 |
+
]
|
| 511 |
+
},
|
| 512 |
+
{
|
| 513 |
+
"cell_type": "markdown",
|
| 514 |
+
"metadata": {},
|
| 515 |
+
"source": [
|
| 516 |
+
"## 3. Generate the synthetic dataset\n",
|
| 517 |
+
"\n",
|
| 518 |
+
"Every chart is made from a known random seed, so the dataset is exactly reproducible.\n",
|
| 519 |
+
"Truncated axis is only drawn on **bar** charts: a zoomed-in **line** chart is normal, so those count as `none`."
|
| 520 |
+
]
|
| 521 |
+
},
|
| 522 |
+
{
|
| 523 |
+
"cell_type": "code",
|
| 524 |
+
"metadata": {},
|
| 525 |
+
"execution_count": null,
|
| 526 |
+
"outputs": [],
|
| 527 |
+
"source": [
|
| 528 |
+
"import chartgen, os\n",
|
| 529 |
+
"rows = chartgen.generate(DATA, per_class=2000, start_seed=0, workers=os.cpu_count())\n",
|
| 530 |
+
"import collections; print(collections.Counter(r['label'] for r in rows))"
|
| 531 |
+
]
|
| 532 |
+
},
|
| 533 |
+
{
|
| 534 |
+
"cell_type": "markdown",
|
| 535 |
+
"metadata": {},
|
| 536 |
+
"source": [
|
| 537 |
+
"### Look at samples from every class before training (catches generator bugs early)"
|
| 538 |
+
]
|
| 539 |
+
},
|
| 540 |
+
{
|
| 541 |
+
"cell_type": "code",
|
| 542 |
+
"metadata": {},
|
| 543 |
+
"execution_count": null,
|
| 544 |
+
"outputs": [],
|
| 545 |
+
"source": [
|
| 546 |
+
"import glob, matplotlib.pyplot as plt\n",
|
| 547 |
+
"from PIL import Image\n",
|
| 548 |
+
"fig, axes = plt.subplots(5, 4, figsize=(16, 18))\n",
|
| 549 |
+
"for r, c in enumerate(chartgen.CLASSES):\n",
|
| 550 |
+
" for k, f in enumerate(sorted(glob.glob(f'{DATA}/img/{c}_*.png'))[:4]):\n",
|
| 551 |
+
" axes[r][k].imshow(Image.open(f)); axes[r][k].axis('off'); axes[r][k].set_title(c)\n",
|
| 552 |
+
"plt.tight_layout(); plt.savefig(f'{WORK}/sample_grid.png', dpi=80); plt.show()"
|
| 553 |
+
]
|
| 554 |
+
},
|
| 555 |
+
{
|
| 556 |
+
"cell_type": "markdown",
|
| 557 |
+
"metadata": {},
|
| 558 |
+
"source": [
|
| 559 |
+
"## 4. Train\n",
|
| 560 |
+
"\n",
|
| 561 |
+
"- Pretrained EfficientNet-B0, input 384x384 (charts padded to square, never squashed)\n",
|
| 562 |
+
"- AdamW, learning rate 3e-4 with one-cycle schedule, label smoothing 0.05, mixed precision\n",
|
| 563 |
+
"- **No flips**: a vertical flip would literally create an inverted axis\n",
|
| 564 |
+
"- Best epoch chosen by validation macro-F1"
|
| 565 |
+
]
|
| 566 |
+
},
|
| 567 |
+
{
|
| 568 |
+
"cell_type": "code",
|
| 569 |
+
"metadata": {},
|
| 570 |
+
"execution_count": null,
|
| 571 |
+
"outputs": [],
|
| 572 |
+
"source": [
|
| 573 |
+
"import chart_train as ct\n",
|
| 574 |
+
"model, results = ct.train(DATA, RUN, arch='efficientnet_b0', epochs=6, batch=32, lr=3e-4, workers=2)"
|
| 575 |
+
]
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"cell_type": "markdown",
|
| 579 |
+
"metadata": {},
|
| 580 |
+
"source": [
|
| 581 |
+
"## 5. Training curves and confusion matrix (screenshot these for your documentation)"
|
| 582 |
+
]
|
| 583 |
+
},
|
| 584 |
+
{
|
| 585 |
+
"cell_type": "code",
|
| 586 |
+
"metadata": {},
|
| 587 |
+
"execution_count": null,
|
| 588 |
+
"outputs": [],
|
| 589 |
+
"source": [
|
| 590 |
+
"import json, numpy as np, matplotlib.pyplot as plt\n",
|
| 591 |
+
"res = json.load(open(f'{RUN}/results.json'))\n",
|
| 592 |
+
"h = res['history']\n",
|
| 593 |
+
"fig, ax = plt.subplots(1, 2, figsize=(13, 4))\n",
|
| 594 |
+
"ax[0].plot([x['epoch'] for x in h], [x['train_loss'] for x in h], marker='o'); ax[0].set_title('Training loss'); ax[0].set_xlabel('epoch')\n",
|
| 595 |
+
"ax[1].plot([x['epoch'] for x in h], [x['val_macro_f1'] for x in h], marker='o'); ax[1].set_title('Validation macro-F1'); ax[1].set_xlabel('epoch')\n",
|
| 596 |
+
"plt.savefig(f'{RUN}/curves.png', dpi=120); plt.show()\n",
|
| 597 |
+
"\n",
|
| 598 |
+
"cm = np.array(res['synthetic_confusion'])\n",
|
| 599 |
+
"plt.figure(figsize=(6, 5)); plt.imshow(cm, cmap='Blues')\n",
|
| 600 |
+
"plt.xticks(range(5), ct.CLASSES, rotation=40, ha='right'); plt.yticks(range(5), ct.CLASSES)\n",
|
| 601 |
+
"for i in range(5):\n",
|
| 602 |
+
" for j in range(5): plt.text(j, i, cm[i, j], ha='center', va='center')\n",
|
| 603 |
+
"plt.xlabel('predicted'); plt.ylabel('true'); plt.title('Synthetic test set')\n",
|
| 604 |
+
"plt.tight_layout(); plt.savefig(f'{RUN}/confusion_synthetic.png', dpi=120); plt.show()"
|
| 605 |
+
]
|
| 606 |
+
},
|
| 607 |
+
{
|
| 608 |
+
"cell_type": "markdown",
|
| 609 |
+
"metadata": {},
|
| 610 |
+
"source": [
|
| 611 |
+
"## 6. The real test: Misviz (real charts from the web)\n",
|
| 612 |
+
"\n",
|
| 613 |
+
"Synthetic accuracy is easy to get high. This step shows whether the model works on\n",
|
| 614 |
+
"**real** charts it has never seen. Only charts whose tricks TELL covers (or that have no trick)\n",
|
| 615 |
+
"are scored; charts with other tricks (3D, pie chart misuse, etc.) are skipped and counted.\n",
|
| 616 |
+
"\n",
|
| 617 |
+
"Misviz annotations are CC-BY-SA 4.0 and the dataset is for academic research; we only use it to\n",
|
| 618 |
+
"**test**, never to train, and we don't redistribute any images."
|
| 619 |
+
]
|
| 620 |
+
},
|
| 621 |
+
{
|
| 622 |
+
"cell_type": "code",
|
| 623 |
+
"metadata": {},
|
| 624 |
+
"execution_count": null,
|
| 625 |
+
"outputs": [],
|
| 626 |
+
"source": [
|
| 627 |
+
"from huggingface_hub import notebook_login\n",
|
| 628 |
+
"notebook_login() # paste your Hugging Face token"
|
| 629 |
+
]
|
| 630 |
+
},
|
| 631 |
+
{
|
| 632 |
+
"cell_type": "code",
|
| 633 |
+
"metadata": {},
|
| 634 |
+
"execution_count": null,
|
| 635 |
+
"outputs": [],
|
| 636 |
+
"source": [
|
| 637 |
+
"from datasets import load_dataset\n",
|
| 638 |
+
"mv = load_dataset('UKPLab/misviz', token=True)\n",
|
| 639 |
+
"print(mv)\n",
|
| 640 |
+
"print(mv['test'].features)"
|
| 641 |
+
]
|
| 642 |
+
},
|
| 643 |
+
{
|
| 644 |
+
"cell_type": "code",
|
| 645 |
+
"metadata": {},
|
| 646 |
+
"execution_count": null,
|
| 647 |
+
"outputs": [],
|
| 648 |
+
"source": [
|
| 649 |
+
"import torch\n",
|
| 650 |
+
"device = 'cuda'\n",
|
| 651 |
+
"real = ct.eval_misviz(model.to(device), mv['test'], device)\n",
|
| 652 |
+
"print('charts scored:', real['charts_scored'])\n",
|
| 653 |
+
"print('accuracy on real charts:', round(real['accuracy'], 3))\n",
|
| 654 |
+
"print('recall per class:', {k: (round(v, 3) if v is not None else None) for k, v in real['per_class_recall'].items()})\n",
|
| 655 |
+
"print('count per class:', real['per_class_count'])\n",
|
| 656 |
+
"json.dump(real, open(f'{RUN}/results_misviz.json', 'w'), indent=2)"
|
| 657 |
+
]
|
| 658 |
+
},
|
| 659 |
+
{
|
| 660 |
+
"cell_type": "markdown",
|
| 661 |
+
"metadata": {},
|
| 662 |
+
"source": [
|
| 663 |
+
"### Gallery: real charts with the model's guess (great for your video and the 'technical challenge' answer)"
|
| 664 |
+
]
|
| 665 |
+
},
|
| 666 |
+
{
|
| 667 |
+
"cell_type": "code",
|
| 668 |
+
"metadata": {},
|
| 669 |
+
"execution_count": null,
|
| 670 |
+
"outputs": [],
|
| 671 |
+
"source": [
|
| 672 |
+
"shown = 0\n",
|
| 673 |
+
"fig, axes = plt.subplots(3, 4, figsize=(18, 13))\n",
|
| 674 |
+
"model.eval()\n",
|
| 675 |
+
"for r in mv['test']:\n",
|
| 676 |
+
" mis = r['misleader'] or []\n",
|
| 677 |
+
" mapped = {ct.MISVIZ_MAP[m] for m in mis if m in ct.MISVIZ_MAP}\n",
|
| 678 |
+
" if mis and len(mapped) < len(mis):\n",
|
| 679 |
+
" continue\n",
|
| 680 |
+
" x = ct.to_tensor(ct.pad_resize(r['image'])).unsqueeze(0).to(device)\n",
|
| 681 |
+
" with torch.no_grad():\n",
|
| 682 |
+
" pred = ct.CLASSES[model(x).argmax(1).item()]\n",
|
| 683 |
+
" a = axes.flat[shown]; a.imshow(r['image']); a.axis('off')\n",
|
| 684 |
+
" truth = sorted(mapped) or ['none']\n",
|
| 685 |
+
" a.set_title(f\"pred: {pred}\\ntrue: {', '.join(truth)}\", color='green' if pred in truth else 'red')\n",
|
| 686 |
+
" shown += 1\n",
|
| 687 |
+
" if shown == 12: break\n",
|
| 688 |
+
"plt.tight_layout(); plt.savefig(f'{RUN}/misviz_gallery.png', dpi=90); plt.show()"
|
| 689 |
+
]
|
| 690 |
+
},
|
| 691 |
+
{
|
| 692 |
+
"cell_type": "markdown",
|
| 693 |
+
"metadata": {},
|
| 694 |
+
"source": [
|
| 695 |
+
"## 7. (Optional) Misviz-synth\n",
|
| 696 |
+
"\n",
|
| 697 |
+
"If you downloaded Misviz-synth from TUdatalib (https://tudatalib.ulb.tu-darmstadt.de/handle/tudatalib/5003)\n",
|
| 698 |
+
"into Drive, this cell shows its structure so you can map its labels to TELL's 5 classes and add it to training.\n",
|
| 699 |
+
"Skip it if you're short on time; the generated dataset is enough."
|
| 700 |
+
]
|
| 701 |
+
},
|
| 702 |
+
{
|
| 703 |
+
"cell_type": "code",
|
| 704 |
+
"metadata": {},
|
| 705 |
+
"execution_count": null,
|
| 706 |
+
"outputs": [],
|
| 707 |
+
"source": [
|
| 708 |
+
"import glob, json\n",
|
| 709 |
+
"cands = glob.glob('/content/drive/MyDrive/TELL/misviz_synth/**/*.json', recursive=True)\n",
|
| 710 |
+
"print(cands[:5])\n",
|
| 711 |
+
"if cands:\n",
|
| 712 |
+
" d = json.load(open(cands[0]))\n",
|
| 713 |
+
" first = d[0] if isinstance(d, list) else next(iter(d.values()))\n",
|
| 714 |
+
" print(type(d), len(d)); print(json.dumps(first, indent=1)[:1500])"
|
| 715 |
+
]
|
| 716 |
+
},
|
| 717 |
+
{
|
| 718 |
+
"cell_type": "markdown",
|
| 719 |
+
"metadata": {},
|
| 720 |
+
"source": [
|
| 721 |
+
"## 8. Upload the model so the app can use it"
|
| 722 |
+
]
|
| 723 |
+
},
|
| 724 |
+
{
|
| 725 |
+
"cell_type": "code",
|
| 726 |
+
"metadata": {},
|
| 727 |
+
"execution_count": null,
|
| 728 |
+
"outputs": [],
|
| 729 |
+
"source": [
|
| 730 |
+
"from huggingface_hub import HfApi, create_repo, whoami\n",
|
| 731 |
+
"user = whoami()['name']\n",
|
| 732 |
+
"REPO = f'{user}/tell-chartcheck'\n",
|
| 733 |
+
"create_repo(REPO, exist_ok=True)\n",
|
| 734 |
+
"\n",
|
| 735 |
+
"r = json.load(open(f'{RUN}/results.json')); m = json.load(open(f'{RUN}/results_misviz.json'))\n",
|
| 736 |
+
"card = f'''---\n",
|
| 737 |
+
"license: mit\n",
|
| 738 |
+
"tags: [image-classification, misinformation, charts]\n",
|
| 739 |
+
"---\n",
|
| 740 |
+
"# TELL ChartCheck\n",
|
| 741 |
+
"\n",
|
| 742 |
+
"Names misleading chart tricks: {\", \".join(ct.CLASSES)}.\n",
|
| 743 |
+
"Fine-tuned EfficientNet-B0 on 10,000 synthetic charts generated with matplotlib (seeds 0 to 9999).\n",
|
| 744 |
+
"\n",
|
| 745 |
+
"| Test set | Result |\n",
|
| 746 |
+
"| --- | --- |\n",
|
| 747 |
+
"| Synthetic held-out, macro-F1 | {r[\"synthetic_test\"][\"macro avg\"][\"f1-score\"]:.3f} |\n",
|
| 748 |
+
"| Real Misviz charts, accuracy ({m[\"charts_scored\"]} charts) | {m[\"accuracy\"]:.3f} |\n",
|
| 749 |
+
"\n",
|
| 750 |
+
"Part of TELL, a 2026 Congressional App Challenge entry. Real-chart evaluation uses the Misviz benchmark\n",
|
| 751 |
+
"(Tonglet et al., ACL 2026), test only.\n",
|
| 752 |
+
"'''\n",
|
| 753 |
+
"open(f'{RUN}/README.md', 'w').write(card)\n",
|
| 754 |
+
"HfApi().upload_folder(folder_path=RUN, repo_id=REPO,\n",
|
| 755 |
+
" allow_patterns=['config.json', 'model.safetensors', 'results.json', 'results_misviz.json', 'README.md',\n",
|
| 756 |
+
" 'curves.png', 'confusion_synthetic.png']) # no Misviz images: we don't redistribute them\n",
|
| 757 |
+
"print('done: set CHARTCHECK_MODEL =', REPO, 'in your Space secrets')"
|
| 758 |
+
]
|
| 759 |
+
}
|
| 760 |
+
],
|
| 761 |
+
"metadata": {
|
| 762 |
+
"accelerator": "GPU",
|
| 763 |
+
"colab": {
|
| 764 |
+
"provenance": [],
|
| 765 |
+
"gpuType": "T4"
|
| 766 |
+
},
|
| 767 |
+
"kernelspec": {
|
| 768 |
+
"name": "python3",
|
| 769 |
+
"display_name": "Python 3"
|
| 770 |
+
},
|
| 771 |
+
"language_info": {
|
| 772 |
+
"name": "python"
|
| 773 |
+
}
|
| 774 |
+
},
|
| 775 |
+
"nbformat": 4,
|
| 776 |
+
"nbformat_minor": 0
|
| 777 |
+
}
|
notebooks/TELL_ScreenshotCheck_Quick.ipynb
ADDED
|
@@ -0,0 +1,787 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"metadata": {},
|
| 6 |
+
"source": [
|
| 7 |
+
"# TELL ScreenshotCheck: quick train (no screenshots needed)\n",
|
| 8 |
+
"\n",
|
| 9 |
+
"Trains the \"edited screenshot\" detector from public data, then uploads it to Hugging Face.\n",
|
| 10 |
+
"\n",
|
| 11 |
+
"**Data (all built automatically):**\n",
|
| 12 |
+
"- **Real:** 1,500 Android app screenshots from RICO ([rootsautomation/RICO-Screen2Words](https://huggingface.co/datasets/rootsautomation/RICO-Screen2Words), CC BY 4.0) plus 600 synthetic chat and social-post screenshots (text from [cardiffnlp/tweet_eval](https://huggingface.co/datasets/cardiffnlp/tweet_eval)).\n",
|
| 13 |
+
"- **Edited:** 2 copies of each real image with one text box covered and retyped (EasyOCR finds the text).\n",
|
| 14 |
+
"- Half the real images are JPEG-compressed before editing, the way a shared screenshot would be.\n",
|
| 15 |
+
"\n",
|
| 16 |
+
"**Not included:** the \"made on a fake-chat generator\" class. That needs screenshots from real generator sites, so ScreenshotCheck only flags edits for now.\n",
|
| 17 |
+
"\n",
|
| 18 |
+
"**Before you start:** Runtime > Change runtime type > **T4 GPU**. Then Runtime > Run all. About 30 to 45 minutes total."
|
| 19 |
+
]
|
| 20 |
+
},
|
| 21 |
+
{
|
| 22 |
+
"cell_type": "markdown",
|
| 23 |
+
"metadata": {},
|
| 24 |
+
"source": [
|
| 25 |
+
"## 1. Setup"
|
| 26 |
+
]
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"cell_type": "code",
|
| 30 |
+
"metadata": {},
|
| 31 |
+
"execution_count": null,
|
| 32 |
+
"outputs": [],
|
| 33 |
+
"source": [
|
| 34 |
+
"!nvidia-smi -L\n",
|
| 35 |
+
"!pip -q install easyocr timm safetensors datasets scikit-learn\n",
|
| 36 |
+
"!apt-get -qq install -y fonts-noto-core fonts-liberation fonts-dejavu-core fonts-freefont-ttf > /dev/null"
|
| 37 |
+
]
|
| 38 |
+
},
|
| 39 |
+
{
|
| 40 |
+
"cell_type": "code",
|
| 41 |
+
"metadata": {},
|
| 42 |
+
"execution_count": null,
|
| 43 |
+
"outputs": [],
|
| 44 |
+
"source": [
|
| 45 |
+
"from google.colab import drive\n",
|
| 46 |
+
"drive.mount('/content/drive')\n",
|
| 47 |
+
"import os\n",
|
| 48 |
+
"RUN = '/content/drive/MyDrive/TELL/screenshotcheck/auto_run1' # results survive disconnects\n",
|
| 49 |
+
"DATA = '/content/screens_auto' # images stay local (fast)\n",
|
| 50 |
+
"os.makedirs(RUN, exist_ok=True)"
|
| 51 |
+
]
|
| 52 |
+
},
|
| 53 |
+
{
|
| 54 |
+
"cell_type": "markdown",
|
| 55 |
+
"metadata": {},
|
| 56 |
+
"source": [
|
| 57 |
+
"## 2. Code (same files as `training/` in the project)"
|
| 58 |
+
]
|
| 59 |
+
},
|
| 60 |
+
{
|
| 61 |
+
"cell_type": "code",
|
| 62 |
+
"metadata": {},
|
| 63 |
+
"execution_count": null,
|
| 64 |
+
"outputs": [],
|
| 65 |
+
"source": [
|
| 66 |
+
"%%writefile screen_edits.py\n",
|
| 67 |
+
"\"\"\"\n",
|
| 68 |
+
"TELL ScreenshotCheck: automatically make EDITED screenshots from real ones.\n",
|
| 69 |
+
"\n",
|
| 70 |
+
"For each real screenshot:\n",
|
| 71 |
+
" 1. OCR finds the text boxes (messages, names, times).\n",
|
| 72 |
+
" 2. Pick one box, cover it with the bubble's background color.\n",
|
| 73 |
+
" 3. Type replacement text in a similar size and color.\n",
|
| 74 |
+
" 4. Save the edited image and the exact edit box (used for training and for\n",
|
| 75 |
+
" checking whether the heatmap points at the right place).\n",
|
| 76 |
+
"\n",
|
| 77 |
+
"Why this is realistic: this is exactly how most people fake a screenshot\n",
|
| 78 |
+
"(inspect element or a photo editor: cover a message, type a new one).\n",
|
| 79 |
+
"Known limit to document: the replacement font (DejaVu Sans) differs from\n",
|
| 80 |
+
"iOS/Android system fonts, so the model may partly learn \"wrong font\".\n",
|
| 81 |
+
"That is still a real signal people miss, but say it honestly.\n",
|
| 82 |
+
"\"\"\"\n",
|
| 83 |
+
"\n",
|
| 84 |
+
"import json\n",
|
| 85 |
+
"import random\n",
|
| 86 |
+
"from pathlib import Path\n",
|
| 87 |
+
"\n",
|
| 88 |
+
"import numpy as np\n",
|
| 89 |
+
"from PIL import Image, ImageDraw, ImageFont\n",
|
| 90 |
+
"\n",
|
| 91 |
+
"SWAPS = [\"no\", \"yes\", \"lol ok\", \"never\", \"i did it\", \"not me\", \"tomorrow\", \"sure\",\n",
|
| 92 |
+
" \"i hate this\", \"ok fine\", \"it was him\", \"i'm leaving\", \"deal\", \"who cares\",\n",
|
| 93 |
+
" \"be there at 5\", \"send it\", \"nope\", \"for real\", \"i lied\", \"i know\"]\n",
|
| 94 |
+
"FONT_PATHS = [\"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf\",\n",
|
| 95 |
+
" \"/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf\"]\n",
|
| 96 |
+
"\n",
|
| 97 |
+
"\n",
|
| 98 |
+
"def find_text_boxes(img, reader=None):\n",
|
| 99 |
+
" \"\"\"Returns [(x0, y0, x1, y1, text)] using EasyOCR. Pass a shared reader for speed.\"\"\"\n",
|
| 100 |
+
" import easyocr\n",
|
| 101 |
+
" reader = reader or easyocr.Reader([\"en\"], gpu=True)\n",
|
| 102 |
+
" out = []\n",
|
| 103 |
+
" for box, text, conf in reader.readtext(np.array(img.convert(\"RGB\"))):\n",
|
| 104 |
+
" xs, ys = [p[0] for p in box], [p[1] for p in box]\n",
|
| 105 |
+
" if conf > 0.3 and (max(ys) - min(ys)) >= 12 and len(text) >= 2:\n",
|
| 106 |
+
" out.append((int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys)), text))\n",
|
| 107 |
+
" return out\n",
|
| 108 |
+
"\n",
|
| 109 |
+
"\n",
|
| 110 |
+
"def _font(size, rng):\n",
|
| 111 |
+
" for p in rng.sample(FONT_PATHS, len(FONT_PATHS)):\n",
|
| 112 |
+
" try:\n",
|
| 113 |
+
" return ImageFont.truetype(p, size)\n",
|
| 114 |
+
" except OSError:\n",
|
| 115 |
+
" continue\n",
|
| 116 |
+
" return ImageFont.load_default()\n",
|
| 117 |
+
"\n",
|
| 118 |
+
"\n",
|
| 119 |
+
"def edit_one(img, box, rng):\n",
|
| 120 |
+
" \"\"\"Cover one text box and write new text. Returns (edited_img, edit_box).\"\"\"\n",
|
| 121 |
+
" img = img.convert(\"RGB\").copy()\n",
|
| 122 |
+
" x0, y0, x1, y1, _ = box\n",
|
| 123 |
+
" pad = 3\n",
|
| 124 |
+
" a = np.asarray(img)\n",
|
| 125 |
+
" h, w = a.shape[:2]\n",
|
| 126 |
+
" ring = np.concatenate([\n",
|
| 127 |
+
" a[max(0, y0 - pad), max(0, x0 - pad):min(w, x1 + pad)],\n",
|
| 128 |
+
" a[min(h - 1, y1 + pad), max(0, x0 - pad):min(w, x1 + pad)],\n",
|
| 129 |
+
" ])\n",
|
| 130 |
+
" bg = tuple(int(v) for v in np.median(ring, axis=0))\n",
|
| 131 |
+
" inner = a[y0:y1, x0:x1].reshape(-1, 3)\n",
|
| 132 |
+
" lum = inner.mean(1)\n",
|
| 133 |
+
" bg_lum = sum(bg) / 3\n",
|
| 134 |
+
" ink = inner[np.argmin(lum)] if bg_lum > 128 else inner[np.argmax(lum)]\n",
|
| 135 |
+
" d = ImageDraw.Draw(img)\n",
|
| 136 |
+
" d.rectangle([x0 - 1, y0 - 1, x1 + 1, y1 + 1], fill=bg)\n",
|
| 137 |
+
" size = max(10, int((y1 - y0) * rng.uniform(0.75, 0.9)))\n",
|
| 138 |
+
" text = rng.choice(SWAPS)\n",
|
| 139 |
+
" d.text((x0, y0 + (y1 - y0 - size) // 2), text, fill=tuple(int(v) for v in ink),\n",
|
| 140 |
+
" font=_font(size, rng))\n",
|
| 141 |
+
" tw = int(d.textlength(text, font=_font(size, rng)))\n",
|
| 142 |
+
" return img, [x0 - 1, y0 - 1, max(x1, x0 + tw) + 1, y1 + 1]\n",
|
| 143 |
+
"\n",
|
| 144 |
+
"\n",
|
| 145 |
+
"def make_edited_set(real_dir, out_dir, per_image=2, seed=0, reader=None):\n",
|
| 146 |
+
" \"\"\"Creates edited/ images plus edits.json with source and edit box for each.\"\"\"\n",
|
| 147 |
+
" rng = random.Random(seed)\n",
|
| 148 |
+
" real_dir, out_dir = Path(real_dir), Path(out_dir)\n",
|
| 149 |
+
" out_dir.mkdir(parents=True, exist_ok=True)\n",
|
| 150 |
+
" rows = []\n",
|
| 151 |
+
" for p in sorted(real_dir.glob(\"*\")):\n",
|
| 152 |
+
" if p.suffix.lower() not in {\".png\", \".jpg\", \".jpeg\"}:\n",
|
| 153 |
+
" continue\n",
|
| 154 |
+
" img = Image.open(p).convert(\"RGB\")\n",
|
| 155 |
+
" boxes = find_text_boxes(img, reader)\n",
|
| 156 |
+
" rng.shuffle(boxes)\n",
|
| 157 |
+
" for k, box in enumerate(boxes[:per_image]):\n",
|
| 158 |
+
" edited, ebox = edit_one(img, box, rng)\n",
|
| 159 |
+
" name = f\"{p.stem}__edit{k}.png\"\n",
|
| 160 |
+
" edited.save(out_dir / name)\n",
|
| 161 |
+
" rows.append({\"path\": name, \"source\": p.name, \"edit_box\": ebox})\n",
|
| 162 |
+
" (out_dir / \"edits.json\").write_text(json.dumps(rows, indent=1))\n",
|
| 163 |
+
" print(f\"made {len(rows)} edited screenshots from {real_dir}\")\n",
|
| 164 |
+
" return rows"
|
| 165 |
+
]
|
| 166 |
+
},
|
| 167 |
+
{
|
| 168 |
+
"cell_type": "code",
|
| 169 |
+
"metadata": {},
|
| 170 |
+
"execution_count": null,
|
| 171 |
+
"outputs": [],
|
| 172 |
+
"source": [
|
| 173 |
+
"%%writefile screen_train.py\n",
|
| 174 |
+
"\"\"\"\n",
|
| 175 |
+
"TELL ScreenshotCheck training.\n",
|
| 176 |
+
"\n",
|
| 177 |
+
"Three classes: real, generated (made on a fake-chat website), edited (real with\n",
|
| 178 |
+
"one message replaced). Trains on 256x256 PATCHES at full resolution, because\n",
|
| 179 |
+
"resizing the whole screenshot would erase the tiny pixel clues we need.\n",
|
| 180 |
+
"\n",
|
| 181 |
+
"Data layout on Google Drive (see TRAINING.md):\n",
|
| 182 |
+
" screens/real/*.png your + friends' screenshots (blurred names)\n",
|
| 183 |
+
" screens/generated/<site>_*.png fakes made on fake-chat sites, filename starts with site name\n",
|
| 184 |
+
" screens/edited/*.png + edits.json made by screen_edits.py from real/\n",
|
| 185 |
+
"\n",
|
| 186 |
+
"Leakage rules (important to document):\n",
|
| 187 |
+
" - A real screenshot and every edited copy of it ALWAYS land in the same split,\n",
|
| 188 |
+
" otherwise the model \"recognizes\" the original and the test score is fake.\n",
|
| 189 |
+
" - One generator site is held out completely for testing, so the test number\n",
|
| 190 |
+
" says whether TELL catches a generator it has never seen.\n",
|
| 191 |
+
"\"\"\"\n",
|
| 192 |
+
"\n",
|
| 193 |
+
"import io\n",
|
| 194 |
+
"import json\n",
|
| 195 |
+
"import random\n",
|
| 196 |
+
"from pathlib import Path\n",
|
| 197 |
+
"\n",
|
| 198 |
+
"import numpy as np\n",
|
| 199 |
+
"import torch\n",
|
| 200 |
+
"import torch.nn as nn\n",
|
| 201 |
+
"from PIL import Image\n",
|
| 202 |
+
"\n",
|
| 203 |
+
"CLASSES = [\"real\", \"generated\", \"edited\"]\n",
|
| 204 |
+
"PATCH, STRIDE = 256, 192\n",
|
| 205 |
+
"MEAN = (0.485, 0.456, 0.406)\n",
|
| 206 |
+
"STD = (0.229, 0.224, 0.225)\n",
|
| 207 |
+
"MIN_STD = 8.0 # skip nearly blank patches (plain background tells us nothing)\n",
|
| 208 |
+
"\n",
|
| 209 |
+
"\n",
|
| 210 |
+
"# ---------- Building the list of images ----------\n",
|
| 211 |
+
"\n",
|
| 212 |
+
"def build_index(root, holdout_site=None, seed=42, val=0.15, test=0.15):\n",
|
| 213 |
+
" root = Path(root)\n",
|
| 214 |
+
" rng = random.Random(seed)\n",
|
| 215 |
+
" reals = sorted(p.name for p in (root / \"real\").iterdir() if p.suffix.lower() in {\".png\", \".jpg\", \".jpeg\"})\n",
|
| 216 |
+
" edits = json.loads((root / \"edited\" / \"edits.json\").read_text())\n",
|
| 217 |
+
" gen_dir = root / \"generated\" # optional: the auto-data notebook trains without it\n",
|
| 218 |
+
" gens = sorted(p.name for p in gen_dir.iterdir() if p.suffix.lower() in {\".png\", \".jpg\", \".jpeg\"}) if gen_dir.exists() else []\n",
|
| 219 |
+
"\n",
|
| 220 |
+
" # split REAL by source; edits follow their source\n",
|
| 221 |
+
" rng.shuffle(reals)\n",
|
| 222 |
+
" nv, nt = int(len(reals) * val), int(len(reals) * test)\n",
|
| 223 |
+
" real_split = {n: \"val\" for n in reals[:nv]}\n",
|
| 224 |
+
" real_split.update({n: \"test\" for n in reals[nv:nv + nt]})\n",
|
| 225 |
+
" real_split.update({n: \"train\" for n in reals[nv + nt:]})\n",
|
| 226 |
+
"\n",
|
| 227 |
+
" rows = [{\"path\": f\"real/{n}\", \"label\": \"real\", \"split\": real_split[n], \"group\": n} for n in reals]\n",
|
| 228 |
+
" rows += [{\"path\": f\"edited/{e['path']}\", \"label\": \"edited\", \"split\": real_split[e[\"source\"]],\n",
|
| 229 |
+
" \"group\": e[\"source\"], \"edit_box\": e[\"edit_box\"]} for e in edits if e[\"source\"] in real_split]\n",
|
| 230 |
+
"\n",
|
| 231 |
+
" # generated: hold one site out for test, split the rest\n",
|
| 232 |
+
" sites = sorted({g.split(\"_\")[0] for g in gens})\n",
|
| 233 |
+
" holdout_site = holdout_site or (sites[-1] if len(sites) > 1 else None)\n",
|
| 234 |
+
" others = [g for g in gens if g.split(\"_\")[0] != holdout_site]\n",
|
| 235 |
+
" rng.shuffle(others)\n",
|
| 236 |
+
" gv = int(len(others) * val)\n",
|
| 237 |
+
" for i, g in enumerate(others):\n",
|
| 238 |
+
" rows.append({\"path\": f\"generated/{g}\", \"label\": \"generated\", \"split\": \"val\" if i < gv else \"train\",\n",
|
| 239 |
+
" \"group\": g, \"site\": g.split(\"_\")[0]})\n",
|
| 240 |
+
" for g in gens:\n",
|
| 241 |
+
" if g.split(\"_\")[0] == holdout_site:\n",
|
| 242 |
+
" rows.append({\"path\": f\"generated/{g}\", \"label\": \"generated\", \"split\": \"test\",\n",
|
| 243 |
+
" \"group\": g, \"site\": g.split(\"_\")[0], \"unseen_site\": True})\n",
|
| 244 |
+
" return rows, holdout_site\n",
|
| 245 |
+
"\n",
|
| 246 |
+
"\n",
|
| 247 |
+
"# ---------- Augmentation applied to the WHOLE image, same for every class ----------\n",
|
| 248 |
+
"\n",
|
| 249 |
+
"def realistic_noise(img, box, rng):\n",
|
| 250 |
+
" \"\"\"Social media recompresses and rescales screenshots. Apply to all classes\n",
|
| 251 |
+
" equally so the model can't cheat by learning 'JPEG means fake'.\"\"\"\n",
|
| 252 |
+
" if rng.random() < 0.3:\n",
|
| 253 |
+
" f = rng.uniform(0.75, 1.0)\n",
|
| 254 |
+
" img = img.resize((int(img.width * f), int(img.height * f)), Image.BILINEAR)\n",
|
| 255 |
+
" if box:\n",
|
| 256 |
+
" box = [int(v * f) for v in box]\n",
|
| 257 |
+
" if rng.random() < 0.5:\n",
|
| 258 |
+
" buf = io.BytesIO()\n",
|
| 259 |
+
" img.save(buf, \"JPEG\", quality=rng.randint(60, 95))\n",
|
| 260 |
+
" buf.seek(0)\n",
|
| 261 |
+
" img = Image.open(buf).convert(\"RGB\")\n",
|
| 262 |
+
" return img, box\n",
|
| 263 |
+
"\n",
|
| 264 |
+
"\n",
|
| 265 |
+
"def _norm(patch):\n",
|
| 266 |
+
" a = (np.asarray(patch, dtype=np.float32) / 255.0 - MEAN) / STD\n",
|
| 267 |
+
" return torch.from_numpy(a.transpose(2, 0, 1)).float()\n",
|
| 268 |
+
"\n",
|
| 269 |
+
"\n",
|
| 270 |
+
"def _overlap(px, py, box):\n",
|
| 271 |
+
" x0, y0, x1, y1 = box\n",
|
| 272 |
+
" ix = max(0, min(px + PATCH, x1) - max(px, x0))\n",
|
| 273 |
+
" iy = max(0, min(py + PATCH, y1) - max(py, y0))\n",
|
| 274 |
+
" area = max(1, (x1 - x0) * (y1 - y0))\n",
|
| 275 |
+
" return ix * iy / area\n",
|
| 276 |
+
"\n",
|
| 277 |
+
"\n",
|
| 278 |
+
"class PatchDataset(torch.utils.data.Dataset):\n",
|
| 279 |
+
" \"\"\"Each item = one random informative patch from one image.\"\"\"\n",
|
| 280 |
+
"\n",
|
| 281 |
+
" def __init__(self, rows, root, train=True, repeats=8):\n",
|
| 282 |
+
" self.rows, self.root, self.train, self.repeats = rows, Path(root), train, repeats\n",
|
| 283 |
+
"\n",
|
| 284 |
+
" def __len__(self):\n",
|
| 285 |
+
" return len(self.rows) * self.repeats\n",
|
| 286 |
+
"\n",
|
| 287 |
+
" def __getitem__(self, i):\n",
|
| 288 |
+
" r = self.rows[i % len(self.rows)]\n",
|
| 289 |
+
" rng = random.Random() if self.train else random.Random(i)\n",
|
| 290 |
+
" img = Image.open(self.root / r[\"path\"]).convert(\"RGB\")\n",
|
| 291 |
+
" box = r.get(\"edit_box\")\n",
|
| 292 |
+
" if self.train:\n",
|
| 293 |
+
" img, box = realistic_noise(img, box, rng)\n",
|
| 294 |
+
" if img.width < PATCH or img.height < PATCH:\n",
|
| 295 |
+
" img = img.resize((max(img.width, PATCH), max(img.height, PATCH)))\n",
|
| 296 |
+
" max_x, max_y = img.width - PATCH, img.height - PATCH\n",
|
| 297 |
+
" patch = None\n",
|
| 298 |
+
" for _ in range(15):\n",
|
| 299 |
+
" if box: # edited: the patch must contain most of the edit\n",
|
| 300 |
+
" lo_x = min(max(0, box[2] - PATCH), max_x)\n",
|
| 301 |
+
" hi_x = max(lo_x, min(box[0], max_x))\n",
|
| 302 |
+
" lo_y = min(max(0, box[3] - PATCH), max_y)\n",
|
| 303 |
+
" hi_y = max(lo_y, min(box[1], max_y))\n",
|
| 304 |
+
" px, py = rng.randint(lo_x, hi_x), rng.randint(lo_y, hi_y)\n",
|
| 305 |
+
" if _overlap(px, py, box) < 0.5:\n",
|
| 306 |
+
" continue\n",
|
| 307 |
+
" else:\n",
|
| 308 |
+
" px, py = rng.randint(0, max_x), rng.randint(0, max_y)\n",
|
| 309 |
+
" cand = img.crop((px, py, px + PATCH, py + PATCH))\n",
|
| 310 |
+
" if np.asarray(cand).std() >= MIN_STD:\n",
|
| 311 |
+
" patch = cand\n",
|
| 312 |
+
" break\n",
|
| 313 |
+
" if patch is None: # fallback: center on the edit, or the image center\n",
|
| 314 |
+
" cx = (box[0] + box[2]) // 2 if box else img.width // 2\n",
|
| 315 |
+
" cy = (box[1] + box[3]) // 2 if box else img.height // 2\n",
|
| 316 |
+
" px = min(max(0, cx - PATCH // 2), max_x)\n",
|
| 317 |
+
" py = min(max(0, cy - PATCH // 2), max_y)\n",
|
| 318 |
+
" patch = img.crop((px, py, px + PATCH, py + PATCH))\n",
|
| 319 |
+
" return _norm(patch), CLASSES.index(r[\"label\"])\n",
|
| 320 |
+
"\n",
|
| 321 |
+
"\n",
|
| 322 |
+
"# ---------- Whole-image prediction by tiling (MUST match tools/screenshotcheck.py) ----------\n",
|
| 323 |
+
"\n",
|
| 324 |
+
"@torch.no_grad()\n",
|
| 325 |
+
"def tile_predict(model, img, device=\"cpu\", batch=32):\n",
|
| 326 |
+
" \"\"\"Returns per-patch probabilities [(x, y, p_real, p_gen, p_edit)].\"\"\"\n",
|
| 327 |
+
" img = img.convert(\"RGB\")\n",
|
| 328 |
+
" if img.width < PATCH or img.height < PATCH:\n",
|
| 329 |
+
" img = img.resize((max(img.width, PATCH), max(img.height, PATCH)))\n",
|
| 330 |
+
" xs = list(range(0, img.width - PATCH + 1, STRIDE)) or [0]\n",
|
| 331 |
+
" ys = list(range(0, img.height - PATCH + 1, STRIDE)) or [0]\n",
|
| 332 |
+
" coords, tensors = [], []\n",
|
| 333 |
+
" for y in ys:\n",
|
| 334 |
+
" for x in xs:\n",
|
| 335 |
+
" p = img.crop((x, y, x + PATCH, y + PATCH))\n",
|
| 336 |
+
" if np.asarray(p).std() >= MIN_STD:\n",
|
| 337 |
+
" coords.append((x, y))\n",
|
| 338 |
+
" tensors.append(_norm(p))\n",
|
| 339 |
+
" out = []\n",
|
| 340 |
+
" model.eval()\n",
|
| 341 |
+
" for i in range(0, len(tensors), batch):\n",
|
| 342 |
+
" probs = torch.softmax(model(torch.stack(tensors[i:i + batch]).to(device)).float(), 1).cpu()\n",
|
| 343 |
+
" out += [(*coords[i + k], *probs[k].tolist()) for k in range(len(probs))]\n",
|
| 344 |
+
" return out\n",
|
| 345 |
+
"\n",
|
| 346 |
+
"\n",
|
| 347 |
+
"def image_scores(patches):\n",
|
| 348 |
+
" \"\"\"Turn patch probabilities into two image-level scores.\"\"\"\n",
|
| 349 |
+
" if not patches:\n",
|
| 350 |
+
" return 0.0, 0.0\n",
|
| 351 |
+
" gen = float(np.mean([p[3] for p in patches]))\n",
|
| 352 |
+
" edit = float(np.mean(sorted([p[4] for p in patches], reverse=True)[:2]))\n",
|
| 353 |
+
" return gen, edit\n",
|
| 354 |
+
"\n",
|
| 355 |
+
"\n",
|
| 356 |
+
"def decide(gen, edit, t_gen, t_edit):\n",
|
| 357 |
+
" if gen >= t_gen:\n",
|
| 358 |
+
" return \"generated\"\n",
|
| 359 |
+
" if edit >= t_edit:\n",
|
| 360 |
+
" return \"edited\"\n",
|
| 361 |
+
" return \"real\"\n",
|
| 362 |
+
"\n",
|
| 363 |
+
"\n",
|
| 364 |
+
"# ---------- Training ----------\n",
|
| 365 |
+
"\n",
|
| 366 |
+
"def build_model(arch=\"efficientnet_b0\", pretrained=True):\n",
|
| 367 |
+
" import timm\n",
|
| 368 |
+
" return timm.create_model(arch, pretrained=pretrained, num_classes=len(CLASSES))\n",
|
| 369 |
+
"\n",
|
| 370 |
+
"\n",
|
| 371 |
+
"def train(root, out_dir, arch=\"efficientnet_b0\", epochs=8, batch=48, lr=3e-4, workers=2,\n",
|
| 372 |
+
" holdout_site=None, seed=42, pretrained=True):\n",
|
| 373 |
+
" torch.manual_seed(seed)\n",
|
| 374 |
+
" root, out_dir = Path(root), Path(out_dir)\n",
|
| 375 |
+
" out_dir.mkdir(parents=True, exist_ok=True)\n",
|
| 376 |
+
" rows, holdout_site = build_index(root, holdout_site, seed)\n",
|
| 377 |
+
" (out_dir / \"index.json\").write_text(json.dumps(rows, indent=1))\n",
|
| 378 |
+
" part = lambda s: [r for r in rows if r[\"split\"] == s]\n",
|
| 379 |
+
" counts = {s: {c: sum(r[\"label\"] == c for r in part(s)) for c in CLASSES} for s in [\"train\", \"val\", \"test\"]}\n",
|
| 380 |
+
" print(\"images per split:\", counts, \"| held-out generator site:\", holdout_site)\n",
|
| 381 |
+
"\n",
|
| 382 |
+
" device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
|
| 383 |
+
" tr = part(\"train\")\n",
|
| 384 |
+
" # balance classes: each class contributes equally per epoch\n",
|
| 385 |
+
" weights = [1.0 / sum(x[\"label\"] == r[\"label\"] for x in tr) for r in tr] * 8\n",
|
| 386 |
+
" sampler = torch.utils.data.WeightedRandomSampler(weights, num_samples=len(weights), replacement=True)\n",
|
| 387 |
+
" dl = torch.utils.data.DataLoader(PatchDataset(tr, root, True, 8), batch_size=batch,\n",
|
| 388 |
+
" sampler=sampler, num_workers=workers)\n",
|
| 389 |
+
" dl_val = torch.utils.data.DataLoader(PatchDataset(part(\"val\"), root, False, 4), batch_size=batch,\n",
|
| 390 |
+
" num_workers=workers)\n",
|
| 391 |
+
"\n",
|
| 392 |
+
" model = build_model(arch, pretrained).to(device)\n",
|
| 393 |
+
" opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)\n",
|
| 394 |
+
" sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=epochs * len(dl), pct_start=0.1)\n",
|
| 395 |
+
" scaler = torch.amp.GradScaler(enabled=device == \"cuda\")\n",
|
| 396 |
+
" loss_fn = nn.CrossEntropyLoss(label_smoothing=0.05)\n",
|
| 397 |
+
"\n",
|
| 398 |
+
" from sklearn.metrics import f1_score\n",
|
| 399 |
+
" best, history = -1, []\n",
|
| 400 |
+
" for ep in range(epochs):\n",
|
| 401 |
+
" model.train()\n",
|
| 402 |
+
" tot = 0.0\n",
|
| 403 |
+
" for x, y in dl:\n",
|
| 404 |
+
" x, y = x.to(device), y.to(device)\n",
|
| 405 |
+
" with torch.autocast(device_type=device, enabled=device == \"cuda\"):\n",
|
| 406 |
+
" loss = loss_fn(model(x), y)\n",
|
| 407 |
+
" opt.zero_grad()\n",
|
| 408 |
+
" scaler.scale(loss).backward()\n",
|
| 409 |
+
" scaler.step(opt)\n",
|
| 410 |
+
" scaler.update()\n",
|
| 411 |
+
" sched.step()\n",
|
| 412 |
+
" tot += loss.item() * len(y)\n",
|
| 413 |
+
" model.eval()\n",
|
| 414 |
+
" ys, ps = [], []\n",
|
| 415 |
+
" with torch.no_grad():\n",
|
| 416 |
+
" for x, y in dl_val:\n",
|
| 417 |
+
" ps.append(model(x.to(device)).argmax(1).cpu())\n",
|
| 418 |
+
" ys.append(y)\n",
|
| 419 |
+
" f1 = f1_score(torch.cat(ys), torch.cat(ps), average=\"macro\")\n",
|
| 420 |
+
" history.append({\"epoch\": ep + 1, \"train_loss\": tot / len(dl.dataset), \"val_patch_macro_f1\": f1})\n",
|
| 421 |
+
" print(history[-1])\n",
|
| 422 |
+
" if f1 > best:\n",
|
| 423 |
+
" best = f1\n",
|
| 424 |
+
" torch.save(model.state_dict(), out_dir / \"best.pt\")\n",
|
| 425 |
+
"\n",
|
| 426 |
+
" model.load_state_dict(torch.load(out_dir / \"best.pt\", map_location=device))\n",
|
| 427 |
+
" t_gen, t_edit = tune_thresholds(model, part(\"val\"), root, device)\n",
|
| 428 |
+
" results = evaluate(model, part(\"test\"), root, device, t_gen, t_edit)\n",
|
| 429 |
+
" results.update({\"history\": history, \"thresholds\": {\"generated\": t_gen, \"edited\": t_edit},\n",
|
| 430 |
+
" \"held_out_site\": holdout_site, \"images_per_split\": counts})\n",
|
| 431 |
+
" (out_dir / \"results.json\").write_text(json.dumps(results, indent=2))\n",
|
| 432 |
+
" save_for_app(model, out_dir, arch, t_gen, t_edit)\n",
|
| 433 |
+
" return model, results\n",
|
| 434 |
+
"\n",
|
| 435 |
+
"\n",
|
| 436 |
+
"def _scores_for(model, rows, root, device):\n",
|
| 437 |
+
" out = []\n",
|
| 438 |
+
" for r in rows:\n",
|
| 439 |
+
" patches = tile_predict(model, Image.open(Path(root) / r[\"path\"]), device)\n",
|
| 440 |
+
" out.append((r, patches, *image_scores(patches)))\n",
|
| 441 |
+
" return out\n",
|
| 442 |
+
"\n",
|
| 443 |
+
"\n",
|
| 444 |
+
"def tune_thresholds(model, val_rows, root, device):\n",
|
| 445 |
+
" \"\"\"Pick the cutoffs that best separate classes on VALIDATION images (never test).\"\"\"\n",
|
| 446 |
+
" from sklearn.metrics import f1_score\n",
|
| 447 |
+
" scored = _scores_for(model, val_rows, root, device)\n",
|
| 448 |
+
" grid = np.linspace(0.2, 0.9, 15)\n",
|
| 449 |
+
" best = (0.5, 0.5, -1)\n",
|
| 450 |
+
" # no generated examples means that class can't be tuned, so turn it off (threshold above 1)\n",
|
| 451 |
+
" has_gen = any(r[\"label\"] == \"generated\" for r in val_rows)\n",
|
| 452 |
+
" for tg in (grid if has_gen else [1.01]):\n",
|
| 453 |
+
" for te in grid:\n",
|
| 454 |
+
" y = [r[\"label\"] for r, *_ in scored]\n",
|
| 455 |
+
" p = [decide(g, e, tg, te) for _, _, g, e in scored]\n",
|
| 456 |
+
" f = f1_score(y, p, average=\"macro\", labels=CLASSES, zero_division=0)\n",
|
| 457 |
+
" if f > best[2]:\n",
|
| 458 |
+
" best = (float(tg), float(te), f)\n",
|
| 459 |
+
" print(f\"thresholds: generated={best[0]:.2f} edited={best[1]:.2f} (val macro-F1 {best[2]:.3f})\")\n",
|
| 460 |
+
" return best[0], best[1]\n",
|
| 461 |
+
"\n",
|
| 462 |
+
"\n",
|
| 463 |
+
"def evaluate(model, test_rows, root, device, t_gen, t_edit):\n",
|
| 464 |
+
" from sklearn.metrics import classification_report, confusion_matrix\n",
|
| 465 |
+
" scored = _scores_for(model, test_rows, root, device)\n",
|
| 466 |
+
" y = [r[\"label\"] for r, *_ in scored]\n",
|
| 467 |
+
" p = [decide(g, e, t_gen, t_edit) for _, _, g, e in scored]\n",
|
| 468 |
+
" report = classification_report(y, p, labels=CLASSES, output_dict=True, zero_division=0)\n",
|
| 469 |
+
" print(classification_report(y, p, labels=CLASSES, zero_division=0))\n",
|
| 470 |
+
" unseen = [(r, d) for (r, *_), d in zip(scored, p) if r.get(\"unseen_site\")]\n",
|
| 471 |
+
" loc_hits, loc_n = 0, 0\n",
|
| 472 |
+
" for r, patches, g, e in scored:\n",
|
| 473 |
+
" if r[\"label\"] == \"edited\" and patches:\n",
|
| 474 |
+
" top = max(patches, key=lambda q: q[4])\n",
|
| 475 |
+
" loc_hits += _overlap(top[0], top[1], r[\"edit_box\"]) > 0\n",
|
| 476 |
+
" loc_n += 1\n",
|
| 477 |
+
" return {\n",
|
| 478 |
+
" \"test_report\": report,\n",
|
| 479 |
+
" \"test_confusion\": confusion_matrix(y, p, labels=CLASSES).tolist(),\n",
|
| 480 |
+
" \"unseen_site_caught\": sum(d == \"generated\" for _, d in unseen) / max(1, len(unseen)),\n",
|
| 481 |
+
" \"unseen_site_images\": len(unseen),\n",
|
| 482 |
+
" \"edit_localization\": loc_hits / max(1, loc_n),\n",
|
| 483 |
+
" }\n",
|
| 484 |
+
"\n",
|
| 485 |
+
"\n",
|
| 486 |
+
"def save_for_app(model, out_dir, arch, t_gen, t_edit):\n",
|
| 487 |
+
" from safetensors.torch import save_file\n",
|
| 488 |
+
" out_dir = Path(out_dir)\n",
|
| 489 |
+
" save_file({k: v.contiguous().cpu() for k, v in model.state_dict().items()}, out_dir / \"model.safetensors\")\n",
|
| 490 |
+
" (out_dir / \"config.json\").write_text(json.dumps({\n",
|
| 491 |
+
" \"arch\": arch, \"classes\": CLASSES, \"patch\": PATCH, \"stride\": STRIDE, \"min_std\": MIN_STD,\n",
|
| 492 |
+
" \"mean\": MEAN, \"std\": STD, \"t_generated\": t_gen, \"t_edited\": t_edit}, indent=2))"
|
| 493 |
+
]
|
| 494 |
+
},
|
| 495 |
+
{
|
| 496 |
+
"cell_type": "code",
|
| 497 |
+
"metadata": {},
|
| 498 |
+
"execution_count": null,
|
| 499 |
+
"outputs": [],
|
| 500 |
+
"source": [
|
| 501 |
+
"%%writefile screen_autodata.py\n",
|
| 502 |
+
"\"\"\"\n",
|
| 503 |
+
"Builds a ScreenshotCheck training set with no hand-collected screenshots.\n",
|
| 504 |
+
"\n",
|
| 505 |
+
"real/\n",
|
| 506 |
+
" rico_*.png real Android app screenshots from RICO (via rootsautomation/RICO-Screen2Words\n",
|
| 507 |
+
" on Hugging Face, CC BY 4.0). Resized to phone-share width.\n",
|
| 508 |
+
" chat_*.png synthetic chat and social-post screenshots, so the model also sees\n",
|
| 509 |
+
" chat layouts. Text comes from tweet_eval (cardiffnlp/tweet_eval).\n",
|
| 510 |
+
"edited/ made from real/ by screen_edits.make_edited_set (cover one text box, type new text)\n",
|
| 511 |
+
"\n",
|
| 512 |
+
"Half of the real images get JPEG-compressed BEFORE editing, the way a shared screenshot\n",
|
| 513 |
+
"would be. The edit then gets pasted on top of the compressed pixels, the same thing that\n",
|
| 514 |
+
"happens when someone edits a screenshot they were sent.\n",
|
| 515 |
+
"\n",
|
| 516 |
+
"No \"generated\" class here: that needs output from real fake-chat sites.\n",
|
| 517 |
+
"\"\"\"\n",
|
| 518 |
+
"\n",
|
| 519 |
+
"import io\n",
|
| 520 |
+
"import random\n",
|
| 521 |
+
"import textwrap\n",
|
| 522 |
+
"from pathlib import Path\n",
|
| 523 |
+
"\n",
|
| 524 |
+
"from PIL import Image, ImageDraw, ImageFont\n",
|
| 525 |
+
"\n",
|
| 526 |
+
"WIDTH = 720\n",
|
| 527 |
+
"\n",
|
| 528 |
+
"FONT_DIRS = [\"/usr/share/fonts/truetype/noto\", \"/usr/share/fonts/truetype/liberation\",\n",
|
| 529 |
+
" \"/usr/share/fonts/truetype/dejavu\", \"/usr/share/fonts/truetype/freefont\"]\n",
|
| 530 |
+
"\n",
|
| 531 |
+
"\n",
|
| 532 |
+
"# Latin UI-style families only. Noto ships hundreds of script-specific fonts that\n",
|
| 533 |
+
"# would render tweets as empty boxes, and emoji fonts can't be sized at all.\n",
|
| 534 |
+
"FONT_PREFIXES = (\"NotoSans-\", \"NotoSerif-\", \"LiberationSans-\", \"LiberationSerif-\",\n",
|
| 535 |
+
" \"DejaVuSans.\", \"DejaVuSans-Bold\", \"DejaVuSansCondensed\", \"FreeSans\")\n",
|
| 536 |
+
"\n",
|
| 537 |
+
"\n",
|
| 538 |
+
"def _fonts():\n",
|
| 539 |
+
" found = []\n",
|
| 540 |
+
" for d in FONT_DIRS:\n",
|
| 541 |
+
" for f in Path(d).glob(\"*.ttf\") if Path(d).exists() else []:\n",
|
| 542 |
+
" if f.name.startswith(FONT_PREFIXES) and \"Italic\" not in f.name and \"Oblique\" not in f.name:\n",
|
| 543 |
+
" try:\n",
|
| 544 |
+
" ImageFont.truetype(str(f), 30)\n",
|
| 545 |
+
" found.append(str(f))\n",
|
| 546 |
+
" except OSError:\n",
|
| 547 |
+
" pass\n",
|
| 548 |
+
" return found or [None]\n",
|
| 549 |
+
"\n",
|
| 550 |
+
"\n",
|
| 551 |
+
"def _jpeg(img, rng, p=0.5):\n",
|
| 552 |
+
" if rng.random() >= p:\n",
|
| 553 |
+
" return img\n",
|
| 554 |
+
" buf = io.BytesIO()\n",
|
| 555 |
+
" img.save(buf, \"JPEG\", quality=rng.randint(70, 95))\n",
|
| 556 |
+
" return Image.open(buf).convert(\"RGB\")\n",
|
| 557 |
+
"\n",
|
| 558 |
+
"\n",
|
| 559 |
+
"def _to_width(img, w=WIDTH):\n",
|
| 560 |
+
" return img.resize((w, int(img.height * w / img.width)), Image.BILINEAR)\n",
|
| 561 |
+
"\n",
|
| 562 |
+
"\n",
|
| 563 |
+
"# ---------- real screenshots ----------\n",
|
| 564 |
+
"\n",
|
| 565 |
+
"def fetch_rico(out_dir, n=1500, seed=0):\n",
|
| 566 |
+
" from datasets import load_dataset\n",
|
| 567 |
+
" rng = random.Random(seed)\n",
|
| 568 |
+
" out = Path(out_dir)\n",
|
| 569 |
+
" out.mkdir(parents=True, exist_ok=True)\n",
|
| 570 |
+
" ds = load_dataset(\"rootsautomation/RICO-Screen2Words\", split=\"train\", streaming=True)\n",
|
| 571 |
+
" ds = ds.shuffle(seed=seed, buffer_size=2000)\n",
|
| 572 |
+
" saved = 0\n",
|
| 573 |
+
" for row in ds:\n",
|
| 574 |
+
" img = row[\"image\"].convert(\"RGB\")\n",
|
| 575 |
+
" if img.width < 300:\n",
|
| 576 |
+
" continue\n",
|
| 577 |
+
" img = _jpeg(_to_width(img), rng)\n",
|
| 578 |
+
" img.save(out / f\"rico_{row['screenId']}.png\")\n",
|
| 579 |
+
" saved += 1\n",
|
| 580 |
+
" if saved >= n:\n",
|
| 581 |
+
" break\n",
|
| 582 |
+
" print(f\"saved {saved} RICO screenshots\")\n",
|
| 583 |
+
" return saved\n",
|
| 584 |
+
"\n",
|
| 585 |
+
"\n",
|
| 586 |
+
"# ---------- synthetic chats and posts ----------\n",
|
| 587 |
+
"\n",
|
| 588 |
+
"THEMES = [\n",
|
| 589 |
+
" {\"bg\": (255, 255, 255), \"me\": (0, 122, 255), \"them\": (233, 233, 235), \"me_ink\": (255, 255, 255), \"ink\": (0, 0, 0), \"sub\": (142, 142, 147)},\n",
|
| 590 |
+
" {\"bg\": (0, 0, 0), \"me\": (10, 132, 255), \"them\": (38, 38, 40), \"me_ink\": (255, 255, 255), \"ink\": (255, 255, 255), \"sub\": (142, 142, 147)},\n",
|
| 591 |
+
" {\"bg\": (236, 229, 221), \"me\": (220, 248, 198), \"them\": (255, 255, 255), \"me_ink\": (17, 27, 33), \"ink\": (17, 27, 33), \"sub\": (102, 119, 129)},\n",
|
| 592 |
+
" {\"bg\": (17, 27, 33), \"me\": (0, 92, 75), \"them\": (32, 44, 51), \"me_ink\": (233, 237, 239), \"ink\": (233, 237, 239), \"sub\": (134, 150, 160)},\n",
|
| 593 |
+
" {\"bg\": (255, 255, 255), \"me\": (52, 199, 89), \"them\": (229, 229, 234), \"me_ink\": (255, 255, 255), \"ink\": (0, 0, 0), \"sub\": (120, 120, 128)},\n",
|
| 594 |
+
"]\n",
|
| 595 |
+
"NAMES = [\"Sam\", \"Jordan\", \"Alex\", \"Mia\", \"Chris\", \"Taylor\", \"Riley\", \"Maya\", \"Jay\", \"Noah\", \"Ava\", \"Leo\"]\n",
|
| 596 |
+
"\n",
|
| 597 |
+
"\n",
|
| 598 |
+
"def _clean(t):\n",
|
| 599 |
+
" return \" \".join(w for w in t.replace(\"@user\", \"\").split() if not w.startswith(\"http\"))[:220]\n",
|
| 600 |
+
"\n",
|
| 601 |
+
"\n",
|
| 602 |
+
"def render_chat(texts, rng, fonts):\n",
|
| 603 |
+
" th = rng.choice(THEMES)\n",
|
| 604 |
+
" size = rng.randint(26, 34)\n",
|
| 605 |
+
" font = ImageFont.truetype(rng.choice(fonts), size) if fonts[0] else ImageFont.load_default()\n",
|
| 606 |
+
" small = ImageFont.truetype(rng.choice(fonts), int(size * 0.7)) if fonts[0] else font\n",
|
| 607 |
+
" h = rng.randint(1100, 1500)\n",
|
| 608 |
+
" img = Image.new(\"RGB\", (WIDTH, h), th[\"bg\"])\n",
|
| 609 |
+
" d = ImageDraw.Draw(img)\n",
|
| 610 |
+
" d.text((24, 18), f\"{rng.randint(1, 12)}:{rng.randint(0, 59):02d}\", fill=th[\"ink\"], font=small)\n",
|
| 611 |
+
" d.text((WIDTH // 2 - 40, 70), rng.choice(NAMES), fill=th[\"ink\"], font=font)\n",
|
| 612 |
+
" d.line([(0, 130), (WIDTH, 130)], fill=th[\"sub\"], width=1)\n",
|
| 613 |
+
" y = 160\n",
|
| 614 |
+
" for t in texts:\n",
|
| 615 |
+
" mine = rng.random() < 0.5\n",
|
| 616 |
+
" lines = textwrap.wrap(t, width=rng.randint(22, 30)) or [\"ok\"]\n",
|
| 617 |
+
" lh = size + 8\n",
|
| 618 |
+
" bw = min(WIDTH - 140, max(int(d.textlength(L, font=font)) for L in lines) + 36)\n",
|
| 619 |
+
" bh = lh * len(lines) + 20\n",
|
| 620 |
+
" if y + bh > h - 60:\n",
|
| 621 |
+
" break\n",
|
| 622 |
+
" x = WIDTH - bw - 20 if mine else 20\n",
|
| 623 |
+
" d.rounded_rectangle([x, y, x + bw, y + bh], radius=rng.randint(16, 26), fill=th[\"me\" if mine else \"them\"])\n",
|
| 624 |
+
" for i, L in enumerate(lines):\n",
|
| 625 |
+
" d.text((x + 18, y + 10 + i * lh), L, fill=th[\"me_ink\" if mine else \"ink\"], font=font)\n",
|
| 626 |
+
" y += bh + rng.randint(10, 24)\n",
|
| 627 |
+
" return img\n",
|
| 628 |
+
"\n",
|
| 629 |
+
"\n",
|
| 630 |
+
"def render_post(text, rng, fonts):\n",
|
| 631 |
+
" th = rng.choice([THEMES[0], THEMES[1]])\n",
|
| 632 |
+
" size = rng.randint(28, 34)\n",
|
| 633 |
+
" pick = lambda s: ImageFont.truetype(rng.choice(fonts), s) if fonts[0] else ImageFont.load_default()\n",
|
| 634 |
+
" font, small = pick(size), pick(int(size * 0.75))\n",
|
| 635 |
+
" lines = textwrap.wrap(text, width=rng.randint(30, 38)) or [\"...\"]\n",
|
| 636 |
+
" h = 260 + len(lines) * (size + 10) + rng.randint(40, 200)\n",
|
| 637 |
+
" img = Image.new(\"RGB\", (WIDTH, h), th[\"bg\"])\n",
|
| 638 |
+
" d = ImageDraw.Draw(img)\n",
|
| 639 |
+
" d.ellipse([24, 30, 104, 110], fill=tuple(rng.randint(60, 220) for _ in range(3)))\n",
|
| 640 |
+
" name = rng.choice(NAMES)\n",
|
| 641 |
+
" d.text((124, 36), name, fill=th[\"ink\"], font=font)\n",
|
| 642 |
+
" d.text((124, 36 + size + 6), f\"@{name.lower()}{rng.randint(1, 999)} \u00b7 {rng.randint(1, 23)}h\", fill=th[\"sub\"], font=small)\n",
|
| 643 |
+
" y = 140\n",
|
| 644 |
+
" for L in lines:\n",
|
| 645 |
+
" d.text((24, y), L, fill=th[\"ink\"], font=font)\n",
|
| 646 |
+
" y += size + 10\n",
|
| 647 |
+
" y += 30\n",
|
| 648 |
+
" for i, label in enumerate([\"Reply\", \"Repost\", \"Like\"]):\n",
|
| 649 |
+
" d.text((24 + i * 220, y), f\"{label} {rng.randint(0, 999)}\", fill=th[\"sub\"], font=small)\n",
|
| 650 |
+
" return img\n",
|
| 651 |
+
"\n",
|
| 652 |
+
"\n",
|
| 653 |
+
"def make_synthetic(out_dir, n=600, seed=0):\n",
|
| 654 |
+
" from datasets import load_dataset\n",
|
| 655 |
+
" rng = random.Random(seed)\n",
|
| 656 |
+
" out = Path(out_dir)\n",
|
| 657 |
+
" out.mkdir(parents=True, exist_ok=True)\n",
|
| 658 |
+
" texts = [_clean(t) for t in load_dataset(\"cardiffnlp/tweet_eval\", \"sentiment\", split=\"train\")[\"text\"]]\n",
|
| 659 |
+
" texts = [t for t in texts if len(t) > 15]\n",
|
| 660 |
+
" fonts = _fonts()\n",
|
| 661 |
+
" for i in range(n):\n",
|
| 662 |
+
" if rng.random() < 0.6:\n",
|
| 663 |
+
" img = render_chat(rng.sample(texts, rng.randint(4, 9)), rng, fonts)\n",
|
| 664 |
+
" else:\n",
|
| 665 |
+
" img = render_post(rng.choice(texts), rng, fonts)\n",
|
| 666 |
+
" _jpeg(img, rng).save(out / f\"chat_{i:05d}.png\")\n",
|
| 667 |
+
" print(f\"rendered {n} synthetic chats/posts\")"
|
| 668 |
+
]
|
| 669 |
+
},
|
| 670 |
+
{
|
| 671 |
+
"cell_type": "markdown",
|
| 672 |
+
"metadata": {},
|
| 673 |
+
"source": [
|
| 674 |
+
"## 3. Build the dataset (about 15 minutes)"
|
| 675 |
+
]
|
| 676 |
+
},
|
| 677 |
+
{
|
| 678 |
+
"cell_type": "code",
|
| 679 |
+
"metadata": {},
|
| 680 |
+
"execution_count": null,
|
| 681 |
+
"outputs": [],
|
| 682 |
+
"source": [
|
| 683 |
+
"import screen_autodata as ad\n",
|
| 684 |
+
"ad.fetch_rico(f'{DATA}/real', n=1500, seed=0)\n",
|
| 685 |
+
"ad.make_synthetic(f'{DATA}/real', n=600, seed=0)"
|
| 686 |
+
]
|
| 687 |
+
},
|
| 688 |
+
{
|
| 689 |
+
"cell_type": "code",
|
| 690 |
+
"metadata": {},
|
| 691 |
+
"execution_count": null,
|
| 692 |
+
"outputs": [],
|
| 693 |
+
"source": [
|
| 694 |
+
"import easyocr, screen_edits as se\n",
|
| 695 |
+
"reader = easyocr.Reader(['en'], gpu=True)\n",
|
| 696 |
+
"rows = se.make_edited_set(f'{DATA}/real', f'{DATA}/edited', per_image=2, seed=0, reader=reader)"
|
| 697 |
+
]
|
| 698 |
+
},
|
| 699 |
+
{
|
| 700 |
+
"cell_type": "code",
|
| 701 |
+
"metadata": {},
|
| 702 |
+
"execution_count": null,
|
| 703 |
+
"outputs": [],
|
| 704 |
+
"source": [
|
| 705 |
+
"# quick look: a real image next to its edited copy (red box = where the edit is)\n",
|
| 706 |
+
"import json, matplotlib.pyplot as plt\n",
|
| 707 |
+
"from PIL import Image, ImageDraw\n",
|
| 708 |
+
"edits = json.load(open(f'{DATA}/edited/edits.json'))\n",
|
| 709 |
+
"fig, ax = plt.subplots(2, 4, figsize=(16, 12))\n",
|
| 710 |
+
"for k, e in enumerate(edits[:8:2]):\n",
|
| 711 |
+
" ax[0, k].imshow(Image.open(f\"{DATA}/real/{e['source']}\")); ax[0, k].set_title('real'); ax[0, k].axis('off')\n",
|
| 712 |
+
" im = Image.open(f\"{DATA}/edited/{e['path']}\").copy(); ImageDraw.Draw(im).rectangle(e['edit_box'], outline='red', width=4)\n",
|
| 713 |
+
" ax[1, k].imshow(im); ax[1, k].set_title('edited'); ax[1, k].axis('off')\n",
|
| 714 |
+
"plt.tight_layout(); plt.show()"
|
| 715 |
+
]
|
| 716 |
+
},
|
| 717 |
+
{
|
| 718 |
+
"cell_type": "markdown",
|
| 719 |
+
"metadata": {},
|
| 720 |
+
"source": [
|
| 721 |
+
"## 4. Train (about 15 to 25 minutes)"
|
| 722 |
+
]
|
| 723 |
+
},
|
| 724 |
+
{
|
| 725 |
+
"cell_type": "code",
|
| 726 |
+
"metadata": {},
|
| 727 |
+
"execution_count": null,
|
| 728 |
+
"outputs": [],
|
| 729 |
+
"source": [
|
| 730 |
+
"import screen_train as st\n",
|
| 731 |
+
"model, results = st.train(DATA, RUN, arch='efficientnet_b0', epochs=6, batch=48, workers=2)\n",
|
| 732 |
+
"print('edited threshold:', results['thresholds']['edited'])"
|
| 733 |
+
]
|
| 734 |
+
},
|
| 735 |
+
{
|
| 736 |
+
"cell_type": "markdown",
|
| 737 |
+
"metadata": {},
|
| 738 |
+
"source": [
|
| 739 |
+
"## 5. Upload to Hugging Face\n",
|
| 740 |
+
"\n",
|
| 741 |
+
"Use a token with **Write** access (huggingface.co > Settings > Access Tokens). The repo is private."
|
| 742 |
+
]
|
| 743 |
+
},
|
| 744 |
+
{
|
| 745 |
+
"cell_type": "code",
|
| 746 |
+
"metadata": {},
|
| 747 |
+
"execution_count": null,
|
| 748 |
+
"outputs": [],
|
| 749 |
+
"source": [
|
| 750 |
+
"from huggingface_hub import notebook_login\n",
|
| 751 |
+
"notebook_login()"
|
| 752 |
+
]
|
| 753 |
+
},
|
| 754 |
+
{
|
| 755 |
+
"cell_type": "code",
|
| 756 |
+
"metadata": {},
|
| 757 |
+
"execution_count": null,
|
| 758 |
+
"outputs": [],
|
| 759 |
+
"source": [
|
| 760 |
+
"from huggingface_hub import HfApi, create_repo, whoami\n",
|
| 761 |
+
"REPO = f\"{whoami()['name']}/tell-screenshotcheck\"\n",
|
| 762 |
+
"create_repo(REPO, private=True, exist_ok=True)\n",
|
| 763 |
+
"open(f'{RUN}/README.md', 'w').write('''# TELL ScreenshotCheck (edit detector)\n",
|
| 764 |
+
"\n",
|
| 765 |
+
"EfficientNet-B0 on 256px patches. Classes: real, generated (not trained in this version), edited.\n",
|
| 766 |
+
"Real: RICO app screenshots (CC BY 4.0) + synthetic chat/post renders (tweet_eval text).\n",
|
| 767 |
+
"Edited: automatic text-box replacement with EasyOCR. Part of TELL, 2026 Congressional App Challenge.\n",
|
| 768 |
+
"''')\n",
|
| 769 |
+
"HfApi().upload_folder(folder_path=RUN, repo_id=REPO,\n",
|
| 770 |
+
" allow_patterns=['config.json', 'model.safetensors', 'results.json', 'README.md'])\n",
|
| 771 |
+
"print('Done. Set SCREENSHOT_MODEL =', REPO, 'in .env and in your Space secrets (plus HF_TOKEN, since the repo is private).')"
|
| 772 |
+
]
|
| 773 |
+
}
|
| 774 |
+
],
|
| 775 |
+
"metadata": {
|
| 776 |
+
"accelerator": "GPU",
|
| 777 |
+
"colab": {
|
| 778 |
+
"gpuType": "T4"
|
| 779 |
+
},
|
| 780 |
+
"kernelspec": {
|
| 781 |
+
"name": "python3",
|
| 782 |
+
"display_name": "Python 3"
|
| 783 |
+
}
|
| 784 |
+
},
|
| 785 |
+
"nbformat": 4,
|
| 786 |
+
"nbformat_minor": 0
|
| 787 |
+
}
|
notebooks/TELL_ScreenshotCheck_Training.ipynb
ADDED
|
@@ -0,0 +1,676 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"cells": [
|
| 3 |
+
{
|
| 4 |
+
"cell_type": "markdown",
|
| 5 |
+
"metadata": {},
|
| 6 |
+
"source": [
|
| 7 |
+
"# TELL: ScreenshotCheck training\n",
|
| 8 |
+
"\n",
|
| 9 |
+
"Trains the model that spots **generated** (fake-chat website) and **edited** screenshots,\n",
|
| 10 |
+
"and draws a heatmap of where an edit is.\n",
|
| 11 |
+
"\n",
|
| 12 |
+
"## Collect the data first (on your phone and laptop)\n",
|
| 13 |
+
"\n",
|
| 14 |
+
"Put these in Google Drive at `MyDrive/TELL/screens/`:\n",
|
| 15 |
+
"\n",
|
| 16 |
+
"| Folder | What | How many | Rules |\n",
|
| 17 |
+
"| --- | --- | --- | --- |\n",
|
| 18 |
+
"| `real/` | Real screenshots of chats and posts (yours, and friends who say yes) | 150+ (more is better) | **Blur names, photos, and phone numbers first.** Don't upload anything private. Mix apps (iMessage, Instagram DMs, Snapchat, Discord). |\n",
|
| 19 |
+
"| `generated/` | Fakes made on free fake-chat websites | 100+ | Use **2 or 3 different sites**. Name files by site: `siteA_001.png`, `siteB_001.png`. The last site alphabetically is held out for testing. |\n",
|
| 20 |
+
"| `edited/` | Made automatically in step 3 below | (auto) | Don't add files yourself. |\n",
|
| 21 |
+
"\n",
|
| 22 |
+
"Screenshots should be original files (AirDrop or Google Photos original quality), not re-sent through\n",
|
| 23 |
+
"a chat app, so the real/fake difference isn't just compression.\n",
|
| 24 |
+
"\n",
|
| 25 |
+
"**Runtime > Change runtime type > T4 GPU.**"
|
| 26 |
+
]
|
| 27 |
+
},
|
| 28 |
+
{
|
| 29 |
+
"cell_type": "code",
|
| 30 |
+
"metadata": {},
|
| 31 |
+
"execution_count": null,
|
| 32 |
+
"outputs": [],
|
| 33 |
+
"source": [
|
| 34 |
+
"!nvidia-smi --query-gpu=name,memory.total --format=csv\n",
|
| 35 |
+
"!pip -q install timm safetensors scikit-learn huggingface_hub easyocr"
|
| 36 |
+
]
|
| 37 |
+
},
|
| 38 |
+
{
|
| 39 |
+
"cell_type": "code",
|
| 40 |
+
"metadata": {},
|
| 41 |
+
"execution_count": null,
|
| 42 |
+
"outputs": [],
|
| 43 |
+
"source": [
|
| 44 |
+
"from google.colab import drive\n",
|
| 45 |
+
"drive.mount('/content/drive')\n",
|
| 46 |
+
"import os\n",
|
| 47 |
+
"SCREENS = '/content/drive/MyDrive/TELL/screens'\n",
|
| 48 |
+
"RUN = '/content/drive/MyDrive/TELL/screenshotcheck/run1'\n",
|
| 49 |
+
"os.makedirs(RUN, exist_ok=True)"
|
| 50 |
+
]
|
| 51 |
+
},
|
| 52 |
+
{
|
| 53 |
+
"cell_type": "markdown",
|
| 54 |
+
"metadata": {},
|
| 55 |
+
"source": [
|
| 56 |
+
"## 1. Data audit (fix problems here before training)"
|
| 57 |
+
]
|
| 58 |
+
},
|
| 59 |
+
{
|
| 60 |
+
"cell_type": "code",
|
| 61 |
+
"metadata": {},
|
| 62 |
+
"execution_count": null,
|
| 63 |
+
"outputs": [],
|
| 64 |
+
"source": [
|
| 65 |
+
"from pathlib import Path\n",
|
| 66 |
+
"from collections import Counter\n",
|
| 67 |
+
"from PIL import Image\n",
|
| 68 |
+
"for d in ['real', 'generated']:\n",
|
| 69 |
+
" files = [p for p in Path(SCREENS, d).iterdir() if p.suffix.lower() in {'.png', '.jpg', '.jpeg'}]\n",
|
| 70 |
+
" sizes = Counter(Image.open(p).size for p in files[:300])\n",
|
| 71 |
+
" print(f'{d}: {len(files)} files, most common sizes: {sizes.most_common(3)}')\n",
|
| 72 |
+
"sites = Counter(p.name.split('_')[0] for p in Path(SCREENS, 'generated').iterdir())\n",
|
| 73 |
+
"print('generator sites:', dict(sites))\n",
|
| 74 |
+
"if len(sites) < 2: print('WARNING: use at least 2 generator sites so one can be held out for testing')"
|
| 75 |
+
]
|
| 76 |
+
},
|
| 77 |
+
{
|
| 78 |
+
"cell_type": "markdown",
|
| 79 |
+
"metadata": {},
|
| 80 |
+
"source": [
|
| 81 |
+
"## 2. Write the training code (same files as `training/` in the repo)"
|
| 82 |
+
]
|
| 83 |
+
},
|
| 84 |
+
{
|
| 85 |
+
"cell_type": "code",
|
| 86 |
+
"metadata": {},
|
| 87 |
+
"execution_count": null,
|
| 88 |
+
"outputs": [],
|
| 89 |
+
"source": [
|
| 90 |
+
"%%writefile screen_edits.py\n",
|
| 91 |
+
"\"\"\"\n",
|
| 92 |
+
"TELL ScreenshotCheck: automatically make EDITED screenshots from real ones.\n",
|
| 93 |
+
"\n",
|
| 94 |
+
"For each real screenshot:\n",
|
| 95 |
+
" 1. OCR finds the text boxes (messages, names, times).\n",
|
| 96 |
+
" 2. Pick one box, cover it with the bubble's background color.\n",
|
| 97 |
+
" 3. Type replacement text in a similar size and color.\n",
|
| 98 |
+
" 4. Save the edited image and the exact edit box (used for training and for\n",
|
| 99 |
+
" checking whether the heatmap points at the right place).\n",
|
| 100 |
+
"\n",
|
| 101 |
+
"Why this is realistic: this is exactly how most people fake a screenshot\n",
|
| 102 |
+
"(inspect element or a photo editor: cover a message, type a new one).\n",
|
| 103 |
+
"Known limit to document: the replacement font (DejaVu Sans) differs from\n",
|
| 104 |
+
"iOS/Android system fonts, so the model may partly learn \"wrong font\".\n",
|
| 105 |
+
"That is still a real signal people miss, but say it honestly.\n",
|
| 106 |
+
"\"\"\"\n",
|
| 107 |
+
"\n",
|
| 108 |
+
"import json\n",
|
| 109 |
+
"import random\n",
|
| 110 |
+
"from pathlib import Path\n",
|
| 111 |
+
"\n",
|
| 112 |
+
"import numpy as np\n",
|
| 113 |
+
"from PIL import Image, ImageDraw, ImageFont\n",
|
| 114 |
+
"\n",
|
| 115 |
+
"SWAPS = [\"no\", \"yes\", \"lol ok\", \"never\", \"i did it\", \"not me\", \"tomorrow\", \"sure\",\n",
|
| 116 |
+
" \"i hate this\", \"ok fine\", \"it was him\", \"i'm leaving\", \"deal\", \"who cares\",\n",
|
| 117 |
+
" \"be there at 5\", \"send it\", \"nope\", \"for real\", \"i lied\", \"i know\"]\n",
|
| 118 |
+
"FONT_PATHS = [\"/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf\",\n",
|
| 119 |
+
" \"/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf\"]\n",
|
| 120 |
+
"\n",
|
| 121 |
+
"\n",
|
| 122 |
+
"def find_text_boxes(img, reader=None):\n",
|
| 123 |
+
" \"\"\"Returns [(x0, y0, x1, y1, text)] using EasyOCR. Pass a shared reader for speed.\"\"\"\n",
|
| 124 |
+
" import easyocr\n",
|
| 125 |
+
" reader = reader or easyocr.Reader([\"en\"], gpu=True)\n",
|
| 126 |
+
" out = []\n",
|
| 127 |
+
" for box, text, conf in reader.readtext(np.array(img.convert(\"RGB\"))):\n",
|
| 128 |
+
" xs, ys = [p[0] for p in box], [p[1] for p in box]\n",
|
| 129 |
+
" if conf > 0.3 and (max(ys) - min(ys)) >= 12 and len(text) >= 2:\n",
|
| 130 |
+
" out.append((int(min(xs)), int(min(ys)), int(max(xs)), int(max(ys)), text))\n",
|
| 131 |
+
" return out\n",
|
| 132 |
+
"\n",
|
| 133 |
+
"\n",
|
| 134 |
+
"def _font(size, rng):\n",
|
| 135 |
+
" for p in rng.sample(FONT_PATHS, len(FONT_PATHS)):\n",
|
| 136 |
+
" try:\n",
|
| 137 |
+
" return ImageFont.truetype(p, size)\n",
|
| 138 |
+
" except OSError:\n",
|
| 139 |
+
" continue\n",
|
| 140 |
+
" return ImageFont.load_default()\n",
|
| 141 |
+
"\n",
|
| 142 |
+
"\n",
|
| 143 |
+
"def edit_one(img, box, rng):\n",
|
| 144 |
+
" \"\"\"Cover one text box and write new text. Returns (edited_img, edit_box).\"\"\"\n",
|
| 145 |
+
" img = img.convert(\"RGB\").copy()\n",
|
| 146 |
+
" x0, y0, x1, y1, _ = box\n",
|
| 147 |
+
" pad = 3\n",
|
| 148 |
+
" a = np.asarray(img)\n",
|
| 149 |
+
" h, w = a.shape[:2]\n",
|
| 150 |
+
" ring = np.concatenate([\n",
|
| 151 |
+
" a[max(0, y0 - pad), max(0, x0 - pad):min(w, x1 + pad)],\n",
|
| 152 |
+
" a[min(h - 1, y1 + pad), max(0, x0 - pad):min(w, x1 + pad)],\n",
|
| 153 |
+
" ])\n",
|
| 154 |
+
" bg = tuple(int(v) for v in np.median(ring, axis=0))\n",
|
| 155 |
+
" inner = a[y0:y1, x0:x1].reshape(-1, 3)\n",
|
| 156 |
+
" lum = inner.mean(1)\n",
|
| 157 |
+
" bg_lum = sum(bg) / 3\n",
|
| 158 |
+
" ink = inner[np.argmin(lum)] if bg_lum > 128 else inner[np.argmax(lum)]\n",
|
| 159 |
+
" d = ImageDraw.Draw(img)\n",
|
| 160 |
+
" d.rectangle([x0 - 1, y0 - 1, x1 + 1, y1 + 1], fill=bg)\n",
|
| 161 |
+
" size = max(10, int((y1 - y0) * rng.uniform(0.75, 0.9)))\n",
|
| 162 |
+
" text = rng.choice(SWAPS)\n",
|
| 163 |
+
" d.text((x0, y0 + (y1 - y0 - size) // 2), text, fill=tuple(int(v) for v in ink),\n",
|
| 164 |
+
" font=_font(size, rng))\n",
|
| 165 |
+
" tw = int(d.textlength(text, font=_font(size, rng)))\n",
|
| 166 |
+
" return img, [x0 - 1, y0 - 1, max(x1, x0 + tw) + 1, y1 + 1]\n",
|
| 167 |
+
"\n",
|
| 168 |
+
"\n",
|
| 169 |
+
"def make_edited_set(real_dir, out_dir, per_image=2, seed=0, reader=None):\n",
|
| 170 |
+
" \"\"\"Creates edited/ images plus edits.json with source and edit box for each.\"\"\"\n",
|
| 171 |
+
" rng = random.Random(seed)\n",
|
| 172 |
+
" real_dir, out_dir = Path(real_dir), Path(out_dir)\n",
|
| 173 |
+
" out_dir.mkdir(parents=True, exist_ok=True)\n",
|
| 174 |
+
" rows = []\n",
|
| 175 |
+
" for p in sorted(real_dir.glob(\"*\")):\n",
|
| 176 |
+
" if p.suffix.lower() not in {\".png\", \".jpg\", \".jpeg\"}:\n",
|
| 177 |
+
" continue\n",
|
| 178 |
+
" img = Image.open(p).convert(\"RGB\")\n",
|
| 179 |
+
" boxes = find_text_boxes(img, reader)\n",
|
| 180 |
+
" rng.shuffle(boxes)\n",
|
| 181 |
+
" for k, box in enumerate(boxes[:per_image]):\n",
|
| 182 |
+
" edited, ebox = edit_one(img, box, rng)\n",
|
| 183 |
+
" name = f\"{p.stem}__edit{k}.png\"\n",
|
| 184 |
+
" edited.save(out_dir / name)\n",
|
| 185 |
+
" rows.append({\"path\": name, \"source\": p.name, \"edit_box\": ebox})\n",
|
| 186 |
+
" (out_dir / \"edits.json\").write_text(json.dumps(rows, indent=1))\n",
|
| 187 |
+
" print(f\"made {len(rows)} edited screenshots from {real_dir}\")\n",
|
| 188 |
+
" return rows"
|
| 189 |
+
]
|
| 190 |
+
},
|
| 191 |
+
{
|
| 192 |
+
"cell_type": "code",
|
| 193 |
+
"metadata": {},
|
| 194 |
+
"execution_count": null,
|
| 195 |
+
"outputs": [],
|
| 196 |
+
"source": [
|
| 197 |
+
"%%writefile screen_train.py\n",
|
| 198 |
+
"\"\"\"\n",
|
| 199 |
+
"TELL ScreenshotCheck training.\n",
|
| 200 |
+
"\n",
|
| 201 |
+
"Three classes: real, generated (made on a fake-chat website), edited (real with\n",
|
| 202 |
+
"one message replaced). Trains on 256x256 PATCHES at full resolution, because\n",
|
| 203 |
+
"resizing the whole screenshot would erase the tiny pixel clues we need.\n",
|
| 204 |
+
"\n",
|
| 205 |
+
"Data layout on Google Drive (see TRAINING.md):\n",
|
| 206 |
+
" screens/real/*.png your + friends' screenshots (blurred names)\n",
|
| 207 |
+
" screens/generated/<site>_*.png fakes made on fake-chat sites, filename starts with site name\n",
|
| 208 |
+
" screens/edited/*.png + edits.json made by screen_edits.py from real/\n",
|
| 209 |
+
"\n",
|
| 210 |
+
"Leakage rules (important to document):\n",
|
| 211 |
+
" - A real screenshot and every edited copy of it ALWAYS land in the same split,\n",
|
| 212 |
+
" otherwise the model \"recognizes\" the original and the test score is fake.\n",
|
| 213 |
+
" - One generator site is held out completely for testing, so the test number\n",
|
| 214 |
+
" says whether TELL catches a generator it has never seen.\n",
|
| 215 |
+
"\"\"\"\n",
|
| 216 |
+
"\n",
|
| 217 |
+
"import io\n",
|
| 218 |
+
"import json\n",
|
| 219 |
+
"import random\n",
|
| 220 |
+
"from pathlib import Path\n",
|
| 221 |
+
"\n",
|
| 222 |
+
"import numpy as np\n",
|
| 223 |
+
"import torch\n",
|
| 224 |
+
"import torch.nn as nn\n",
|
| 225 |
+
"from PIL import Image\n",
|
| 226 |
+
"\n",
|
| 227 |
+
"CLASSES = [\"real\", \"generated\", \"edited\"]\n",
|
| 228 |
+
"PATCH, STRIDE = 256, 192\n",
|
| 229 |
+
"MEAN = (0.485, 0.456, 0.406)\n",
|
| 230 |
+
"STD = (0.229, 0.224, 0.225)\n",
|
| 231 |
+
"MIN_STD = 8.0 # skip nearly blank patches (plain background tells us nothing)\n",
|
| 232 |
+
"\n",
|
| 233 |
+
"\n",
|
| 234 |
+
"# ---------- Building the list of images ----------\n",
|
| 235 |
+
"\n",
|
| 236 |
+
"def build_index(root, holdout_site=None, seed=42, val=0.15, test=0.15):\n",
|
| 237 |
+
" root = Path(root)\n",
|
| 238 |
+
" rng = random.Random(seed)\n",
|
| 239 |
+
" reals = sorted(p.name for p in (root / \"real\").iterdir() if p.suffix.lower() in {\".png\", \".jpg\", \".jpeg\"})\n",
|
| 240 |
+
" edits = json.loads((root / \"edited\" / \"edits.json\").read_text())\n",
|
| 241 |
+
" gens = sorted(p.name for p in (root / \"generated\").iterdir() if p.suffix.lower() in {\".png\", \".jpg\", \".jpeg\"})\n",
|
| 242 |
+
"\n",
|
| 243 |
+
" # split REAL by source; edits follow their source\n",
|
| 244 |
+
" rng.shuffle(reals)\n",
|
| 245 |
+
" nv, nt = int(len(reals) * val), int(len(reals) * test)\n",
|
| 246 |
+
" real_split = {n: \"val\" for n in reals[:nv]}\n",
|
| 247 |
+
" real_split.update({n: \"test\" for n in reals[nv:nv + nt]})\n",
|
| 248 |
+
" real_split.update({n: \"train\" for n in reals[nv + nt:]})\n",
|
| 249 |
+
"\n",
|
| 250 |
+
" rows = [{\"path\": f\"real/{n}\", \"label\": \"real\", \"split\": real_split[n], \"group\": n} for n in reals]\n",
|
| 251 |
+
" rows += [{\"path\": f\"edited/{e['path']}\", \"label\": \"edited\", \"split\": real_split[e[\"source\"]],\n",
|
| 252 |
+
" \"group\": e[\"source\"], \"edit_box\": e[\"edit_box\"]} for e in edits if e[\"source\"] in real_split]\n",
|
| 253 |
+
"\n",
|
| 254 |
+
" # generated: hold one site out for test, split the rest\n",
|
| 255 |
+
" sites = sorted({g.split(\"_\")[0] for g in gens})\n",
|
| 256 |
+
" holdout_site = holdout_site or (sites[-1] if len(sites) > 1 else None)\n",
|
| 257 |
+
" others = [g for g in gens if g.split(\"_\")[0] != holdout_site]\n",
|
| 258 |
+
" rng.shuffle(others)\n",
|
| 259 |
+
" gv = int(len(others) * val)\n",
|
| 260 |
+
" for i, g in enumerate(others):\n",
|
| 261 |
+
" rows.append({\"path\": f\"generated/{g}\", \"label\": \"generated\", \"split\": \"val\" if i < gv else \"train\",\n",
|
| 262 |
+
" \"group\": g, \"site\": g.split(\"_\")[0]})\n",
|
| 263 |
+
" for g in gens:\n",
|
| 264 |
+
" if g.split(\"_\")[0] == holdout_site:\n",
|
| 265 |
+
" rows.append({\"path\": f\"generated/{g}\", \"label\": \"generated\", \"split\": \"test\",\n",
|
| 266 |
+
" \"group\": g, \"site\": g.split(\"_\")[0], \"unseen_site\": True})\n",
|
| 267 |
+
" return rows, holdout_site\n",
|
| 268 |
+
"\n",
|
| 269 |
+
"\n",
|
| 270 |
+
"# ---------- Augmentation applied to the WHOLE image, same for every class ----------\n",
|
| 271 |
+
"\n",
|
| 272 |
+
"def realistic_noise(img, box, rng):\n",
|
| 273 |
+
" \"\"\"Social media recompresses and rescales screenshots. Apply to all classes\n",
|
| 274 |
+
" equally so the model can't cheat by learning 'JPEG means fake'.\"\"\"\n",
|
| 275 |
+
" if rng.random() < 0.3:\n",
|
| 276 |
+
" f = rng.uniform(0.75, 1.0)\n",
|
| 277 |
+
" img = img.resize((int(img.width * f), int(img.height * f)), Image.BILINEAR)\n",
|
| 278 |
+
" if box:\n",
|
| 279 |
+
" box = [int(v * f) for v in box]\n",
|
| 280 |
+
" if rng.random() < 0.5:\n",
|
| 281 |
+
" buf = io.BytesIO()\n",
|
| 282 |
+
" img.save(buf, \"JPEG\", quality=rng.randint(60, 95))\n",
|
| 283 |
+
" buf.seek(0)\n",
|
| 284 |
+
" img = Image.open(buf).convert(\"RGB\")\n",
|
| 285 |
+
" return img, box\n",
|
| 286 |
+
"\n",
|
| 287 |
+
"\n",
|
| 288 |
+
"def _norm(patch):\n",
|
| 289 |
+
" a = (np.asarray(patch, dtype=np.float32) / 255.0 - MEAN) / STD\n",
|
| 290 |
+
" return torch.from_numpy(a.transpose(2, 0, 1)).float()\n",
|
| 291 |
+
"\n",
|
| 292 |
+
"\n",
|
| 293 |
+
"def _overlap(px, py, box):\n",
|
| 294 |
+
" x0, y0, x1, y1 = box\n",
|
| 295 |
+
" ix = max(0, min(px + PATCH, x1) - max(px, x0))\n",
|
| 296 |
+
" iy = max(0, min(py + PATCH, y1) - max(py, y0))\n",
|
| 297 |
+
" area = max(1, (x1 - x0) * (y1 - y0))\n",
|
| 298 |
+
" return ix * iy / area\n",
|
| 299 |
+
"\n",
|
| 300 |
+
"\n",
|
| 301 |
+
"class PatchDataset(torch.utils.data.Dataset):\n",
|
| 302 |
+
" \"\"\"Each item = one random informative patch from one image.\"\"\"\n",
|
| 303 |
+
"\n",
|
| 304 |
+
" def __init__(self, rows, root, train=True, repeats=8):\n",
|
| 305 |
+
" self.rows, self.root, self.train, self.repeats = rows, Path(root), train, repeats\n",
|
| 306 |
+
"\n",
|
| 307 |
+
" def __len__(self):\n",
|
| 308 |
+
" return len(self.rows) * self.repeats\n",
|
| 309 |
+
"\n",
|
| 310 |
+
" def __getitem__(self, i):\n",
|
| 311 |
+
" r = self.rows[i % len(self.rows)]\n",
|
| 312 |
+
" rng = random.Random() if self.train else random.Random(i)\n",
|
| 313 |
+
" img = Image.open(self.root / r[\"path\"]).convert(\"RGB\")\n",
|
| 314 |
+
" box = r.get(\"edit_box\")\n",
|
| 315 |
+
" if self.train:\n",
|
| 316 |
+
" img, box = realistic_noise(img, box, rng)\n",
|
| 317 |
+
" if img.width < PATCH or img.height < PATCH:\n",
|
| 318 |
+
" img = img.resize((max(img.width, PATCH), max(img.height, PATCH)))\n",
|
| 319 |
+
" max_x, max_y = img.width - PATCH, img.height - PATCH\n",
|
| 320 |
+
" patch = None\n",
|
| 321 |
+
" for _ in range(15):\n",
|
| 322 |
+
" if box: # edited: the patch must contain most of the edit\n",
|
| 323 |
+
" lo_x = min(max(0, box[2] - PATCH), max_x)\n",
|
| 324 |
+
" hi_x = max(lo_x, min(box[0], max_x))\n",
|
| 325 |
+
" lo_y = min(max(0, box[3] - PATCH), max_y)\n",
|
| 326 |
+
" hi_y = max(lo_y, min(box[1], max_y))\n",
|
| 327 |
+
" px, py = rng.randint(lo_x, hi_x), rng.randint(lo_y, hi_y)\n",
|
| 328 |
+
" if _overlap(px, py, box) < 0.5:\n",
|
| 329 |
+
" continue\n",
|
| 330 |
+
" else:\n",
|
| 331 |
+
" px, py = rng.randint(0, max_x), rng.randint(0, max_y)\n",
|
| 332 |
+
" cand = img.crop((px, py, px + PATCH, py + PATCH))\n",
|
| 333 |
+
" if np.asarray(cand).std() >= MIN_STD:\n",
|
| 334 |
+
" patch = cand\n",
|
| 335 |
+
" break\n",
|
| 336 |
+
" if patch is None: # fallback: center on the edit, or the image center\n",
|
| 337 |
+
" cx = (box[0] + box[2]) // 2 if box else img.width // 2\n",
|
| 338 |
+
" cy = (box[1] + box[3]) // 2 if box else img.height // 2\n",
|
| 339 |
+
" px = min(max(0, cx - PATCH // 2), max_x)\n",
|
| 340 |
+
" py = min(max(0, cy - PATCH // 2), max_y)\n",
|
| 341 |
+
" patch = img.crop((px, py, px + PATCH, py + PATCH))\n",
|
| 342 |
+
" return _norm(patch), CLASSES.index(r[\"label\"])\n",
|
| 343 |
+
"\n",
|
| 344 |
+
"\n",
|
| 345 |
+
"# ---------- Whole-image prediction by tiling (MUST match tools/screenshotcheck.py) ----------\n",
|
| 346 |
+
"\n",
|
| 347 |
+
"@torch.no_grad()\n",
|
| 348 |
+
"def tile_predict(model, img, device=\"cpu\", batch=32):\n",
|
| 349 |
+
" \"\"\"Returns per-patch probabilities [(x, y, p_real, p_gen, p_edit)].\"\"\"\n",
|
| 350 |
+
" img = img.convert(\"RGB\")\n",
|
| 351 |
+
" if img.width < PATCH or img.height < PATCH:\n",
|
| 352 |
+
" img = img.resize((max(img.width, PATCH), max(img.height, PATCH)))\n",
|
| 353 |
+
" xs = list(range(0, img.width - PATCH + 1, STRIDE)) or [0]\n",
|
| 354 |
+
" ys = list(range(0, img.height - PATCH + 1, STRIDE)) or [0]\n",
|
| 355 |
+
" coords, tensors = [], []\n",
|
| 356 |
+
" for y in ys:\n",
|
| 357 |
+
" for x in xs:\n",
|
| 358 |
+
" p = img.crop((x, y, x + PATCH, y + PATCH))\n",
|
| 359 |
+
" if np.asarray(p).std() >= MIN_STD:\n",
|
| 360 |
+
" coords.append((x, y))\n",
|
| 361 |
+
" tensors.append(_norm(p))\n",
|
| 362 |
+
" out = []\n",
|
| 363 |
+
" model.eval()\n",
|
| 364 |
+
" for i in range(0, len(tensors), batch):\n",
|
| 365 |
+
" probs = torch.softmax(model(torch.stack(tensors[i:i + batch]).to(device)).float(), 1).cpu()\n",
|
| 366 |
+
" out += [(*coords[i + k], *probs[k].tolist()) for k in range(len(probs))]\n",
|
| 367 |
+
" return out\n",
|
| 368 |
+
"\n",
|
| 369 |
+
"\n",
|
| 370 |
+
"def image_scores(patches):\n",
|
| 371 |
+
" \"\"\"Turn patch probabilities into two image-level scores.\"\"\"\n",
|
| 372 |
+
" if not patches:\n",
|
| 373 |
+
" return 0.0, 0.0\n",
|
| 374 |
+
" gen = float(np.mean([p[3] for p in patches]))\n",
|
| 375 |
+
" edit = float(np.mean(sorted([p[4] for p in patches], reverse=True)[:2]))\n",
|
| 376 |
+
" return gen, edit\n",
|
| 377 |
+
"\n",
|
| 378 |
+
"\n",
|
| 379 |
+
"def decide(gen, edit, t_gen, t_edit):\n",
|
| 380 |
+
" if gen >= t_gen:\n",
|
| 381 |
+
" return \"generated\"\n",
|
| 382 |
+
" if edit >= t_edit:\n",
|
| 383 |
+
" return \"edited\"\n",
|
| 384 |
+
" return \"real\"\n",
|
| 385 |
+
"\n",
|
| 386 |
+
"\n",
|
| 387 |
+
"# ---------- Training ----------\n",
|
| 388 |
+
"\n",
|
| 389 |
+
"def build_model(arch=\"efficientnet_b0\", pretrained=True):\n",
|
| 390 |
+
" import timm\n",
|
| 391 |
+
" return timm.create_model(arch, pretrained=pretrained, num_classes=len(CLASSES))\n",
|
| 392 |
+
"\n",
|
| 393 |
+
"\n",
|
| 394 |
+
"def train(root, out_dir, arch=\"efficientnet_b0\", epochs=8, batch=48, lr=3e-4, workers=2,\n",
|
| 395 |
+
" holdout_site=None, seed=42, pretrained=True):\n",
|
| 396 |
+
" torch.manual_seed(seed)\n",
|
| 397 |
+
" root, out_dir = Path(root), Path(out_dir)\n",
|
| 398 |
+
" out_dir.mkdir(parents=True, exist_ok=True)\n",
|
| 399 |
+
" rows, holdout_site = build_index(root, holdout_site, seed)\n",
|
| 400 |
+
" (out_dir / \"index.json\").write_text(json.dumps(rows, indent=1))\n",
|
| 401 |
+
" part = lambda s: [r for r in rows if r[\"split\"] == s]\n",
|
| 402 |
+
" counts = {s: {c: sum(r[\"label\"] == c for r in part(s)) for c in CLASSES} for s in [\"train\", \"val\", \"test\"]}\n",
|
| 403 |
+
" print(\"images per split:\", counts, \"| held-out generator site:\", holdout_site)\n",
|
| 404 |
+
"\n",
|
| 405 |
+
" device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n",
|
| 406 |
+
" tr = part(\"train\")\n",
|
| 407 |
+
" # balance classes: each class contributes equally per epoch\n",
|
| 408 |
+
" weights = [1.0 / sum(x[\"label\"] == r[\"label\"] for x in tr) for r in tr] * 8\n",
|
| 409 |
+
" sampler = torch.utils.data.WeightedRandomSampler(weights, num_samples=len(weights), replacement=True)\n",
|
| 410 |
+
" dl = torch.utils.data.DataLoader(PatchDataset(tr, root, True, 8), batch_size=batch,\n",
|
| 411 |
+
" sampler=sampler, num_workers=workers)\n",
|
| 412 |
+
" dl_val = torch.utils.data.DataLoader(PatchDataset(part(\"val\"), root, False, 4), batch_size=batch,\n",
|
| 413 |
+
" num_workers=workers)\n",
|
| 414 |
+
"\n",
|
| 415 |
+
" model = build_model(arch, pretrained).to(device)\n",
|
| 416 |
+
" opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)\n",
|
| 417 |
+
" sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=epochs * len(dl), pct_start=0.1)\n",
|
| 418 |
+
" scaler = torch.amp.GradScaler(enabled=device == \"cuda\")\n",
|
| 419 |
+
" loss_fn = nn.CrossEntropyLoss(label_smoothing=0.05)\n",
|
| 420 |
+
"\n",
|
| 421 |
+
" from sklearn.metrics import f1_score\n",
|
| 422 |
+
" best, history = -1, []\n",
|
| 423 |
+
" for ep in range(epochs):\n",
|
| 424 |
+
" model.train()\n",
|
| 425 |
+
" tot = 0.0\n",
|
| 426 |
+
" for x, y in dl:\n",
|
| 427 |
+
" x, y = x.to(device), y.to(device)\n",
|
| 428 |
+
" with torch.autocast(device_type=device, enabled=device == \"cuda\"):\n",
|
| 429 |
+
" loss = loss_fn(model(x), y)\n",
|
| 430 |
+
" opt.zero_grad()\n",
|
| 431 |
+
" scaler.scale(loss).backward()\n",
|
| 432 |
+
" scaler.step(opt)\n",
|
| 433 |
+
" scaler.update()\n",
|
| 434 |
+
" sched.step()\n",
|
| 435 |
+
" tot += loss.item() * len(y)\n",
|
| 436 |
+
" model.eval()\n",
|
| 437 |
+
" ys, ps = [], []\n",
|
| 438 |
+
" with torch.no_grad():\n",
|
| 439 |
+
" for x, y in dl_val:\n",
|
| 440 |
+
" ps.append(model(x.to(device)).argmax(1).cpu())\n",
|
| 441 |
+
" ys.append(y)\n",
|
| 442 |
+
" f1 = f1_score(torch.cat(ys), torch.cat(ps), average=\"macro\")\n",
|
| 443 |
+
" history.append({\"epoch\": ep + 1, \"train_loss\": tot / len(dl.dataset), \"val_patch_macro_f1\": f1})\n",
|
| 444 |
+
" print(history[-1])\n",
|
| 445 |
+
" if f1 > best:\n",
|
| 446 |
+
" best = f1\n",
|
| 447 |
+
" torch.save(model.state_dict(), out_dir / \"best.pt\")\n",
|
| 448 |
+
"\n",
|
| 449 |
+
" model.load_state_dict(torch.load(out_dir / \"best.pt\", map_location=device))\n",
|
| 450 |
+
" t_gen, t_edit = tune_thresholds(model, part(\"val\"), root, device)\n",
|
| 451 |
+
" results = evaluate(model, part(\"test\"), root, device, t_gen, t_edit)\n",
|
| 452 |
+
" results.update({\"history\": history, \"thresholds\": {\"generated\": t_gen, \"edited\": t_edit},\n",
|
| 453 |
+
" \"held_out_site\": holdout_site, \"images_per_split\": counts})\n",
|
| 454 |
+
" (out_dir / \"results.json\").write_text(json.dumps(results, indent=2))\n",
|
| 455 |
+
" save_for_app(model, out_dir, arch, t_gen, t_edit)\n",
|
| 456 |
+
" return model, results\n",
|
| 457 |
+
"\n",
|
| 458 |
+
"\n",
|
| 459 |
+
"def _scores_for(model, rows, root, device):\n",
|
| 460 |
+
" out = []\n",
|
| 461 |
+
" for r in rows:\n",
|
| 462 |
+
" patches = tile_predict(model, Image.open(Path(root) / r[\"path\"]), device)\n",
|
| 463 |
+
" out.append((r, patches, *image_scores(patches)))\n",
|
| 464 |
+
" return out\n",
|
| 465 |
+
"\n",
|
| 466 |
+
"\n",
|
| 467 |
+
"def tune_thresholds(model, val_rows, root, device):\n",
|
| 468 |
+
" \"\"\"Pick the cutoffs that best separate classes on VALIDATION images (never test).\"\"\"\n",
|
| 469 |
+
" from sklearn.metrics import f1_score\n",
|
| 470 |
+
" scored = _scores_for(model, val_rows, root, device)\n",
|
| 471 |
+
" grid = np.linspace(0.2, 0.9, 15)\n",
|
| 472 |
+
" best = (0.5, 0.5, -1)\n",
|
| 473 |
+
" for tg in grid:\n",
|
| 474 |
+
" for te in grid:\n",
|
| 475 |
+
" y = [r[\"label\"] for r, *_ in scored]\n",
|
| 476 |
+
" p = [decide(g, e, tg, te) for _, _, g, e in scored]\n",
|
| 477 |
+
" f = f1_score(y, p, average=\"macro\", labels=CLASSES, zero_division=0)\n",
|
| 478 |
+
" if f > best[2]:\n",
|
| 479 |
+
" best = (float(tg), float(te), f)\n",
|
| 480 |
+
" print(f\"thresholds: generated={best[0]:.2f} edited={best[1]:.2f} (val macro-F1 {best[2]:.3f})\")\n",
|
| 481 |
+
" return best[0], best[1]\n",
|
| 482 |
+
"\n",
|
| 483 |
+
"\n",
|
| 484 |
+
"def evaluate(model, test_rows, root, device, t_gen, t_edit):\n",
|
| 485 |
+
" from sklearn.metrics import classification_report, confusion_matrix\n",
|
| 486 |
+
" scored = _scores_for(model, test_rows, root, device)\n",
|
| 487 |
+
" y = [r[\"label\"] for r, *_ in scored]\n",
|
| 488 |
+
" p = [decide(g, e, t_gen, t_edit) for _, _, g, e in scored]\n",
|
| 489 |
+
" report = classification_report(y, p, labels=CLASSES, output_dict=True, zero_division=0)\n",
|
| 490 |
+
" print(classification_report(y, p, labels=CLASSES, zero_division=0))\n",
|
| 491 |
+
" unseen = [(r, d) for (r, *_), d in zip(scored, p) if r.get(\"unseen_site\")]\n",
|
| 492 |
+
" loc_hits, loc_n = 0, 0\n",
|
| 493 |
+
" for r, patches, g, e in scored:\n",
|
| 494 |
+
" if r[\"label\"] == \"edited\" and patches:\n",
|
| 495 |
+
" top = max(patches, key=lambda q: q[4])\n",
|
| 496 |
+
" loc_hits += _overlap(top[0], top[1], r[\"edit_box\"]) > 0\n",
|
| 497 |
+
" loc_n += 1\n",
|
| 498 |
+
" return {\n",
|
| 499 |
+
" \"test_report\": report,\n",
|
| 500 |
+
" \"test_confusion\": confusion_matrix(y, p, labels=CLASSES).tolist(),\n",
|
| 501 |
+
" \"unseen_site_caught\": sum(d == \"generated\" for _, d in unseen) / max(1, len(unseen)),\n",
|
| 502 |
+
" \"unseen_site_images\": len(unseen),\n",
|
| 503 |
+
" \"edit_localization\": loc_hits / max(1, loc_n),\n",
|
| 504 |
+
" }\n",
|
| 505 |
+
"\n",
|
| 506 |
+
"\n",
|
| 507 |
+
"def save_for_app(model, out_dir, arch, t_gen, t_edit):\n",
|
| 508 |
+
" from safetensors.torch import save_file\n",
|
| 509 |
+
" out_dir = Path(out_dir)\n",
|
| 510 |
+
" save_file({k: v.contiguous().cpu() for k, v in model.state_dict().items()}, out_dir / \"model.safetensors\")\n",
|
| 511 |
+
" (out_dir / \"config.json\").write_text(json.dumps({\n",
|
| 512 |
+
" \"arch\": arch, \"classes\": CLASSES, \"patch\": PATCH, \"stride\": STRIDE, \"min_std\": MIN_STD,\n",
|
| 513 |
+
" \"mean\": MEAN, \"std\": STD, \"t_generated\": t_gen, \"t_edited\": t_edit}, indent=2))"
|
| 514 |
+
]
|
| 515 |
+
},
|
| 516 |
+
{
|
| 517 |
+
"cell_type": "markdown",
|
| 518 |
+
"metadata": {},
|
| 519 |
+
"source": [
|
| 520 |
+
"## 3. Automatically make edited screenshots\n",
|
| 521 |
+
"\n",
|
| 522 |
+
"For each real screenshot, OCR finds the messages; the script covers one and types a replacement,\n",
|
| 523 |
+
"recording the exact edit box. Two edits per screenshot."
|
| 524 |
+
]
|
| 525 |
+
},
|
| 526 |
+
{
|
| 527 |
+
"cell_type": "code",
|
| 528 |
+
"metadata": {},
|
| 529 |
+
"execution_count": null,
|
| 530 |
+
"outputs": [],
|
| 531 |
+
"source": [
|
| 532 |
+
"import easyocr, screen_edits as se\n",
|
| 533 |
+
"reader = easyocr.Reader(['en'], gpu=True)\n",
|
| 534 |
+
"edits = se.make_edited_set(f'{SCREENS}/real', f'{SCREENS}/edited', per_image=2, seed=0, reader=reader)"
|
| 535 |
+
]
|
| 536 |
+
},
|
| 537 |
+
{
|
| 538 |
+
"cell_type": "code",
|
| 539 |
+
"metadata": {},
|
| 540 |
+
"execution_count": null,
|
| 541 |
+
"outputs": [],
|
| 542 |
+
"source": [
|
| 543 |
+
"# Check the edits look like real fakes (the red box marks the edit)\n",
|
| 544 |
+
"import matplotlib.pyplot as plt, matplotlib.patches as mp\n",
|
| 545 |
+
"fig, axes = plt.subplots(1, 4, figsize=(18, 9))\n",
|
| 546 |
+
"for a, e in zip(axes, edits[:4]):\n",
|
| 547 |
+
" a.imshow(Image.open(f\"{SCREENS}/edited/{e['path']}\")); a.axis('off')\n",
|
| 548 |
+
" x0, y0, x1, y1 = e['edit_box']\n",
|
| 549 |
+
" a.add_patch(mp.Rectangle((x0, y0), x1 - x0, y1 - y0, fill=False, color='red', lw=2))\n",
|
| 550 |
+
"plt.savefig(f'{RUN}/edit_examples.png', dpi=80); plt.show()"
|
| 551 |
+
]
|
| 552 |
+
},
|
| 553 |
+
{
|
| 554 |
+
"cell_type": "markdown",
|
| 555 |
+
"metadata": {},
|
| 556 |
+
"source": [
|
| 557 |
+
"## 4. Train\n",
|
| 558 |
+
"\n",
|
| 559 |
+
"- 256x256 **patches at full resolution** (resizing would erase pixel-level clues)\n",
|
| 560 |
+
"- Patches from edited images always contain at least half of the edit\n",
|
| 561 |
+
"- Nearly blank patches are skipped\n",
|
| 562 |
+
"- JPEG recompression and rescaling are applied to **every** class equally, so the model can't cheat\n",
|
| 563 |
+
"- Classes balanced per epoch; best epoch by validation macro-F1\n",
|
| 564 |
+
"- Decision thresholds tuned on **validation** images only, never the test set"
|
| 565 |
+
]
|
| 566 |
+
},
|
| 567 |
+
{
|
| 568 |
+
"cell_type": "code",
|
| 569 |
+
"metadata": {},
|
| 570 |
+
"execution_count": null,
|
| 571 |
+
"outputs": [],
|
| 572 |
+
"source": [
|
| 573 |
+
"import screen_train as st\n",
|
| 574 |
+
"model, results = st.train(SCREENS, RUN, arch='efficientnet_b0', epochs=8, batch=48, lr=3e-4, workers=2)"
|
| 575 |
+
]
|
| 576 |
+
},
|
| 577 |
+
{
|
| 578 |
+
"cell_type": "markdown",
|
| 579 |
+
"metadata": {},
|
| 580 |
+
"source": [
|
| 581 |
+
"## 5. Results (screenshot these for your documentation)"
|
| 582 |
+
]
|
| 583 |
+
},
|
| 584 |
+
{
|
| 585 |
+
"cell_type": "code",
|
| 586 |
+
"metadata": {},
|
| 587 |
+
"execution_count": null,
|
| 588 |
+
"outputs": [],
|
| 589 |
+
"source": [
|
| 590 |
+
"import json, numpy as np\n",
|
| 591 |
+
"res = json.load(open(f'{RUN}/results.json'))\n",
|
| 592 |
+
"print('held-out generator site:', res['held_out_site'])\n",
|
| 593 |
+
"print('images per split:', res['images_per_split'])\n",
|
| 594 |
+
"print('thresholds:', res['thresholds'])\n",
|
| 595 |
+
"print(f\"unseen-site fakes caught: {res['unseen_site_caught']:.1%} of {res['unseen_site_images']}\")\n",
|
| 596 |
+
"print(f\"edit localization (top patch touches the edit): {res['edit_localization']:.1%}\")\n",
|
| 597 |
+
"cm = np.array(res['test_confusion'])\n",
|
| 598 |
+
"plt.figure(figsize=(5, 4)); plt.imshow(cm, cmap='Blues')\n",
|
| 599 |
+
"plt.xticks(range(3), st.CLASSES); plt.yticks(range(3), st.CLASSES)\n",
|
| 600 |
+
"for i in range(3):\n",
|
| 601 |
+
" for j in range(3): plt.text(j, i, cm[i, j], ha='center', va='center')\n",
|
| 602 |
+
"plt.xlabel('predicted'); plt.ylabel('true'); plt.title('Test images')\n",
|
| 603 |
+
"plt.tight_layout(); plt.savefig(f'{RUN}/confusion_test.png', dpi=120); plt.show()"
|
| 604 |
+
]
|
| 605 |
+
},
|
| 606 |
+
{
|
| 607 |
+
"cell_type": "markdown",
|
| 608 |
+
"metadata": {},
|
| 609 |
+
"source": [
|
| 610 |
+
"## 6. Heatmaps on test images (use one of these in your demo video)"
|
| 611 |
+
]
|
| 612 |
+
},
|
| 613 |
+
{
|
| 614 |
+
"cell_type": "code",
|
| 615 |
+
"metadata": {},
|
| 616 |
+
"execution_count": null,
|
| 617 |
+
"outputs": [],
|
| 618 |
+
"source": [
|
| 619 |
+
"import torch\n",
|
| 620 |
+
"idx = json.load(open(f'{RUN}/index.json'))\n",
|
| 621 |
+
"tests = [r for r in idx if r['split'] == 'test' and r['label'] == 'edited'][:3]\n",
|
| 622 |
+
"fig, axes = plt.subplots(1, len(tests), figsize=(6 * len(tests), 10))\n",
|
| 623 |
+
"for a, r in zip(np.atleast_1d(axes), tests):\n",
|
| 624 |
+
" img = Image.open(f\"{SCREENS}/{r['path']}\").convert('RGB')\n",
|
| 625 |
+
" patches = st.tile_predict(model, img, 'cuda')\n",
|
| 626 |
+
" heat = np.zeros((img.height, img.width)); cnt = np.zeros_like(heat)\n",
|
| 627 |
+
" for x, y, _, _, e in patches:\n",
|
| 628 |
+
" heat[y:y+st.PATCH, x:x+st.PATCH] += e; cnt[y:y+st.PATCH, x:x+st.PATCH] += 1\n",
|
| 629 |
+
" a.imshow(img); a.imshow(np.divide(heat, cnt, out=np.zeros_like(heat), where=cnt > 0), cmap='Reds', alpha=0.5)\n",
|
| 630 |
+
" x0, y0, x1, y1 = r['edit_box']\n",
|
| 631 |
+
" a.add_patch(mp.Rectangle((x0, y0), x1 - x0, y1 - y0, fill=False, color='blue', lw=2)); a.axis('off')\n",
|
| 632 |
+
"plt.savefig(f'{RUN}/heatmaps.png', dpi=80); plt.show()"
|
| 633 |
+
]
|
| 634 |
+
},
|
| 635 |
+
{
|
| 636 |
+
"cell_type": "markdown",
|
| 637 |
+
"metadata": {},
|
| 638 |
+
"source": [
|
| 639 |
+
"## 7. Upload the model (PRIVATE, because it was trained on personal screenshots)\n",
|
| 640 |
+
"\n",
|
| 641 |
+
"The app reads it with an `HF_TOKEN` secret in your Space."
|
| 642 |
+
]
|
| 643 |
+
},
|
| 644 |
+
{
|
| 645 |
+
"cell_type": "code",
|
| 646 |
+
"metadata": {},
|
| 647 |
+
"execution_count": null,
|
| 648 |
+
"outputs": [],
|
| 649 |
+
"source": [
|
| 650 |
+
"from huggingface_hub import notebook_login, HfApi, create_repo, whoami\n",
|
| 651 |
+
"notebook_login()\n",
|
| 652 |
+
"REPO = f\"{whoami()['name']}/tell-screenshotcheck\"\n",
|
| 653 |
+
"create_repo(REPO, private=True, exist_ok=True)\n",
|
| 654 |
+
"HfApi().upload_folder(folder_path=RUN, repo_id=REPO,\n",
|
| 655 |
+
" allow_patterns=['config.json', 'model.safetensors', 'results.json']) # no screenshot images, ever\n",
|
| 656 |
+
"print('done: set SCREENSHOT_MODEL =', REPO, 'and HF_TOKEN (read token) in your Space secrets')"
|
| 657 |
+
]
|
| 658 |
+
}
|
| 659 |
+
],
|
| 660 |
+
"metadata": {
|
| 661 |
+
"accelerator": "GPU",
|
| 662 |
+
"colab": {
|
| 663 |
+
"provenance": [],
|
| 664 |
+
"gpuType": "T4"
|
| 665 |
+
},
|
| 666 |
+
"kernelspec": {
|
| 667 |
+
"name": "python3",
|
| 668 |
+
"display_name": "Python 3"
|
| 669 |
+
},
|
| 670 |
+
"language_info": {
|
| 671 |
+
"name": "python"
|
| 672 |
+
}
|
| 673 |
+
},
|
| 674 |
+
"nbformat": 4,
|
| 675 |
+
"nbformat_minor": 0
|
| 676 |
+
}
|
requirements-dev.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
pytest
|
requirements.txt
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
gradio>=5.0
|
| 2 |
+
fastapi
|
| 3 |
+
uvicorn
|
| 4 |
+
python-multipart
|
| 5 |
+
pillow
|
| 6 |
+
numpy
|
| 7 |
+
markdown-it-py
|
| 8 |
+
anthropic
|
| 9 |
+
easyocr
|
| 10 |
+
torch
|
| 11 |
+
torchvision
|
| 12 |
+
transformers
|
| 13 |
+
timm
|
| 14 |
+
huggingface_hub
|
| 15 |
+
safetensors
|
| 16 |
+
python-dotenv
|
scripts/run_eval.py
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Runs every post in eval/posts.json through TELL's text tools and scores it.
|
| 3 |
+
|
| 4 |
+
python scripts/run_eval.py # language + number tricks only (free)
|
| 5 |
+
python scripts/run_eval.py --trace # also source tracing (costs money, see estimate)
|
| 6 |
+
python scripts/run_eval.py --only pratt-zero-votes
|
| 7 |
+
|
| 8 |
+
Writes eval/results.csv and prints the numbers that go in TRAINING.md.
|
| 9 |
+
Scoring uses expected_tricks, which you fill in by hand after reviewing suggested_tricks.
|
| 10 |
+
"""
|
| 11 |
+
|
| 12 |
+
import argparse
|
| 13 |
+
import csv
|
| 14 |
+
import json
|
| 15 |
+
import sys
|
| 16 |
+
from collections import defaultdict
|
| 17 |
+
from pathlib import Path
|
| 18 |
+
|
| 19 |
+
ROOT = Path(__file__).resolve().parent.parent
|
| 20 |
+
sys.path.insert(0, str(ROOT))
|
| 21 |
+
|
| 22 |
+
try:
|
| 23 |
+
from dotenv import load_dotenv
|
| 24 |
+
load_dotenv(ROOT / ".env")
|
| 25 |
+
except ImportError:
|
| 26 |
+
pass
|
| 27 |
+
|
| 28 |
+
import features # noqa: F401,E402
|
| 29 |
+
from core import pipeline # noqa: E402
|
| 30 |
+
from core.post import Post # noqa: E402
|
| 31 |
+
|
| 32 |
+
COST_PER_POST_WITH_TRACE = 0.12 # worst case: 3 claims x 4 searches x $0.01
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
def score(found: set, expected: set):
|
| 36 |
+
return len(found & expected), len(expected - found), len(found - expected)
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
def main():
|
| 40 |
+
ap = argparse.ArgumentParser()
|
| 41 |
+
ap.add_argument("--trace", action="store_true", help="also trace claims to sources")
|
| 42 |
+
ap.add_argument("--only", help="run one post by id")
|
| 43 |
+
args = ap.parse_args()
|
| 44 |
+
|
| 45 |
+
posts = json.loads((ROOT / "eval" / "posts.json").read_text())["posts"]
|
| 46 |
+
if args.only:
|
| 47 |
+
posts = [p for p in posts if p["id"] == args.only]
|
| 48 |
+
if args.trace:
|
| 49 |
+
print(f"Source tracing on: up to ${COST_PER_POST_WITH_TRACE * len(posts):.2f} for {len(posts)} posts.")
|
| 50 |
+
if input("Go? [y/N] ").strip().lower() != "y":
|
| 51 |
+
return
|
| 52 |
+
|
| 53 |
+
unreviewed = [p["id"] for p in posts if not p.get("expected_tricks") and p.get("suggested_tricks")]
|
| 54 |
+
if unreviewed:
|
| 55 |
+
print(f"Note: {len(unreviewed)} posts have no expected_tricks yet, so they score as all false alarms.\n")
|
| 56 |
+
|
| 57 |
+
rows = []
|
| 58 |
+
totals = defaultdict(lambda: [0, 0, 0, 0]) # group -> hits, misses, false alarms, posts
|
| 59 |
+
for p in posts:
|
| 60 |
+
report = pipeline.run(Post(text=p["text"]), {"trace": args.trace})
|
| 61 |
+
found = {f["trick"] for f in report.findings}
|
| 62 |
+
expected = set(p.get("expected_tricks", []))
|
| 63 |
+
hit, miss, fa = score(found, expected)
|
| 64 |
+
|
| 65 |
+
group = "control" if p.get("control") else p["favors"].split(" ")[0]
|
| 66 |
+
t = totals[group]
|
| 67 |
+
t[0] += hit; t[1] += miss; t[2] += fa; t[3] += 1
|
| 68 |
+
|
| 69 |
+
verdicts = [c["verdict"] for s in report.sections for c in s.claim_checks]
|
| 70 |
+
rows.append({
|
| 71 |
+
"id": p["id"], "group": group,
|
| 72 |
+
"found": ";".join(sorted(found)), "expected": ";".join(sorted(expected)),
|
| 73 |
+
"hits": hit, "misses": miss, "false_alarms": fa,
|
| 74 |
+
"trace_verdicts": ";".join(verdicts), "expected_trace": p.get("expected_trace", ""),
|
| 75 |
+
})
|
| 76 |
+
print(f"{p['id']:30s} found={sorted(found)} hits={hit} miss={miss} fa={fa} {verdicts or ''}")
|
| 77 |
+
|
| 78 |
+
out = ROOT / "eval" / "results.csv"
|
| 79 |
+
with out.open("w", newline="") as f:
|
| 80 |
+
w = csv.DictWriter(f, fieldnames=rows[0].keys())
|
| 81 |
+
w.writeheader()
|
| 82 |
+
w.writerows(rows)
|
| 83 |
+
|
| 84 |
+
print("\ngroup posts hits misses false_alarms flags/post")
|
| 85 |
+
for g, (h, m, fa, n) in sorted(totals.items()):
|
| 86 |
+
print(f"{g:12s} {n:5d} {h:5d} {m:7d} {fa:13d} {(h + fa) / n:11.2f}")
|
| 87 |
+
print("\nIf flags/post is a lot higher for one side than the other on similar posts, look at why.")
|
| 88 |
+
print(f"Saved {out.relative_to(ROOT)}")
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
if __name__ == "__main__":
|
| 92 |
+
main()
|
tests/__init__.py
ADDED
|
File without changes
|
tests/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (154 Bytes). View file
|
|
|
tests/__pycache__/test_pipeline.cpython-314-pytest-9.1.1.pyc
ADDED
|
Binary file (31.6 kB). View file
|
|
|
tests/test_pipeline.py
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
Routing tests. The classifier, OCR, and image models are faked so these run in
|
| 3 |
+
a second with no downloads or API keys: python -m pytest tests
|
| 4 |
+
"""
|
| 5 |
+
|
| 6 |
+
import pytest
|
| 7 |
+
from PIL import Image
|
| 8 |
+
|
| 9 |
+
import features # noqa: F401
|
| 10 |
+
from core import pipeline
|
| 11 |
+
from core.post import Post
|
| 12 |
+
from tools import chartcheck, langcheck, screenshotcheck
|
| 13 |
+
|
| 14 |
+
IMG = Image.new("RGB", (64, 64), "white")
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
@pytest.fixture(autouse=True)
|
| 18 |
+
def fake_models(monkeypatch):
|
| 19 |
+
monkeypatch.delenv("ANTHROPIC_API_KEY", raising=False)
|
| 20 |
+
langcheck._cache.clear()
|
| 21 |
+
monkeypatch.setattr(chartcheck, "analyze", lambda img: {"findings": [], "confidence": "fake"})
|
| 22 |
+
monkeypatch.setattr(screenshotcheck, "analyze", lambda img: {"findings": [], "confidence": "fake"})
|
| 23 |
+
monkeypatch.setattr(features.EmotionCheck, "run", lambda self, post, options: {"findings": [], "confidence": "fake"})
|
| 24 |
+
monkeypatch.setattr(features, "domain_age_days", lambda reg: None) # no network in tests
|
| 25 |
+
monkeypatch.setattr(features, "resolve_short", lambda url: None)
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
def run_as(monkeypatch, kind, ocr_text="", text="", image=IMG):
|
| 29 |
+
monkeypatch.setattr(pipeline, "classify_image", lambda img: (kind, 0.9))
|
| 30 |
+
monkeypatch.setattr(pipeline.ocr, "read_text", lambda img: ocr_text)
|
| 31 |
+
return pipeline.run(Post(image=image, text=text))
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
def test_text_only_goes_to_text_tools():
|
| 35 |
+
report = pipeline.run(Post(text="Experts say this is a disaster."))
|
| 36 |
+
assert report.post.kind == "text"
|
| 37 |
+
assert report.route == ["StatCheck", "LangCheck", "EmotionCheck"]
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
def test_chart_skips_ocr_and_text_tools(monkeypatch):
|
| 41 |
+
report = run_as(monkeypatch, "chart", ocr_text="should not be read")
|
| 42 |
+
assert report.route == ["ChartCheck"]
|
| 43 |
+
assert report.post.ocr_text == ""
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def test_chart_with_caption_also_checks_caption(monkeypatch):
|
| 47 |
+
report = run_as(monkeypatch, "chart", text="Crime is up 300%!")
|
| 48 |
+
assert report.route == ["ChartCheck", "StatCheck", "LangCheck", "EmotionCheck"]
|
| 49 |
+
|
| 50 |
+
|
| 51 |
+
def test_screenshot_text_gets_read_and_checked(monkeypatch):
|
| 52 |
+
report = run_as(monkeypatch, "screenshot", ocr_text="SHARE THIS BEFORE IT'S DELETED")
|
| 53 |
+
assert report.route == ["ScreenshotCheck", "StatCheck", "LangCheck", "EmotionCheck"]
|
| 54 |
+
assert any(f["trick"] == "false_urgency" for f in report.findings)
|
| 55 |
+
|
| 56 |
+
|
| 57 |
+
def test_photo_without_key_still_gives_search_tip(monkeypatch):
|
| 58 |
+
report = run_as(monkeypatch, "photo")
|
| 59 |
+
assert report.route == ["ImageCheck"]
|
| 60 |
+
assert "Reverse-search" in report.sections[0].notes[0]
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
def test_link_in_text_routes_to_linkcheck():
|
| 64 |
+
report = pipeline.run(Post(text="read this https://example.com/story"))
|
| 65 |
+
assert "LinkCheck" in report.route
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def test_unknown_image_with_no_text_asks_user(monkeypatch):
|
| 69 |
+
report = run_as(monkeypatch, "unknown")
|
| 70 |
+
assert report.route == []
|
| 71 |
+
assert "Pick Chart" in report.notes[0]
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def test_broken_feature_does_not_break_report(monkeypatch):
|
| 75 |
+
def boom(img):
|
| 76 |
+
raise RuntimeError("model file missing")
|
| 77 |
+
monkeypatch.setattr(chartcheck, "analyze", boom)
|
| 78 |
+
report = run_as(monkeypatch, "chart", text="Experts say so.")
|
| 79 |
+
chart = report.sections[0]
|
| 80 |
+
assert chart.error == "model file missing"
|
| 81 |
+
assert report.sections[2].findings # LangCheck still ran
|
| 82 |
+
|
| 83 |
+
|
| 84 |
+
def test_forced_kind_skips_classifier(monkeypatch):
|
| 85 |
+
monkeypatch.setattr(pipeline, "classify_image", lambda img: pytest.fail("classifier ran"))
|
| 86 |
+
monkeypatch.setattr(pipeline.ocr, "read_text", lambda img: "")
|
| 87 |
+
report = pipeline.run(Post(image=IMG), forced_kind="chart")
|
| 88 |
+
assert report.route == ["ChartCheck"]
|
| 89 |
+
|
| 90 |
+
|
| 91 |
+
def test_empty_post():
|
| 92 |
+
report = pipeline.run(Post())
|
| 93 |
+
assert report.route == [] and report.notes
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
# ---------- quick fact check + local places ----------
|
| 97 |
+
|
| 98 |
+
class FakeResp:
|
| 99 |
+
def __init__(self, data):
|
| 100 |
+
self.data = data
|
| 101 |
+
|
| 102 |
+
def json(self):
|
| 103 |
+
return self.data
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def fake_factcheck_api(monkeypatch, claim, rating):
|
| 107 |
+
import requests
|
| 108 |
+
data = {"claims": [{"text": claim, "claimReview": [
|
| 109 |
+
{"publisher": {"name": "PolitiFact"}, "textualRating": rating, "title": "t", "url": "https://example.org/fc"}]}]}
|
| 110 |
+
monkeypatch.setattr(requests, "get", lambda *a, **k: FakeResp(data))
|
| 111 |
+
monkeypatch.setenv("GOOGLE_FACTCHECK_KEY", "test")
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
def test_same_claim_fact_check_turns_easy_mode_red(monkeypatch):
|
| 115 |
+
from core.report import easy_html
|
| 116 |
+
fake_factcheck_api(monkeypatch, "Dolly Parton refuses to celebrate Pride Month", "Pants on Fire")
|
| 117 |
+
report = pipeline.run(Post(text="BREAKING NEWS: Dolly Parton announces her refusal to celebrate Pride Month in June."))
|
| 118 |
+
assert report.route[0] == "FactCheck"
|
| 119 |
+
fc = report.sections[0].factchecks[0]
|
| 120 |
+
assert fc["match"] and fc["verdict"] == "CONTRADICTED"
|
| 121 |
+
assert "v-red" in easy_html(report) and "Quick fact check" in easy_html(report)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def test_unrelated_fact_check_is_shown_but_not_trusted(monkeypatch):
|
| 125 |
+
from core.report import easy_html
|
| 126 |
+
fake_factcheck_api(monkeypatch, "Vaccines contain microchips for tracking people", "False")
|
| 127 |
+
report = pipeline.run(Post(text="The city council voted to extend weekend hours starting in January."))
|
| 128 |
+
fc = report.sections[0].factchecks[0]
|
| 129 |
+
assert not fc["match"]
|
| 130 |
+
assert "v-red" not in easy_html(report)
|
| 131 |
+
|
| 132 |
+
|
| 133 |
+
def test_local_post_without_location_asks_for_it():
|
| 134 |
+
report = pipeline.run(Post(text="The downtown library is closing forever next week!"))
|
| 135 |
+
local = next(s for s in report.sections if s.feature == "LocalCheck")
|
| 136 |
+
assert "Use my location" in local.notes[0] and not local.places
|
| 137 |
+
|
| 138 |
+
|
| 139 |
+
def test_local_post_with_location_finds_place(monkeypatch):
|
| 140 |
+
place = {"name": "Kirkland Library", "distance": "1.2 miles away",
|
| 141 |
+
"contacts": ["Phone: 425-555-0100"], "contacts_html": ["Phone: 425-555-0100"]}
|
| 142 |
+
monkeypatch.setattr(features, "nearest_place", lambda word, lat, lon: place)
|
| 143 |
+
report = pipeline.run(Post(text="The downtown library is closing forever!"), {"location": (47.68, -122.2)})
|
| 144 |
+
local = next(s for s in report.sections if s.feature == "LocalCheck")
|
| 145 |
+
assert local.places[0]["name"] == "Kirkland Library"
|
| 146 |
+
|
| 147 |
+
|
| 148 |
+
def test_rating_words_map_to_verdicts():
|
| 149 |
+
assert features.rating_to_verdict("Pants on Fire") == "CONTRADICTED"
|
| 150 |
+
assert features.rating_to_verdict("Half True") == "MISSING_CONTEXT"
|
| 151 |
+
assert features.rating_to_verdict("Mostly True") == "SUPPORTED"
|
| 152 |
+
assert features.rating_to_verdict("Satire") == "RATED"
|
| 153 |
+
|
| 154 |
+
|
| 155 |
+
# ---------- links and photos ----------
|
| 156 |
+
|
| 157 |
+
def test_lookalike_and_short_links(monkeypatch):
|
| 158 |
+
monkeypatch.setattr(features, "resolve_short", lambda url: "https://paypa1-secure-login.com/verify")
|
| 159 |
+
monkeypatch.setattr(features, "domain_age_days", lambda reg: 12)
|
| 160 |
+
report = pipeline.run(Post(text="Your account is locked! Fix it now: bit.ly/3xYz"))
|
| 161 |
+
link = next(s for s in report.sections if s.feature == "LinkCheck")
|
| 162 |
+
tricks = {f["trick"] for f in link.findings}
|
| 163 |
+
assert tricks == {"hidden_link", "lookalike_link", "new_site"}
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def test_real_sites_are_not_flagged():
|
| 167 |
+
report = pipeline.run(Post(text="Full story: https://www.reuters.com/world and https://groups.google.com/x"))
|
| 168 |
+
link = next(s for s in report.sections if s.feature == "LinkCheck")
|
| 169 |
+
assert link.findings == []
|
| 170 |
+
|
| 171 |
+
|
| 172 |
+
class FakeVision:
|
| 173 |
+
def __init__(self, answer):
|
| 174 |
+
from types import SimpleNamespace as N
|
| 175 |
+
self.messages = N(create=lambda **k: N(content=[N(type="text", text=answer)]))
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
def test_image_check_needs_two_signs(monkeypatch):
|
| 179 |
+
import json
|
| 180 |
+
answer = json.dumps({"ai_signs": ["six fingers on the left hand", "garbled sign text"], "ai_likelihood": "high",
|
| 181 |
+
"caption_matches": "no", "mismatch_reason": "The caption says Seattle but the signs are in French.",
|
| 182 |
+
"search_tip": "flooded street French signs"})
|
| 183 |
+
monkeypatch.setattr(langcheck, "_client", lambda: FakeVision(answer))
|
| 184 |
+
report = run_as(monkeypatch, "photo", text="Seattle underwater right now!!")
|
| 185 |
+
img = next(s for s in report.sections if s.feature == "ImageCheck")
|
| 186 |
+
assert {f["trick"] for f in img.findings} == {"ai_image", "out_of_context"}
|
| 187 |
+
|
| 188 |
+
one = json.dumps({"ai_signs": ["odd shadow"], "ai_likelihood": "high", "caption_matches": "yes",
|
| 189 |
+
"mismatch_reason": "", "search_tip": "x"})
|
| 190 |
+
monkeypatch.setattr(langcheck, "_client", lambda: FakeVision(one))
|
| 191 |
+
report = run_as(monkeypatch, "photo")
|
| 192 |
+
assert next(s for s in report.sections if s.feature == "ImageCheck").findings == []
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def test_every_trick_has_a_lesson():
|
| 196 |
+
import json
|
| 197 |
+
lessons = json.load(open("lessons/lessons.json"))
|
| 198 |
+
for trick in ["hidden_link", "lookalike_link", "new_site", "ai_image", "out_of_context",
|
| 199 |
+
"emotional_language", "truncated_axis", "screenshot_edited"]:
|
| 200 |
+
assert trick in lessons
|
| 201 |
+
|
| 202 |
+
|
| 203 |
+
def test_single_tool_tab_runs_only_that_check():
|
| 204 |
+
report = pipeline.run_one(Post(text="Experts say this is a disaster!!!"), "LangCheck")
|
| 205 |
+
assert report.route == ["LangCheck"] and len(report.sections) == 1
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def test_single_tool_tab_explains_when_it_does_not_apply():
|
| 209 |
+
report = pipeline.run_one(Post(text="No links here."), "LinkCheck")
|
| 210 |
+
assert report.sections == []
|
tools/__init__.py
ADDED
|
File without changes
|
tools/__pycache__/__init__.cpython-314.pyc
ADDED
|
Binary file (154 Bytes). View file
|
|
|
tools/__pycache__/chartcheck.cpython-314.pyc
ADDED
|
Binary file (10.2 kB). View file
|
|
|
tools/__pycache__/langcheck.cpython-314.pyc
ADDED
|
Binary file (17.1 kB). View file
|
|
|
tools/__pycache__/screenshotcheck.cpython-314.pyc
ADDED
|
Binary file (9.32 kB). View file
|
|
|
tools/__pycache__/statcheck.cpython-314.pyc
ADDED
|
Binary file (8.06 kB). View file
|
|
|
tools/chartcheck.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ChartCheck: finds misleading tricks in charts.
|
| 3 |
+
|
| 4 |
+
Two layers that vote together:
|
| 5 |
+
1. RULE CHECKS on the axis numbers (works now, no training needed).
|
| 6 |
+
2. A CLASSIFIER you fine-tune on Misviz in notebooks/ (plug in when trained).
|
| 7 |
+
|
| 8 |
+
How axis numbers get in:
|
| 9 |
+
- For now: read_axis_ticks() uses OCR to pull the numbers along the left edge.
|
| 10 |
+
- Upgrade later: an axis-extraction model (the Misviz paper used DePlot).
|
| 11 |
+
"""
|
| 12 |
+
|
| 13 |
+
import os
|
| 14 |
+
|
| 15 |
+
# ---------- Layer 1: rule checks on extracted axis numbers ----------
|
| 16 |
+
|
| 17 |
+
def check_truncated_axis(y_ticks, chart_type="bar"):
|
| 18 |
+
"""Bar charts should start at zero. A y-axis starting far above zero exaggerates gaps."""
|
| 19 |
+
if not y_ticks or chart_type != "bar":
|
| 20 |
+
return None
|
| 21 |
+
low, high = min(y_ticks), max(y_ticks)
|
| 22 |
+
if low > 0 and high > 0 and low / high > 0.2:
|
| 23 |
+
return {
|
| 24 |
+
"trick": "truncated_axis",
|
| 25 |
+
"why": f"The y-axis starts at {low:g}, not 0, so small differences look huge.",
|
| 26 |
+
}
|
| 27 |
+
return None
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
def check_inverted_axis(y_ticks_top_to_bottom):
|
| 31 |
+
"""Numbers should get bigger going UP. If they grow going down, the chart is flipped."""
|
| 32 |
+
t = y_ticks_top_to_bottom
|
| 33 |
+
if t and len(t) >= 3 and all(a < b for a, b in zip(t, t[1:])):
|
| 34 |
+
return {
|
| 35 |
+
"trick": "inverted_axis",
|
| 36 |
+
"why": "The y-axis is upside down, so a rise looks like a fall.",
|
| 37 |
+
}
|
| 38 |
+
return None
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def check_uneven_ticks(y_ticks):
|
| 42 |
+
"""Tick labels should be evenly spaced in value (e.g. 0, 10, 20, 30)."""
|
| 43 |
+
t = sorted(y_ticks or [])
|
| 44 |
+
if len(t) < 4:
|
| 45 |
+
return None
|
| 46 |
+
gaps = [b - a for a, b in zip(t, t[1:])]
|
| 47 |
+
if max(gaps) > 1.5 * min(gaps) and min(gaps) > 0:
|
| 48 |
+
return {
|
| 49 |
+
"trick": "uneven_ticks",
|
| 50 |
+
"why": "The axis steps are not equal, which stretches some parts of the chart.",
|
| 51 |
+
}
|
| 52 |
+
return None
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def rule_checks(y_ticks_top_to_bottom, chart_type="bar"):
|
| 56 |
+
findings = []
|
| 57 |
+
for f in (
|
| 58 |
+
check_truncated_axis(y_ticks_top_to_bottom, chart_type),
|
| 59 |
+
check_inverted_axis(y_ticks_top_to_bottom),
|
| 60 |
+
check_uneven_ticks(y_ticks_top_to_bottom),
|
| 61 |
+
):
|
| 62 |
+
if f:
|
| 63 |
+
findings.append(f)
|
| 64 |
+
return findings
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# ---------- Axis reading with OCR ----------
|
| 68 |
+
|
| 69 |
+
def read_axis_ticks(image):
|
| 70 |
+
"""
|
| 71 |
+
Very simple first version: OCR the left 18% of the chart and keep the numbers,
|
| 72 |
+
ordered top to bottom. Good enough for clean charts; improve later.
|
| 73 |
+
"""
|
| 74 |
+
try:
|
| 75 |
+
import numpy as np
|
| 76 |
+
from core.ocr import get_reader
|
| 77 |
+
_reader = get_reader()
|
| 78 |
+
w, h = image.size
|
| 79 |
+
strip = image.crop((0, 0, int(w * 0.18), h))
|
| 80 |
+
results = _reader.readtext(np.array(strip))
|
| 81 |
+
nums = []
|
| 82 |
+
for box, text, conf in results:
|
| 83 |
+
clean = text.replace(",", "").replace("%", "").replace("$", "").strip()
|
| 84 |
+
try:
|
| 85 |
+
y_center = sum(p[1] for p in box) / 4
|
| 86 |
+
nums.append((y_center, float(clean)))
|
| 87 |
+
except ValueError:
|
| 88 |
+
pass
|
| 89 |
+
nums.sort(key=lambda p: p[0]) # top to bottom
|
| 90 |
+
return [v for _, v in nums]
|
| 91 |
+
except Exception as e:
|
| 92 |
+
print("ChartCheck OCR error:", e)
|
| 93 |
+
return []
|
| 94 |
+
|
| 95 |
+
|
| 96 |
+
# ---------- Layer 2: trained classifier (you train this in notebooks/) ----------
|
| 97 |
+
|
| 98 |
+
MODEL_REPO = os.environ.get("CHARTCHECK_MODEL") # "your-hf-name/tell-chartcheck" or a local folder
|
| 99 |
+
_model, _cfg = None, None
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
def _pad_resize(img, size):
|
| 103 |
+
"""MUST match pad_resize in training/chart_train.py."""
|
| 104 |
+
from PIL import Image
|
| 105 |
+
img = img.convert("RGB")
|
| 106 |
+
w, h = img.size
|
| 107 |
+
s = max(w, h)
|
| 108 |
+
canvas = Image.new("RGB", (s, s), (255, 255, 255))
|
| 109 |
+
canvas.paste(img, ((s - w) // 2, (s - h) // 2))
|
| 110 |
+
return canvas.resize((size, size), Image.BILINEAR)
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
def _load():
|
| 114 |
+
global _model, _cfg
|
| 115 |
+
if _model is not None:
|
| 116 |
+
return
|
| 117 |
+
import json
|
| 118 |
+
import timm
|
| 119 |
+
from safetensors.torch import load_file
|
| 120 |
+
if os.path.isdir(MODEL_REPO):
|
| 121 |
+
cfg_path = os.path.join(MODEL_REPO, "config.json")
|
| 122 |
+
w_path = os.path.join(MODEL_REPO, "model.safetensors")
|
| 123 |
+
else:
|
| 124 |
+
from huggingface_hub import hf_hub_download
|
| 125 |
+
tok = os.environ.get("HF_TOKEN")
|
| 126 |
+
cfg_path = hf_hub_download(MODEL_REPO, "config.json", token=tok)
|
| 127 |
+
w_path = hf_hub_download(MODEL_REPO, "model.safetensors", token=tok)
|
| 128 |
+
_cfg = json.load(open(cfg_path))
|
| 129 |
+
m = timm.create_model(_cfg["arch"], pretrained=False, num_classes=len(_cfg["classes"]))
|
| 130 |
+
m.load_state_dict(load_file(w_path))
|
| 131 |
+
_model = m.eval()
|
| 132 |
+
|
| 133 |
+
|
| 134 |
+
def classifier_predict(image):
|
| 135 |
+
"""Returns [(label, probability)] for tricks above the threshold. [] if no model."""
|
| 136 |
+
if not MODEL_REPO:
|
| 137 |
+
return []
|
| 138 |
+
try:
|
| 139 |
+
import numpy as np
|
| 140 |
+
import torch
|
| 141 |
+
_load()
|
| 142 |
+
img = _pad_resize(image, _cfg["img_size"])
|
| 143 |
+
a = (np.asarray(img, dtype=np.float32) / 255.0 - _cfg["mean"]) / _cfg["std"]
|
| 144 |
+
x = torch.from_numpy(a.transpose(2, 0, 1)).float().unsqueeze(0)
|
| 145 |
+
with torch.no_grad():
|
| 146 |
+
probs = torch.softmax(_model(x), 1)[0].tolist()
|
| 147 |
+
return [(c, p) for c, p in zip(_cfg["classes"], probs)
|
| 148 |
+
if c != "none" and p >= _cfg["threshold"]]
|
| 149 |
+
except Exception as e:
|
| 150 |
+
print("ChartCheck model error:", e)
|
| 151 |
+
return []
|
| 152 |
+
|
| 153 |
+
|
| 154 |
+
def analyze(image, chart_type="bar"):
|
| 155 |
+
ticks = read_axis_ticks(image)
|
| 156 |
+
findings = rule_checks(ticks, chart_type)
|
| 157 |
+
model_hits = classifier_predict(image)
|
| 158 |
+
for label, p in model_hits:
|
| 159 |
+
if label not in {f["trick"] for f in findings}:
|
| 160 |
+
findings.append({"trick": label, "why": f"Flagged by the trained ChartCheck model ({p:.0%} sure)."})
|
| 161 |
+
return {
|
| 162 |
+
"tool": "ChartCheck",
|
| 163 |
+
"findings": findings,
|
| 164 |
+
"y_ticks": ticks,
|
| 165 |
+
"confidence": ("high (rules and model agree)" if findings and model_hits and ticks else
|
| 166 |
+
"high" if findings else
|
| 167 |
+
"nothing flagged" if (ticks or MODEL_REPO) else "low: couldn't read the axis"),
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
if __name__ == "__main__":
|
| 172 |
+
print(rule_checks([100, 99, 98, 97, 96, 95])) # truncated
|
| 173 |
+
print(rule_checks([0, 10, 20, 30, 40])) # inverted
|
| 174 |
+
print(rule_checks([100, 50, 20, 10, 0])) # uneven
|
| 175 |
+
print(rule_checks([40, 30, 20, 10, 0])) # honest
|
tools/langcheck.py
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
LangCheck: reads HOW a post is written, then traces its claims back to sources.
|
| 3 |
+
|
| 4 |
+
Part 1, Language analysis
|
| 5 |
+
Labels manipulation techniques from prebunking research (emotional language,
|
| 6 |
+
false dichotomies, scapegoating, attacks on a person) plus a few common ones
|
| 7 |
+
(vague sourcing, false urgency, absolute claims, us vs them). Every label must
|
| 8 |
+
quote the exact words it's about. Labels are about TECHNIQUE, never which side
|
| 9 |
+
of an issue is right.
|
| 10 |
+
- With ANTHROPIC_API_KEY: Claude Haiku returns strict JSON (structured output).
|
| 11 |
+
- Without a key: rule-based fallback (regex), so the app still works.
|
| 12 |
+
|
| 13 |
+
Part 2, Source trace
|
| 14 |
+
Pulls out up to 3 checkable factual claims and, for each one:
|
| 15 |
+
- Searches the web (Claude web search tool) and reports what sources say,
|
| 16 |
+
using a fixed set of outcomes. Links shown to the user come from the API's
|
| 17 |
+
citation data (real URLs), never from text the model typed.
|
| 18 |
+
- Optionally checks Google's Fact Check Tools API for published fact-checks
|
| 19 |
+
from professional fact-checkers (needs GOOGLE_FACTCHECK_KEY).
|
| 20 |
+
TELL never decides truth from the model's memory, only from sources it found.
|
| 21 |
+
"""
|
| 22 |
+
|
| 23 |
+
import concurrent.futures as cf
|
| 24 |
+
import hashlib
|
| 25 |
+
import json
|
| 26 |
+
import os
|
| 27 |
+
import re
|
| 28 |
+
|
| 29 |
+
LANG_MODEL = os.environ.get("TELL_LLM_MODEL", "claude-haiku-5-5")
|
| 30 |
+
SEARCH_MODEL = os.environ.get("TELL_SEARCH_MODEL", "claude-haiku-5-5")
|
| 31 |
+
MAX_CLAIMS = 3
|
| 32 |
+
MAX_SEARCHES_PER_CLAIM = 4
|
| 33 |
+
|
| 34 |
+
TECHNIQUES = {
|
| 35 |
+
"emotional_language": "Words chosen to make you feel fear, anger, or outrage instead of think",
|
| 36 |
+
"false_dichotomy": "Pretends there are only two options when there are more",
|
| 37 |
+
"scapegoating": "Blames one group or person for a complicated problem",
|
| 38 |
+
"ad_hominem": "Attacks a person instead of their argument",
|
| 39 |
+
"vague_sourcing": "Claims backed by unnamed sources: 'experts say', 'studies show', 'people are saying'",
|
| 40 |
+
"false_urgency": "Pressures you to act or share right now",
|
| 41 |
+
"absolute_claims": "Words like always, never, everyone, 100%, that are rarely literally true",
|
| 42 |
+
"us_vs_them": "Splits people into a good 'us' and a bad 'them'",
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
OUTCOMES = {
|
| 46 |
+
"SUPPORTED": "Sources found support this claim",
|
| 47 |
+
"CONTRADICTED": "Sources found say something different",
|
| 48 |
+
"MISSING_CONTEXT": "Partly true, but leaves out something important",
|
| 49 |
+
"DISPUTED": "Reliable sources disagree with each other",
|
| 50 |
+
"NO_SOURCE": "Couldn't find a reliable source for this",
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
# =============================== Part 1: language ===============================
|
| 55 |
+
|
| 56 |
+
RULES = [
|
| 57 |
+
("false_urgency", re.compile(r"\b(share (this )?(now|before)|before (it'?s|they) (deleted|removed|taken down)|"
|
| 58 |
+
r"act now|right now|don'?t wait|last chance|urgent|breaking)\b", re.I)),
|
| 59 |
+
("vague_sourcing", re.compile(r"\b(experts (say|agree|warn)|studies (show|prove)|scientists (say|agree)|"
|
| 60 |
+
r"people are saying|sources say|they don'?t want you to know|many are saying|"
|
| 61 |
+
r"doctors (say|warn|hate))\b", re.I)),
|
| 62 |
+
("absolute_claims", re.compile(r"\b(always|never|everyone|no one|nobody|100%|every single|all of them|"
|
| 63 |
+
r"completely|totally|proven fact)\b", re.I)),
|
| 64 |
+
("emotional_language", re.compile(r"\b(shocking|terrifying|horrifying|outrageous|disgusting|insane|"
|
| 65 |
+
r"destroy(s|ed|ing)?|nightmare|evil|disaster|catastroph\w*|you won'?t believe)\b", re.I)),
|
| 66 |
+
("us_vs_them", re.compile(r"\b(people like us|real (americans|people)|those people|the elites|"
|
| 67 |
+
r"they want to|wake up|sheep(le)?)\b", re.I)),
|
| 68 |
+
("false_dichotomy", re.compile(r"\b(either\b.{1,60}\bor\b|you'?re (either )?with us or|"
|
| 69 |
+
r"if you don'?t\b.{1,40}\bthen you)\b", re.I)),
|
| 70 |
+
]
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
def rule_language(text):
|
| 74 |
+
found = []
|
| 75 |
+
for trick, pat in RULES:
|
| 76 |
+
for m in pat.finditer(text):
|
| 77 |
+
found.append({"trick": trick, "quote": m.group(0),
|
| 78 |
+
"why": TECHNIQUES[trick] + "."})
|
| 79 |
+
caps = re.findall(r"\b[A-Z]{4,}\b", text)
|
| 80 |
+
if len(caps) >= 3:
|
| 81 |
+
found.append({"trick": "emotional_language", "quote": " ".join(caps[:4]),
|
| 82 |
+
"why": "Lots of ALL-CAPS words, which push emotion over facts."})
|
| 83 |
+
if text.count("!") >= 3:
|
| 84 |
+
found.append({"trick": "emotional_language", "quote": "!!!",
|
| 85 |
+
"why": "Stacked exclamation points push emotion over facts."})
|
| 86 |
+
seen, out = set(), []
|
| 87 |
+
for f in found: # one finding per trick+quote
|
| 88 |
+
k = (f["trick"], f["quote"].lower())
|
| 89 |
+
if k not in seen:
|
| 90 |
+
seen.add(k)
|
| 91 |
+
out.append(f)
|
| 92 |
+
return out
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
LANG_SCHEMA = {
|
| 96 |
+
"type": "object",
|
| 97 |
+
"properties": {
|
| 98 |
+
"techniques": {"type": "array", "items": {
|
| 99 |
+
"type": "object",
|
| 100 |
+
"properties": {
|
| 101 |
+
"trick": {"type": "string", "enum": list(TECHNIQUES)},
|
| 102 |
+
"quote": {"type": "string"},
|
| 103 |
+
"why": {"type": "string"},
|
| 104 |
+
},
|
| 105 |
+
"required": ["trick", "quote", "why"], "additionalProperties": False}},
|
| 106 |
+
"claims": {"type": "array", "items": {"type": "string"}},
|
| 107 |
+
},
|
| 108 |
+
"required": ["techniques", "claims"], "additionalProperties": False,
|
| 109 |
+
}
|
| 110 |
+
|
| 111 |
+
LANG_PROMPT = f"""You analyze HOW a social media post tries to persuade, for a media literacy app used by teens.
|
| 112 |
+
|
| 113 |
+
1. TECHNIQUES: label manipulation techniques actually present, using ONLY these names:
|
| 114 |
+
{json.dumps(TECHNIQUES, indent=1)}
|
| 115 |
+
Rules:
|
| 116 |
+
- "quote" must be copied EXACTLY from the post (a few words to one sentence).
|
| 117 |
+
- "why" is one plain sentence a 14-year-old understands.
|
| 118 |
+
- Label the technique the same way no matter which political side, group, or topic it's about.
|
| 119 |
+
- Never say whether an opinion is right or wrong. Only name techniques.
|
| 120 |
+
- If none are present, return an empty list. Don't stretch to find something.
|
| 121 |
+
|
| 122 |
+
2. CLAIMS: list up to {MAX_CLAIMS} specific, checkable factual claims in the post (things that
|
| 123 |
+
could be confirmed or disproven with sources, like numbers, events, quotes, or study results).
|
| 124 |
+
Rewrite each as a short, neutral, standalone sentence. Skip opinions and predictions.
|
| 125 |
+
|
| 126 |
+
POST:
|
| 127 |
+
"""
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _client():
|
| 131 |
+
key = os.environ.get("ANTHROPIC_API_KEY")
|
| 132 |
+
if not key:
|
| 133 |
+
return None
|
| 134 |
+
import anthropic
|
| 135 |
+
return anthropic.Anthropic(api_key=key)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def llm_language(text, client):
|
| 139 |
+
resp = client.messages.create(
|
| 140 |
+
model=LANG_MODEL, max_tokens=1500,
|
| 141 |
+
messages=[{"role": "user", "content": LANG_PROMPT + text}],
|
| 142 |
+
output_config={"format": {"type": "json_schema", "schema": LANG_SCHEMA}},
|
| 143 |
+
)
|
| 144 |
+
data = json.loads(next(b.text for b in resp.content if b.type == "text"))
|
| 145 |
+
# keep only quotes that really appear in the post (guards against made-up quotes)
|
| 146 |
+
norm = lambda s: re.sub(r"\s+", " ", s).strip().lower()
|
| 147 |
+
post = norm(text)
|
| 148 |
+
techniques = [t for t in data["techniques"] if norm(t["quote"]) and norm(t["quote"]) in post]
|
| 149 |
+
return techniques, data["claims"][:MAX_CLAIMS]
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
# =============================== Part 2: source trace ===============================
|
| 153 |
+
|
| 154 |
+
TRACE_PROMPT = """You are checking ONE claim from a social media post for a media literacy app.
|
| 155 |
+
|
| 156 |
+
Search the web for reliable sources about the claim (official data, government or university
|
| 157 |
+
sources, established news outlets, published studies, professional fact-checkers).
|
| 158 |
+
Base your answer ONLY on what the sources you find say, not on your own memory.
|
| 159 |
+
|
| 160 |
+
Reply in exactly this format:
|
| 161 |
+
VERDICT: <one of SUPPORTED, CONTRADICTED, MISSING_CONTEXT, DISPUTED, NO_SOURCE>
|
| 162 |
+
SUMMARY: <2 or 3 plain sentences a teen understands: what the sources actually say, and how it compares to the claim>
|
| 163 |
+
|
| 164 |
+
Use NO_SOURCE if you can't find a reliable source. Never guess.
|
| 165 |
+
|
| 166 |
+
CLAIM: """
|
| 167 |
+
|
| 168 |
+
|
| 169 |
+
def _run_with_search(client, prompt):
|
| 170 |
+
"""Calls Claude with the web search tool; follows pause_turn up to 2 times."""
|
| 171 |
+
messages = [{"role": "user", "content": prompt}]
|
| 172 |
+
blocks = []
|
| 173 |
+
for _ in range(3):
|
| 174 |
+
resp = client.messages.create(
|
| 175 |
+
model=SEARCH_MODEL, max_tokens=1200, messages=messages,
|
| 176 |
+
tools=[{"type": "web_search_20250305", "name": "web_search",
|
| 177 |
+
"max_uses": MAX_SEARCHES_PER_CLAIM}],
|
| 178 |
+
)
|
| 179 |
+
blocks += list(resp.content)
|
| 180 |
+
if resp.stop_reason != "pause_turn":
|
| 181 |
+
break
|
| 182 |
+
messages = messages + [{"role": "assistant", "content": resp.content}]
|
| 183 |
+
return blocks
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def parse_trace(blocks):
|
| 187 |
+
"""Turns API content blocks into verdict, summary, and REAL cited sources."""
|
| 188 |
+
text = "".join(getattr(b, "text", "") for b in blocks if getattr(b, "type", "") == "text")
|
| 189 |
+
sources, seen = [], set()
|
| 190 |
+
for b in blocks:
|
| 191 |
+
for c in (getattr(b, "citations", None) or []):
|
| 192 |
+
url = getattr(c, "url", None)
|
| 193 |
+
if url and url not in seen:
|
| 194 |
+
seen.add(url)
|
| 195 |
+
sources.append({"title": getattr(c, "title", url) or url, "url": url,
|
| 196 |
+
"quote": getattr(c, "cited_text", "")})
|
| 197 |
+
searched = sum(1 for b in blocks if getattr(b, "type", "") == "server_tool_use")
|
| 198 |
+
m = re.search(r"VERDICT:\s*([A-Z_]+)", text)
|
| 199 |
+
verdict = m.group(1) if m and m.group(1) in OUTCOMES else "NO_SOURCE"
|
| 200 |
+
s = re.search(r"SUMMARY:\s*(.+)", text, re.S)
|
| 201 |
+
summary = s.group(1).strip() if s else text.strip()
|
| 202 |
+
if verdict != "NO_SOURCE" and not sources:
|
| 203 |
+
# a verdict with no real citation is not trustworthy: downgrade it
|
| 204 |
+
verdict, summary = "NO_SOURCE", "TELL couldn't attach a real source to an answer, so it won't give one."
|
| 205 |
+
return {"verdict": verdict, "summary": summary[:600], "sources": sources[:4], "searches": searched}
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def google_factchecks(claim):
|
| 209 |
+
"""Published fact-checks from professional fact-checkers (Google Fact Check Tools API)."""
|
| 210 |
+
key = os.environ.get("GOOGLE_FACTCHECK_KEY")
|
| 211 |
+
if not key:
|
| 212 |
+
return []
|
| 213 |
+
try:
|
| 214 |
+
import requests
|
| 215 |
+
r = requests.get("https://factchecktools.googleapis.com/v1alpha1/claims:search",
|
| 216 |
+
params={"query": claim[:200], "key": key, "languageCode": "en", "pageSize": 3},
|
| 217 |
+
timeout=8)
|
| 218 |
+
out = []
|
| 219 |
+
for c in r.json().get("claims", []):
|
| 220 |
+
for rev in c.get("claimReview", [])[:1]:
|
| 221 |
+
out.append({"publisher": rev.get("publisher", {}).get("name", "Fact-checker"),
|
| 222 |
+
"rating": rev.get("textualRating", ""),
|
| 223 |
+
"title": rev.get("title", c.get("text", "")),
|
| 224 |
+
"url": rev.get("url", "")})
|
| 225 |
+
return out[:3]
|
| 226 |
+
except Exception as e:
|
| 227 |
+
print("Fact check API error:", e)
|
| 228 |
+
return []
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def trace_claim(client, claim, place=None):
|
| 232 |
+
prompt = TRACE_PROMPT + claim
|
| 233 |
+
if place: # only when the user shared their location
|
| 234 |
+
prompt += f"\n\nThe reader is near {place}. If the claim is about a local place or event, search news for that area."
|
| 235 |
+
try:
|
| 236 |
+
result = parse_trace(_run_with_search(client, prompt))
|
| 237 |
+
except Exception as e:
|
| 238 |
+
print("Source trace error:", e)
|
| 239 |
+
result = {"verdict": "NO_SOURCE", "summary": "The source search failed, so TELL can't say.",
|
| 240 |
+
"sources": [], "searches": 0}
|
| 241 |
+
result["claim"] = claim
|
| 242 |
+
result["factchecks"] = google_factchecks(claim)
|
| 243 |
+
return result
|
| 244 |
+
|
| 245 |
+
|
| 246 |
+
# =============================== Main entry ===============================
|
| 247 |
+
|
| 248 |
+
_cache = {}
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
def analyze(text, trace_sources=True, place=None):
|
| 252 |
+
if not text or not text.strip():
|
| 253 |
+
return {"tool": "LangCheck", "findings": [], "claim_checks": [], "confidence": "n/a"}
|
| 254 |
+
key = hashlib.sha256(f"{trace_sources}|{place}|{text}".encode()).hexdigest()
|
| 255 |
+
if key in _cache: # same post twice: don't pay for the same searches again
|
| 256 |
+
return _cache[key]
|
| 257 |
+
|
| 258 |
+
client = _client()
|
| 259 |
+
claims, mode = [], "rules only (no API key)"
|
| 260 |
+
if client:
|
| 261 |
+
try:
|
| 262 |
+
findings, claims = llm_language(text, client)
|
| 263 |
+
mode = "AI language model"
|
| 264 |
+
except Exception as e:
|
| 265 |
+
print("LangCheck language error:", e)
|
| 266 |
+
findings, mode = rule_language(text), "rules (AI call failed)"
|
| 267 |
+
else:
|
| 268 |
+
findings = rule_language(text)
|
| 269 |
+
|
| 270 |
+
checks = []
|
| 271 |
+
if trace_sources and client and claims:
|
| 272 |
+
with cf.ThreadPoolExecutor(max_workers=MAX_CLAIMS) as pool:
|
| 273 |
+
checks = list(pool.map(lambda c: trace_claim(client, c, place), claims))
|
| 274 |
+
|
| 275 |
+
result = {"tool": "LangCheck", "findings": findings, "claim_checks": checks,
|
| 276 |
+
"confidence": f"language: {mode}; sources: "
|
| 277 |
+
+ (f"{len(checks)} claim(s) traced" if checks else "not traced")}
|
| 278 |
+
_cache[key] = result
|
| 279 |
+
return result
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
if __name__ == "__main__":
|
| 283 |
+
demo = ("SHARE THIS BEFORE IT'S DELETED!!! Experts say the new school lunch program is a "
|
| 284 |
+
"total disaster and NOBODY is talking about it. Either we stop it now or our kids "
|
| 285 |
+
"will never eat a real meal again.")
|
| 286 |
+
for f in analyze(demo)["findings"]:
|
| 287 |
+
print(f"{f['trick']:20s} | {f['quote']}")
|
tools/screenshotcheck.py
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
ScreenshotCheck: finds signs that a chat or post screenshot was faked or edited.
|
| 3 |
+
|
| 4 |
+
With a trained model (SCREENSHOT_MODEL set): tiles the screenshot into patches,
|
| 5 |
+
scores each one, and builds a heatmap of where an edit probably is.
|
| 6 |
+
Without a model: shows an error-level-analysis heatmap as a visual aid only.
|
| 7 |
+
ELA is NOT used to flag anything, because it gave false alarms on clean PNG
|
| 8 |
+
screenshots in testing.
|
| 9 |
+
"""
|
| 10 |
+
|
| 11 |
+
import io
|
| 12 |
+
import json
|
| 13 |
+
import os
|
| 14 |
+
|
| 15 |
+
import numpy as np
|
| 16 |
+
from PIL import Image, ImageChops, ImageEnhance
|
| 17 |
+
|
| 18 |
+
MODEL_REPO = os.environ.get("SCREENSHOT_MODEL") # "your-hf-name/tell-screenshot" or a local folder
|
| 19 |
+
_model, _cfg = None, None
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
# ---------- Fallback visual: error level analysis ----------
|
| 23 |
+
|
| 24 |
+
def error_level_heatmap(image, quality=90):
|
| 25 |
+
original = image.convert("RGB")
|
| 26 |
+
buf = io.BytesIO()
|
| 27 |
+
original.save(buf, "JPEG", quality=quality)
|
| 28 |
+
buf.seek(0)
|
| 29 |
+
diff = ImageChops.difference(original, Image.open(buf))
|
| 30 |
+
max_diff = max(e[1] for e in diff.getextrema()) or 1
|
| 31 |
+
return ImageEnhance.Brightness(diff).enhance(255.0 / max_diff)
|
| 32 |
+
|
| 33 |
+
|
| 34 |
+
# ---------- Trained model ----------
|
| 35 |
+
|
| 36 |
+
def _load():
|
| 37 |
+
global _model, _cfg
|
| 38 |
+
if _model is not None:
|
| 39 |
+
return
|
| 40 |
+
import timm
|
| 41 |
+
from safetensors.torch import load_file
|
| 42 |
+
if os.path.isdir(MODEL_REPO):
|
| 43 |
+
cfg_path = os.path.join(MODEL_REPO, "config.json")
|
| 44 |
+
w_path = os.path.join(MODEL_REPO, "model.safetensors")
|
| 45 |
+
else:
|
| 46 |
+
from huggingface_hub import hf_hub_download
|
| 47 |
+
tok = os.environ.get("HF_TOKEN") # needed if the model repo is private
|
| 48 |
+
cfg_path = hf_hub_download(MODEL_REPO, "config.json", token=tok)
|
| 49 |
+
w_path = hf_hub_download(MODEL_REPO, "model.safetensors", token=tok)
|
| 50 |
+
_cfg = json.load(open(cfg_path))
|
| 51 |
+
m = timm.create_model(_cfg["arch"], pretrained=False, num_classes=len(_cfg["classes"]))
|
| 52 |
+
m.load_state_dict(load_file(w_path))
|
| 53 |
+
_model = m.eval()
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _tile_predict(img):
|
| 57 |
+
"""MUST match tile_predict in training/screen_train.py."""
|
| 58 |
+
import torch
|
| 59 |
+
P, S, mn = _cfg["patch"], _cfg["stride"], _cfg["min_std"]
|
| 60 |
+
mean, std = np.array(_cfg["mean"]), np.array(_cfg["std"])
|
| 61 |
+
img = img.convert("RGB")
|
| 62 |
+
if img.width < P or img.height < P:
|
| 63 |
+
img = img.resize((max(img.width, P), max(img.height, P)))
|
| 64 |
+
xs = list(range(0, img.width - P + 1, S)) or [0]
|
| 65 |
+
ys = list(range(0, img.height - P + 1, S)) or [0]
|
| 66 |
+
coords, tens = [], []
|
| 67 |
+
for y in ys:
|
| 68 |
+
for x in xs:
|
| 69 |
+
p = img.crop((x, y, x + P, y + P))
|
| 70 |
+
a = np.asarray(p, dtype=np.float32)
|
| 71 |
+
if a.std() >= mn:
|
| 72 |
+
coords.append((x, y))
|
| 73 |
+
a = (a / 255.0 - mean) / std
|
| 74 |
+
tens.append(torch.from_numpy(a.transpose(2, 0, 1)).float())
|
| 75 |
+
out = []
|
| 76 |
+
with torch.no_grad():
|
| 77 |
+
for i in range(0, len(tens), 32):
|
| 78 |
+
probs = torch.softmax(_model(torch.stack(tens[i:i + 32])), 1)
|
| 79 |
+
out += [(*coords[i + k], *probs[k].tolist()) for k in range(len(probs))]
|
| 80 |
+
return img, out
|
| 81 |
+
|
| 82 |
+
|
| 83 |
+
def _heatmap(img, patches):
|
| 84 |
+
"""Red overlay where the model thinks an edit is."""
|
| 85 |
+
P = _cfg["patch"]
|
| 86 |
+
heat = np.zeros((img.height, img.width), dtype=np.float32)
|
| 87 |
+
count = np.zeros_like(heat)
|
| 88 |
+
for x, y, _r, _g, e in patches:
|
| 89 |
+
heat[y:y + P, x:x + P] += e
|
| 90 |
+
count[y:y + P, x:x + P] += 1
|
| 91 |
+
heat = np.divide(heat, count, out=np.zeros_like(heat), where=count > 0)
|
| 92 |
+
base = np.asarray(img.convert("RGB"), dtype=np.float32)
|
| 93 |
+
red = np.zeros_like(base)
|
| 94 |
+
red[..., 0] = 255
|
| 95 |
+
alpha = (heat[..., None] * 0.6)
|
| 96 |
+
return Image.fromarray((base * (1 - alpha) + red * alpha).astype(np.uint8))
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
def analyze(image):
|
| 100 |
+
if not MODEL_REPO:
|
| 101 |
+
return {"tool": "ScreenshotCheck", "findings": [],
|
| 102 |
+
"heatmap": error_level_heatmap(image),
|
| 103 |
+
"confidence": "heatmap only (model not trained yet)"}
|
| 104 |
+
try:
|
| 105 |
+
_load()
|
| 106 |
+
img, patches = _tile_predict(image)
|
| 107 |
+
gen = float(np.mean([p[3] for p in patches])) if patches else 0.0
|
| 108 |
+
edit = float(np.mean(sorted([p[4] for p in patches], reverse=True)[:2])) if patches else 0.0
|
| 109 |
+
findings = []
|
| 110 |
+
if gen >= _cfg["t_generated"]:
|
| 111 |
+
findings.append({"trick": "screenshot_generated",
|
| 112 |
+
"why": f"This looks like it was made with a fake-chat generator ({gen:.0%} score)."})
|
| 113 |
+
elif edit >= _cfg["t_edited"]:
|
| 114 |
+
findings.append({"trick": "screenshot_edited",
|
| 115 |
+
"why": f"Part of this screenshot looks edited ({edit:.0%} score). The red area shows where."})
|
| 116 |
+
conf = f"edited {edit:.0%}"
|
| 117 |
+
if _cfg["t_generated"] <= 1: # above 1 means this model wasn't trained on generator fakes
|
| 118 |
+
conf = f"generated {gen:.0%}, " + conf
|
| 119 |
+
return {"tool": "ScreenshotCheck", "findings": findings,
|
| 120 |
+
"heatmap": _heatmap(img, patches), "confidence": conf}
|
| 121 |
+
except Exception as e:
|
| 122 |
+
print("ScreenshotCheck model error:", e)
|
| 123 |
+
return {"tool": "ScreenshotCheck", "findings": [], "heatmap": error_level_heatmap(image),
|
| 124 |
+
"confidence": "model failed to load; heatmap only"}
|
tools/statcheck.py
ADDED
|
@@ -0,0 +1,163 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
StatCheck: finds number tricks in headlines, captions, and posts.
|
| 3 |
+
|
| 4 |
+
Two layers:
|
| 5 |
+
1. A language model pulls out every numerical claim as strict JSON (optional,
|
| 6 |
+
needs ANTHROPIC_API_KEY). If no key is set, StatCheck still runs on rules.
|
| 7 |
+
2. Rule checks are the final judge. They decide which tricks are flagged.
|
| 8 |
+
|
| 9 |
+
Tricks it catches:
|
| 10 |
+
- relative_risk_no_baseline: "doubles your risk" with no starting number
|
| 11 |
+
- big_percent_small_base: "up 400%!" where the base is tiny or unstated
|
| 12 |
+
- missing_denominator: "3,000 people got sick" with no "out of"
|
| 13 |
+
- correlation_as_cause: a study or "link" described with cause words
|
| 14 |
+
"""
|
| 15 |
+
|
| 16 |
+
import json
|
| 17 |
+
import os
|
| 18 |
+
import re
|
| 19 |
+
|
| 20 |
+
# ---------- Rule patterns ----------
|
| 21 |
+
|
| 22 |
+
RELATIVE_WORDS = re.compile(
|
| 23 |
+
r"\b(doubles?|doubled|triples?|tripled|quadruples?|halves|halved|"
|
| 24 |
+
r"\d+(\.\d+)?\s*(x|times)\s+(more|less|higher|lower|as)\b|"
|
| 25 |
+
r"\d+(\.\d+)?\s*%\s*(higher|lower|more|less|increase|decrease|greater)\b[^.]*\brisk|"
|
| 26 |
+
r"(raises?|increases?|cuts?|lowers?|reduces?)\s+(your\s+)?risk)",
|
| 27 |
+
re.IGNORECASE,
|
| 28 |
+
)
|
| 29 |
+
|
| 30 |
+
BASELINE_WORDS = re.compile(
|
| 31 |
+
r"\b(\d+\s+in\s+[\d,]+|\d+\s+out\s+of\s+[\d,]+|per\s+[\d,]+|"
|
| 32 |
+
r"from\s+\d+(\.\d+)?\s*%?\s+to\s+\d+(\.\d+)?\s*%?|baseline|absolute\s+risk)\b",
|
| 33 |
+
re.IGNORECASE,
|
| 34 |
+
)
|
| 35 |
+
|
| 36 |
+
# 3+ digit percent, but not "100% of people" (that's a share, not a jump). Values over 100%
|
| 37 |
+
# can only be changes, so they always count; exactly 100% counts only next to change words.
|
| 38 |
+
BIG_PERCENT = re.compile(r"\b(\d{3,})\s*%(?!\s+of\b)", re.IGNORECASE)
|
| 39 |
+
CHANGE_WORDS = re.compile(r"\b(increase|rise|rose|jump|spike|surge|up|more|growth|grew|higher)\w*", re.IGNORECASE)
|
| 40 |
+
|
| 41 |
+
COUNT_CLAIM = re.compile(
|
| 42 |
+
r"\b([\d,]{3,}|\d+(\.\d+)?\s*(thousand|million))\s+"
|
| 43 |
+
r"(people|students|kids|teens|cases|users|deaths|patients)\b",
|
| 44 |
+
re.IGNORECASE,
|
| 45 |
+
)
|
| 46 |
+
DENOMINATOR = re.compile(r"\b(out\s+of|of\s+(the|all)|per\s+[\d,]+|%)", re.IGNORECASE)
|
| 47 |
+
|
| 48 |
+
STUDY_WORDS = re.compile(r"\b(study|studies|research|survey|data|scientists)\b", re.IGNORECASE)
|
| 49 |
+
CAUSE_WORDS = re.compile(
|
| 50 |
+
r"\b(causes?|caused|makes?\s+you|leads?\s+to|turns?\s+you|"
|
| 51 |
+
r"is\s+why|proves?|will\s+give\s+you)\b",
|
| 52 |
+
re.IGNORECASE,
|
| 53 |
+
)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def _sentences(text):
|
| 57 |
+
parts = re.split(r"(?<=[.!?])\s+|\n+", text.strip())
|
| 58 |
+
return [p for p in parts if p]
|
| 59 |
+
|
| 60 |
+
|
| 61 |
+
def rule_checks(text):
|
| 62 |
+
"""Return a list of findings. Each finding names a trick and quotes the line."""
|
| 63 |
+
findings = []
|
| 64 |
+
for s in _sentences(text):
|
| 65 |
+
if RELATIVE_WORDS.search(s) and not BASELINE_WORDS.search(text):
|
| 66 |
+
findings.append({
|
| 67 |
+
"trick": "relative_risk_no_baseline",
|
| 68 |
+
"quote": s,
|
| 69 |
+
"why": "It says how much the risk changed, but never how big the risk was to begin with.",
|
| 70 |
+
})
|
| 71 |
+
m = BIG_PERCENT.search(s)
|
| 72 |
+
if m and (int(m.group(1)) > 100 or CHANGE_WORDS.search(s)) and not BASELINE_WORDS.search(s):
|
| 73 |
+
findings.append({
|
| 74 |
+
"trick": "big_percent_small_base",
|
| 75 |
+
"quote": s,
|
| 76 |
+
"why": f"A {m.group(1)}% jump sounds huge, but it can come from a tiny starting number.",
|
| 77 |
+
})
|
| 78 |
+
if COUNT_CLAIM.search(s) and not DENOMINATOR.search(s):
|
| 79 |
+
findings.append({
|
| 80 |
+
"trick": "missing_denominator",
|
| 81 |
+
"quote": s,
|
| 82 |
+
"why": "A raw count with no 'out of how many' can't tell you if this is common or rare.",
|
| 83 |
+
})
|
| 84 |
+
if STUDY_WORDS.search(s) and CAUSE_WORDS.search(s):
|
| 85 |
+
findings.append({
|
| 86 |
+
"trick": "correlation_as_cause",
|
| 87 |
+
"quote": s,
|
| 88 |
+
"why": "Studies usually find a link. Words like 'causes' claim more than a link proves.",
|
| 89 |
+
})
|
| 90 |
+
return findings
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
# ---------- Optional language model layer ----------
|
| 94 |
+
|
| 95 |
+
EXTRACT_PROMPT = """Extract every numerical claim from the text below.
|
| 96 |
+
Return ONLY JSON in this exact shape, no other words:
|
| 97 |
+
{"claims": [{"quote": "...", "relative_change": "e.g. doubles / 2x / +40% or null",
|
| 98 |
+
"baseline_before": "e.g. 1 in 10,000 or null", "baseline_after": "or null"}]}
|
| 99 |
+
Only fill baseline fields with numbers that are literally in the text. Never invent numbers.
|
| 100 |
+
|
| 101 |
+
TEXT:
|
| 102 |
+
"""
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def llm_extract(text):
|
| 106 |
+
"""Ask a language model for structured claims. Returns [] if no API key."""
|
| 107 |
+
key = os.environ.get("ANTHROPIC_API_KEY")
|
| 108 |
+
if not key:
|
| 109 |
+
return []
|
| 110 |
+
try:
|
| 111 |
+
import anthropic
|
| 112 |
+
client = anthropic.Anthropic(api_key=key)
|
| 113 |
+
msg = client.messages.create(
|
| 114 |
+
model=os.environ.get("TELL_LLM_MODEL", "claude-haiku-5-5"),
|
| 115 |
+
max_tokens=800,
|
| 116 |
+
messages=[{"role": "user", "content": EXTRACT_PROMPT + text}],
|
| 117 |
+
)
|
| 118 |
+
raw = msg.content[0].text
|
| 119 |
+
raw = raw[raw.find("{"): raw.rfind("}") + 1]
|
| 120 |
+
return json.loads(raw).get("claims", [])
|
| 121 |
+
except Exception as e: # never let the LLM crash the app
|
| 122 |
+
print("StatCheck LLM error:", e)
|
| 123 |
+
return []
|
| 124 |
+
|
| 125 |
+
|
| 126 |
+
def absolute_view(claims):
|
| 127 |
+
"""Turn 'before/after' baselines the text actually gave into a plain sentence."""
|
| 128 |
+
lines = []
|
| 129 |
+
for c in claims:
|
| 130 |
+
b, a = c.get("baseline_before"), c.get("baseline_after")
|
| 131 |
+
if b and a:
|
| 132 |
+
lines.append(f"In real numbers: {b} becomes {a}.")
|
| 133 |
+
return lines
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def analyze(text):
|
| 137 |
+
"""Main entry point used by app.py."""
|
| 138 |
+
if not text or not text.strip():
|
| 139 |
+
return {"tool": "StatCheck", "findings": [], "confidence": "n/a", "notes": ["No text given."]}
|
| 140 |
+
findings = rule_checks(text)
|
| 141 |
+
claims = llm_extract(text)
|
| 142 |
+
notes = absolute_view(claims)
|
| 143 |
+
return {
|
| 144 |
+
"tool": "StatCheck",
|
| 145 |
+
"findings": findings,
|
| 146 |
+
"claims": claims,
|
| 147 |
+
"notes": notes,
|
| 148 |
+
"confidence": "high" if findings else "nothing flagged",
|
| 149 |
+
}
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
if __name__ == "__main__":
|
| 153 |
+
tests = [
|
| 154 |
+
"New study: energy drinks double your risk of heart problems!",
|
| 155 |
+
"Scooter injuries are up 400% this year.",
|
| 156 |
+
"3,000 students said they hate the new schedule.",
|
| 157 |
+
"Research shows that social media causes depression in teens.",
|
| 158 |
+
"Risk went from 1 in 10,000 to 2 in 10,000 after the change.",
|
| 159 |
+
]
|
| 160 |
+
for t in tests:
|
| 161 |
+
print(t)
|
| 162 |
+
for f in analyze(t)["findings"]:
|
| 163 |
+
print(" ->", f["trick"])
|
training/chart_train.py
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
TELL ChartCheck training: fine-tunes a pretrained image model to name chart tricks.
|
| 3 |
+
|
| 4 |
+
Pipeline:
|
| 5 |
+
synthetic charts (chartgen.py) -> split -> train EfficientNet-B0 -> pick best epoch
|
| 6 |
+
on validation macro-F1 -> test on held-out synthetic AND on real Misviz charts.
|
| 7 |
+
|
| 8 |
+
Key decisions (document these):
|
| 9 |
+
- NO horizontal or vertical flips. A vertical flip literally creates an
|
| 10 |
+
"inverted axis", and a horizontal flip moves the y-axis to the wrong side.
|
| 11 |
+
Normal image augmentation would teach the model wrong labels.
|
| 12 |
+
- Pad to square, then resize. Squashing a wide chart distorts text and bars.
|
| 13 |
+
- Mild augmentations that match real life: JPEG compression (screenshots get
|
| 14 |
+
recompressed on social media), small scale changes, color jitter, blur.
|
| 15 |
+
- Model picked by validation macro-F1 (treats every trick equally), not accuracy.
|
| 16 |
+
"""
|
| 17 |
+
|
| 18 |
+
import io
|
| 19 |
+
import json
|
| 20 |
+
import random
|
| 21 |
+
from pathlib import Path
|
| 22 |
+
|
| 23 |
+
import numpy as np
|
| 24 |
+
import torch
|
| 25 |
+
import torch.nn as nn
|
| 26 |
+
from PIL import Image, ImageFilter
|
| 27 |
+
from sklearn.metrics import classification_report, confusion_matrix, f1_score
|
| 28 |
+
|
| 29 |
+
CLASSES = ["none", "truncated_axis", "inverted_axis", "uneven_ticks", "dual_axis"]
|
| 30 |
+
IMG_SIZE = 384
|
| 31 |
+
MEAN = (0.485, 0.456, 0.406)
|
| 32 |
+
STD = (0.229, 0.224, 0.225)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
# ---------- Preprocessing (MUST match tools/chartcheck.py in the app) ----------
|
| 36 |
+
|
| 37 |
+
def pad_resize(img, size=IMG_SIZE):
|
| 38 |
+
img = img.convert("RGB")
|
| 39 |
+
w, h = img.size
|
| 40 |
+
s = max(w, h)
|
| 41 |
+
canvas = Image.new("RGB", (s, s), (255, 255, 255))
|
| 42 |
+
canvas.paste(img, ((s - w) // 2, (s - h) // 2))
|
| 43 |
+
return canvas.resize((size, size), Image.BILINEAR)
|
| 44 |
+
|
| 45 |
+
|
| 46 |
+
def to_tensor(img):
|
| 47 |
+
a = np.asarray(img, dtype=np.float32) / 255.0
|
| 48 |
+
a = (a - MEAN) / STD
|
| 49 |
+
return torch.from_numpy(a.transpose(2, 0, 1)).float()
|
| 50 |
+
|
| 51 |
+
|
| 52 |
+
def augment(img, rng):
|
| 53 |
+
"""Realistic, label-safe augmentations. No flips, no rotations."""
|
| 54 |
+
if rng.random() < 0.5: # social-media JPEG recompression
|
| 55 |
+
buf = io.BytesIO()
|
| 56 |
+
img.convert("RGB").save(buf, "JPEG", quality=rng.randint(45, 95))
|
| 57 |
+
buf.seek(0)
|
| 58 |
+
img = Image.open(buf).convert("RGB")
|
| 59 |
+
if rng.random() < 0.4: # screenshot at a different size
|
| 60 |
+
f = rng.uniform(0.6, 1.0)
|
| 61 |
+
w, h = img.size
|
| 62 |
+
img = img.resize((max(64, int(w * f)), max(64, int(h * f)))).resize((w, h))
|
| 63 |
+
if rng.random() < 0.2:
|
| 64 |
+
img = img.filter(ImageFilter.GaussianBlur(rng.uniform(0.3, 1.0)))
|
| 65 |
+
if rng.random() < 0.3: # slight brightness/contrast change
|
| 66 |
+
a = np.asarray(img).astype(np.float32)
|
| 67 |
+
a = np.clip(a * rng.uniform(0.85, 1.15) + rng.uniform(-15, 15), 0, 255)
|
| 68 |
+
img = Image.fromarray(a.astype(np.uint8))
|
| 69 |
+
return img
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
# ---------- Data ----------
|
| 73 |
+
|
| 74 |
+
class ChartDataset(torch.utils.data.Dataset):
|
| 75 |
+
def __init__(self, rows, img_dir, train=False):
|
| 76 |
+
self.rows, self.img_dir, self.train = rows, Path(img_dir), train
|
| 77 |
+
|
| 78 |
+
def __len__(self):
|
| 79 |
+
return len(self.rows)
|
| 80 |
+
|
| 81 |
+
def __getitem__(self, i):
|
| 82 |
+
r = self.rows[i]
|
| 83 |
+
img = Image.open(self.img_dir / r["path"]).convert("RGB")
|
| 84 |
+
if self.train:
|
| 85 |
+
img = augment(img, random.Random())
|
| 86 |
+
return to_tensor(pad_resize(img)), CLASSES.index(r["label"])
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def split_rows(rows, seed=42, val=0.1, test=0.1):
|
| 90 |
+
"""Stratified split: same class mix in train, val, and test."""
|
| 91 |
+
rng = random.Random(seed)
|
| 92 |
+
out = {"train": [], "val": [], "test": []}
|
| 93 |
+
for c in CLASSES:
|
| 94 |
+
cr = [r for r in rows if r["label"] == c]
|
| 95 |
+
rng.shuffle(cr)
|
| 96 |
+
nv, nt = int(len(cr) * val), int(len(cr) * test)
|
| 97 |
+
out["val"] += cr[:nv]
|
| 98 |
+
out["test"] += cr[nv:nv + nt]
|
| 99 |
+
out["train"] += cr[nv + nt:]
|
| 100 |
+
return out
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
# ---------- Model ----------
|
| 104 |
+
|
| 105 |
+
def build_model(arch="efficientnet_b0", pretrained=True):
|
| 106 |
+
import timm
|
| 107 |
+
return timm.create_model(arch, pretrained=pretrained, num_classes=len(CLASSES))
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
@torch.no_grad()
|
| 111 |
+
def predict(model, loader, device):
|
| 112 |
+
model.eval()
|
| 113 |
+
ys, ps = [], []
|
| 114 |
+
for x, y in loader:
|
| 115 |
+
logits = model(x.to(device))
|
| 116 |
+
ps.append(torch.softmax(logits.float(), 1).cpu())
|
| 117 |
+
ys.append(y)
|
| 118 |
+
return torch.cat(ys).numpy(), torch.cat(ps).numpy()
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
def train(data_dir, out_dir, arch="efficientnet_b0", epochs=6, batch=32, lr=3e-4,
|
| 122 |
+
workers=2, limit=None, seed=42, pretrained=True):
|
| 123 |
+
torch.manual_seed(seed)
|
| 124 |
+
random.seed(seed)
|
| 125 |
+
data_dir, out_dir = Path(data_dir), Path(out_dir)
|
| 126 |
+
out_dir.mkdir(parents=True, exist_ok=True)
|
| 127 |
+
rows = json.loads((data_dir / "labels.json").read_text())
|
| 128 |
+
if limit:
|
| 129 |
+
rows = random.Random(seed).sample(rows, min(limit, len(rows)))
|
| 130 |
+
splits = split_rows(rows, seed)
|
| 131 |
+
(out_dir / "splits.json").write_text(json.dumps(splits))
|
| 132 |
+
print({k: len(v) for k, v in splits.items()})
|
| 133 |
+
|
| 134 |
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
| 135 |
+
mk = lambda part, tr: torch.utils.data.DataLoader(
|
| 136 |
+
ChartDataset(splits[part], data_dir / "img", train=tr),
|
| 137 |
+
batch_size=batch, shuffle=tr, num_workers=workers)
|
| 138 |
+
dl_train, dl_val, dl_test = mk("train", True), mk("val", False), mk("test", False)
|
| 139 |
+
|
| 140 |
+
model = build_model(arch, pretrained=pretrained).to(device)
|
| 141 |
+
opt = torch.optim.AdamW(model.parameters(), lr=lr, weight_decay=0.01)
|
| 142 |
+
steps = epochs * len(dl_train)
|
| 143 |
+
sched = torch.optim.lr_scheduler.OneCycleLR(opt, max_lr=lr, total_steps=steps, pct_start=0.1)
|
| 144 |
+
scaler = torch.amp.GradScaler(enabled=device == "cuda")
|
| 145 |
+
loss_fn = nn.CrossEntropyLoss(label_smoothing=0.05)
|
| 146 |
+
|
| 147 |
+
best_f1, history = -1, []
|
| 148 |
+
for ep in range(epochs):
|
| 149 |
+
model.train()
|
| 150 |
+
total = 0.0
|
| 151 |
+
for x, y in dl_train:
|
| 152 |
+
x, y = x.to(device), y.to(device)
|
| 153 |
+
with torch.autocast(device_type=device, enabled=device == "cuda"):
|
| 154 |
+
loss = loss_fn(model(x), y)
|
| 155 |
+
opt.zero_grad()
|
| 156 |
+
scaler.scale(loss).backward()
|
| 157 |
+
scaler.step(opt)
|
| 158 |
+
scaler.update()
|
| 159 |
+
sched.step()
|
| 160 |
+
total += loss.item() * len(y)
|
| 161 |
+
yv, pv = predict(model, dl_val, device)
|
| 162 |
+
f1 = f1_score(yv, pv.argmax(1), average="macro")
|
| 163 |
+
history.append({"epoch": ep + 1, "train_loss": total / len(dl_train.dataset), "val_macro_f1": f1})
|
| 164 |
+
print(history[-1])
|
| 165 |
+
if f1 > best_f1:
|
| 166 |
+
best_f1 = f1
|
| 167 |
+
torch.save(model.state_dict(), out_dir / "best.pt")
|
| 168 |
+
|
| 169 |
+
model.load_state_dict(torch.load(out_dir / "best.pt", map_location=device))
|
| 170 |
+
yt, pt = predict(model, dl_test, device)
|
| 171 |
+
report = classification_report(yt, pt.argmax(1), labels=list(range(len(CLASSES))),
|
| 172 |
+
target_names=CLASSES, output_dict=True, zero_division=0)
|
| 173 |
+
cm = confusion_matrix(yt, pt.argmax(1), labels=list(range(len(CLASSES)))).tolist()
|
| 174 |
+
results = {"history": history, "best_val_macro_f1": best_f1,
|
| 175 |
+
"synthetic_test": report, "synthetic_confusion": cm}
|
| 176 |
+
(out_dir / "results.json").write_text(json.dumps(results, indent=2))
|
| 177 |
+
print(classification_report(yt, pt.argmax(1), labels=list(range(len(CLASSES))),
|
| 178 |
+
target_names=CLASSES, zero_division=0))
|
| 179 |
+
save_for_app(model, out_dir, arch)
|
| 180 |
+
return model, results
|
| 181 |
+
|
| 182 |
+
|
| 183 |
+
def save_for_app(model, out_dir, arch):
|
| 184 |
+
"""Writes the two files the app downloads: model.safetensors + config.json."""
|
| 185 |
+
from safetensors.torch import save_file
|
| 186 |
+
out_dir = Path(out_dir)
|
| 187 |
+
save_file({k: v.contiguous().cpu() for k, v in model.state_dict().items()},
|
| 188 |
+
out_dir / "model.safetensors")
|
| 189 |
+
(out_dir / "config.json").write_text(json.dumps({
|
| 190 |
+
"arch": arch, "classes": CLASSES, "img_size": IMG_SIZE,
|
| 191 |
+
"mean": MEAN, "std": STD, "threshold": 0.5}, indent=2))
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
# ---------- Real-world test on Misviz ----------
|
| 195 |
+
|
| 196 |
+
MISVIZ_MAP = {"truncated axis": "truncated_axis", "inverted axis": "inverted_axis",
|
| 197 |
+
"inconsistent tick intervals": "uneven_ticks", "dual axis": "dual_axis"}
|
| 198 |
+
|
| 199 |
+
|
| 200 |
+
def eval_misviz(model, hf_rows, device=None):
|
| 201 |
+
"""
|
| 202 |
+
hf_rows: Misviz rows with 'image' (PIL) and 'misleader' (list of names).
|
| 203 |
+
Only keeps charts whose tricks are ones TELL knows, or that have none.
|
| 204 |
+
A chart counts as caught if TELL's top prediction is one of its true tricks.
|
| 205 |
+
"""
|
| 206 |
+
device = device or ("cuda" if torch.cuda.is_available() else "cpu")
|
| 207 |
+
model.eval().to(device)
|
| 208 |
+
kept, correct, per = 0, 0, {c: [0, 0] for c in CLASSES}
|
| 209 |
+
preds_true = []
|
| 210 |
+
for r in hf_rows:
|
| 211 |
+
mis = r["misleader"] or []
|
| 212 |
+
mapped = {MISVIZ_MAP[m] for m in mis if m in MISVIZ_MAP}
|
| 213 |
+
if mis and len(mapped) < len(mis):
|
| 214 |
+
continue # has a trick TELL doesn't cover yet; skip, don't count
|
| 215 |
+
truth = mapped or {"none"}
|
| 216 |
+
x = to_tensor(pad_resize(r["image"])).unsqueeze(0).to(device)
|
| 217 |
+
with torch.no_grad():
|
| 218 |
+
pred = CLASSES[model(x).argmax(1).item()]
|
| 219 |
+
kept += 1
|
| 220 |
+
hit = pred in truth
|
| 221 |
+
correct += hit
|
| 222 |
+
for t in truth:
|
| 223 |
+
per[t][0] += hit
|
| 224 |
+
per[t][1] += 1
|
| 225 |
+
preds_true.append((pred, sorted(truth)))
|
| 226 |
+
return {"charts_scored": kept,
|
| 227 |
+
"accuracy": correct / max(kept, 1),
|
| 228 |
+
"per_class_recall": {c: (h / n if n else None) for c, (h, n) in per.items()},
|
| 229 |
+
"per_class_count": {c: n for c, (h, n) in per.items()},
|
| 230 |
+
"examples": preds_true[:50]}
|
training/chartgen.py
ADDED
|
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""
|
| 2 |
+
TELL ChartCheck synthetic chart generator.
|
| 3 |
+
|
| 4 |
+
Makes labeled charts with matplotlib so every label is known for sure.
|
| 5 |
+
Classes (names match the app's lesson keys):
|
| 6 |
+
none honest chart
|
| 7 |
+
truncated_axis BAR chart whose y-axis starts well above zero
|
| 8 |
+
inverted_axis y-axis flipped so bigger values sit lower
|
| 9 |
+
uneven_ticks equal visual spacing between ticks, unequal values (0,10,20,50,100)
|
| 10 |
+
dual_axis two series on two different y-axes
|
| 11 |
+
|
| 12 |
+
Design notes (worth putting in your documentation):
|
| 13 |
+
- Truncated axis only applies to BAR charts. A line chart that doesn't start
|
| 14 |
+
at zero is normal and honest, so "none" includes zoomed-in line charts.
|
| 15 |
+
Otherwise the model would wrongly learn "any axis not at zero is a trick".
|
| 16 |
+
- Every chart is randomized (style, colors, size, fonts, titles, gridlines,
|
| 17 |
+
number of categories) so the model learns the trick, not one look.
|
| 18 |
+
"""
|
| 19 |
+
|
| 20 |
+
import json
|
| 21 |
+
import random
|
| 22 |
+
from pathlib import Path
|
| 23 |
+
|
| 24 |
+
import matplotlib
|
| 25 |
+
matplotlib.use("Agg")
|
| 26 |
+
import matplotlib.pyplot as plt
|
| 27 |
+
import numpy as np
|
| 28 |
+
|
| 29 |
+
CLASSES = ["none", "truncated_axis", "inverted_axis", "uneven_ticks", "dual_axis"]
|
| 30 |
+
|
| 31 |
+
TITLES = ["Monthly club members", "Cafeteria pizza sales", "Average commute time",
|
| 32 |
+
"Bike rentals per week", "Library visits", "Concert ticket prices",
|
| 33 |
+
"Rainfall by month", "Game downloads", "Team points per season",
|
| 34 |
+
"Snack prices", "Gym attendance", "Bus riders per day", "App ratings",
|
| 35 |
+
"Plant growth", "Website visitors", "Volunteer hours"]
|
| 36 |
+
CAT_GROUPS = [
|
| 37 |
+
["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"],
|
| 38 |
+
["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"],
|
| 39 |
+
["Q1", "Q2", "Q3", "Q4"],
|
| 40 |
+
["A", "B", "C", "D", "E", "F", "G"],
|
| 41 |
+
["North", "South", "East", "West", "Central"],
|
| 42 |
+
["2018", "2019", "2020", "2021", "2022", "2023", "2024", "2025"],
|
| 43 |
+
["Team 1", "Team 2", "Team 3", "Team 4", "Team 5", "Team 6"],
|
| 44 |
+
]
|
| 45 |
+
STYLES = ["default", "ggplot", "seaborn-v0_8", "seaborn-v0_8-whitegrid", "bmh",
|
| 46 |
+
"fivethirtyeight", "classic", "grayscale", "seaborn-v0_8-pastel"]
|
| 47 |
+
UNITS = ["", "$", "%", " min", " k"]
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
def _setup(rng):
|
| 51 |
+
plt.style.use(rng.choice(STYLES))
|
| 52 |
+
w = rng.uniform(4.5, 8.0)
|
| 53 |
+
h = rng.uniform(3.2, 6.0)
|
| 54 |
+
fig, ax = plt.subplots(figsize=(w, h), dpi=rng.choice([80, 100, 120]))
|
| 55 |
+
plt.rcParams["font.size"] = rng.uniform(8, 13)
|
| 56 |
+
return fig, ax
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
def _categories(rng):
|
| 60 |
+
group = rng.choice(CAT_GROUPS)
|
| 61 |
+
n = rng.randint(3, min(8, len(group)))
|
| 62 |
+
start = rng.randint(0, len(group) - n)
|
| 63 |
+
return group[start:start + n]
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def _decorate(ax, rng):
|
| 67 |
+
ax.set_title(rng.choice(TITLES), fontsize=rng.uniform(10, 15))
|
| 68 |
+
if rng.random() < 0.5:
|
| 69 |
+
ax.grid(axis="y", alpha=rng.uniform(0.2, 0.7))
|
| 70 |
+
if rng.random() < 0.3:
|
| 71 |
+
ax.set_ylabel(rng.choice(["Count", "Amount", "Total", "Score", "Value"]))
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def honest(rng, ax):
|
| 75 |
+
cats = _categories(rng)
|
| 76 |
+
if rng.random() < 0.55: # bar chart from zero
|
| 77 |
+
vals = [rng.uniform(5, 100) for _ in cats]
|
| 78 |
+
ax.bar(cats, vals, color=rng.choice(["C0", "C1", "C2", "C3", "C4"]))
|
| 79 |
+
ax.set_ylim(0, max(vals) * rng.uniform(1.05, 1.3))
|
| 80 |
+
else: # line chart: zoomed-in range is fine and honest
|
| 81 |
+
base = rng.uniform(20, 200)
|
| 82 |
+
vals = base + np.cumsum([rng.uniform(-5, 5) for _ in cats])
|
| 83 |
+
ax.plot(cats, vals, marker=rng.choice(["o", "s", None]), linewidth=rng.uniform(1.2, 3))
|
| 84 |
+
if rng.random() < 0.5:
|
| 85 |
+
ax.set_ylim(0, max(vals) * 1.2)
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
def truncated(rng, ax):
|
| 89 |
+
cats = _categories(rng)
|
| 90 |
+
base = rng.uniform(50, 500)
|
| 91 |
+
vals = [base * rng.uniform(0.92, 1.08) for _ in cats]
|
| 92 |
+
ax.bar(cats, vals, color=rng.choice(["C0", "C1", "C2", "C3", "C4"]))
|
| 93 |
+
low = min(vals) * rng.uniform(0.85, 0.98) # starts far above zero
|
| 94 |
+
ax.set_ylim(low, max(vals) * rng.uniform(1.005, 1.03))
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def inverted(rng, ax):
|
| 98 |
+
if rng.random() < 0.5:
|
| 99 |
+
cats = _categories(rng)
|
| 100 |
+
vals = [rng.uniform(5, 100) for _ in cats]
|
| 101 |
+
ax.bar(cats, vals)
|
| 102 |
+
ax.set_ylim(0, max(vals) * 1.15)
|
| 103 |
+
else:
|
| 104 |
+
cats = _categories(rng)
|
| 105 |
+
vals = rng.uniform(20, 200) + np.cumsum([rng.uniform(-5, 8) for _ in cats])
|
| 106 |
+
ax.plot(cats, vals, marker="o")
|
| 107 |
+
ax.invert_yaxis()
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
def uneven(rng, ax):
|
| 111 |
+
"""Ticks equally spaced on screen but with unequal value gaps."""
|
| 112 |
+
tick_vals = sorted(rng.choice([[0, 10, 20, 50, 100], [0, 5, 10, 50, 200],
|
| 113 |
+
[0, 1, 5, 20, 100], [0, 25, 50, 100, 500],
|
| 114 |
+
[0, 2, 4, 10, 40, 100]]))
|
| 115 |
+
pos = list(range(len(tick_vals)))
|
| 116 |
+
to_pos = lambda v: np.interp(v, tick_vals, pos) # nonlinear squish
|
| 117 |
+
cats = _categories(rng)
|
| 118 |
+
vals = [rng.uniform(tick_vals[1], tick_vals[-1]) for _ in cats]
|
| 119 |
+
if rng.random() < 0.6:
|
| 120 |
+
ax.bar(cats, [to_pos(v) for v in vals])
|
| 121 |
+
else:
|
| 122 |
+
ax.plot(cats, [to_pos(v) for v in vals], marker="o")
|
| 123 |
+
ax.set_yticks(pos)
|
| 124 |
+
unit = rng.choice(UNITS)
|
| 125 |
+
ax.set_yticklabels([f"{unit}{v}" if unit == "$" else f"{v}{unit}" for v in tick_vals])
|
| 126 |
+
ax.set_ylim(0, pos[-1] * 1.05)
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
def dual(rng, ax):
|
| 130 |
+
cats = _categories(rng)
|
| 131 |
+
a = [rng.uniform(10, 100) for _ in cats]
|
| 132 |
+
b = [rng.uniform(1000, 9000) for _ in cats]
|
| 133 |
+
if rng.random() < 0.5:
|
| 134 |
+
ax.bar(cats, a, color="C0", alpha=0.8)
|
| 135 |
+
else:
|
| 136 |
+
ax.plot(cats, a, color="C0", marker="o")
|
| 137 |
+
ax2 = ax.twinx()
|
| 138 |
+
ax2.plot(cats, b, color="C3", marker="s")
|
| 139 |
+
ax.tick_params(axis="y", colors="C0")
|
| 140 |
+
ax2.tick_params(axis="y", colors="C3")
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
MAKERS = {"none": honest, "truncated_axis": truncated, "inverted_axis": inverted,
|
| 144 |
+
"uneven_ticks": uneven, "dual_axis": dual}
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def make_chart(label, seed, out_path):
|
| 148 |
+
rng = random.Random(seed)
|
| 149 |
+
np.random.seed(seed % (2**32))
|
| 150 |
+
fig, ax = _setup(rng)
|
| 151 |
+
MAKERS[label](rng, ax)
|
| 152 |
+
_decorate(ax, rng)
|
| 153 |
+
fig.tight_layout()
|
| 154 |
+
fig.savefig(out_path)
|
| 155 |
+
plt.close(fig)
|
| 156 |
+
plt.style.use("default")
|
| 157 |
+
|
| 158 |
+
|
| 159 |
+
def _job(args):
|
| 160 |
+
label, seed, out_path = args
|
| 161 |
+
try:
|
| 162 |
+
make_chart(label, seed, out_path)
|
| 163 |
+
return {"path": out_path.name, "label": label, "seed": seed}
|
| 164 |
+
except Exception as e:
|
| 165 |
+
return {"error": str(e), "label": label, "seed": seed}
|
| 166 |
+
|
| 167 |
+
|
| 168 |
+
def generate(out_dir, per_class=2000, start_seed=0, workers=4):
|
| 169 |
+
"""Make per_class images per class and a labels.json. Seeds make it reproducible."""
|
| 170 |
+
from multiprocessing import Pool
|
| 171 |
+
out_dir = Path(out_dir)
|
| 172 |
+
(out_dir / "img").mkdir(parents=True, exist_ok=True)
|
| 173 |
+
jobs = []
|
| 174 |
+
seed = start_seed
|
| 175 |
+
for label in CLASSES:
|
| 176 |
+
for _ in range(per_class):
|
| 177 |
+
jobs.append((label, seed, out_dir / "img" / f"{label}_{seed:06d}.png"))
|
| 178 |
+
seed += 1
|
| 179 |
+
with Pool(workers) as pool:
|
| 180 |
+
rows = pool.map(_job, jobs, chunksize=32)
|
| 181 |
+
good = [r for r in rows if "error" not in r]
|
| 182 |
+
bad = [r for r in rows if "error" in r]
|
| 183 |
+
(out_dir / "labels.json").write_text(json.dumps(good))
|
| 184 |
+
print(f"made {len(good)} charts, {len(bad)} failed")
|
| 185 |
+
return good
|
| 186 |
+
|
| 187 |
+
|
| 188 |
+
if __name__ == "__main__":
|
| 189 |
+
import sys
|
| 190 |
+
generate(sys.argv[1] if len(sys.argv) > 1 else "synth_charts",
|
| 191 |
+
per_class=int(sys.argv[2]) if len(sys.argv) > 2 else 20)
|