| """Freeze real public pages into fixtures/raw/<id>.txt + <id>.meta.json (run once; fixtures are then immutable). |
| HTML -> text with stdlib only. Issue-tracker pages may arrive as JSON (body + comments) from an official API.""" |
| import json, re, html, os, sys |
| from html.parser import HTMLParser |
|
|
| class T(HTMLParser): |
| def __init__(s): super().__init__(); s.out=[]; s.skip=0 |
| def handle_starttag(s,t,a): |
| if t in("script","style","noscript"): s.skip+=1 |
| if t in("p","br","div","tr","li","h1","h2","h3","h4","table","pre"): s.out.append("\n") |
| if t in("td","th"): s.out.append(" | ") |
| def handle_endtag(s,t): |
| if t in("script","style","noscript"): s.skip-=1 |
| def handle_data(s,d): |
| if not s.skip: s.out.append(d) |
| def h2t(raw): |
| p=T(); p.feed(raw); t="".join(p.out) |
| t=re.sub(r"[ \t\xa0]+"," ",t); return re.sub(r"\n\s*\n+","\n\n",t).strip() |
| def issue_json_to_text(prefix): |
| i=json.load(open(prefix+".json")); c=json.load(open(prefix+"_comments.json")) |
| parts=[f"# {i.get('title','')}\n\n{i.get('body') or ''}"] |
| parts+= [f"\n---\ncomment\n\n{x.get('body') or ''}" for x in c] |
| return "\n".join(parts), "batch-capture", "source:withheld", i.get("title","") |
|
|
