File size: 1,216 Bytes
6cb442b | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 | """Freeze real public pages into fixtures/raw/<id>.txt + <id>.meta.json (run once; fixtures are then immutable).
HTML -> text with stdlib only. Issue-tracker pages may arrive as JSON (body + comments) from an official API."""
import json, re, html, os, sys
from html.parser import HTMLParser
class T(HTMLParser):
def __init__(s): super().__init__(); s.out=[]; s.skip=0
def handle_starttag(s,t,a):
if t in("script","style","noscript"): s.skip+=1
if t in("p","br","div","tr","li","h1","h2","h3","h4","table","pre"): s.out.append("\n")
if t in("td","th"): s.out.append(" | ")
def handle_endtag(s,t):
if t in("script","style","noscript"): s.skip-=1
def handle_data(s,d):
if not s.skip: s.out.append(d)
def h2t(raw):
p=T(); p.feed(raw); t="".join(p.out)
t=re.sub(r"[ \t\xa0]+"," ",t); return re.sub(r"\n\s*\n+","\n\n",t).strip()
def issue_json_to_text(prefix):
i=json.load(open(prefix+".json")); c=json.load(open(prefix+"_comments.json"))
parts=[f"# {i.get('title','')}\n\n{i.get('body') or ''}"]
parts+= [f"\n---\ncomment\n\n{x.get('body') or ''}" for x in c]
return "\n".join(parts), "batch-capture", "source:withheld", i.get("title","")
|