setitools/flashes.py

"""Specular glints: brief flashes folded at a planet's or an asteroid's period."""
import argparse
import csv
import glob
import io
import json
import math
import os
import re
import sys
import time
from concurrent.futures import ThreadPoolExecutor

import numpy as np

from setitools import catalogues, core, science
from setitools.core import num, quiet_warnings


# --------------------------------------------------------------------------------------------------------------

tess20_DOC = r"""Glints in TESS 20 s light curves within 30 pc: 5-sigma flashes of at most three cadences tested for periodicity against draws from good cadences.

    python -I seti.py flashes tess20 targets
    python -I seti.py flashes tess20 fetch [--threads 8] [--per-star N] [--list SHARE.json]
    python -I seti.py flashes tess20 search [--list SHARE.json] [--out glints.json] [--procs N]
    python -I seti.py flashes tess20 merge     (merges the shares' glints_*.json and records the run in the coverage store)"""

tess20_HERE = core.RESULTS
tess20_DL = os.environ.get("SETI_TESS20", core.data("tess20"))
tess20_UA = core.UA


def tess20_get(url, timeout=120):
    """Bytes of a URL, or None when missing or still failing after three tries."""
    return core.get(url, timeout=timeout, retries=3, soft=True, pause=0)


def tess20_targets():
    lists = {}
    for s in range(27, 140):
        b = tess20_get("https://tess.mit.edu/public/target_lists/20s/all_targets_20s_S%03d_v1.csv" % s)
        if not b or not b.lstrip().startswith(b"#"):
            if s > 100:
                break
            continue
        rows = [ln for ln in b.decode(errors="replace").splitlines() if ln and not ln.startswith("#")]
        lists[s] = list(csv.DictReader(io.StringIO("\n".join(rows))))
    print("%d sectors with 20 s target lists, %d target entries" % (len(lists), sum(len(v) for v in lists.values())),
          flush=True)
    near = core.gaia("SELECT source_id, ra, dec, pmra, pmdec, parallax, phot_g_mean_mag, bp_rp FROM "
                         "gaiadr3.gaia_source WHERE parallax > 33.33 AND phot_g_mean_mag < 16")
    ra0 = np.array([float(r["ra"]) - float(r["pmra"] or 0) / 3.6e6 * 16.0 / math.cos(math.radians(float(r["dec"])))
                    for r in near])
    de0 = np.array([float(r["dec"]) - float(r["pmdec"] or 0) / 3.6e6 * 16.0 for r in near])
    out = []
    for s, rows in lists.items():
        for t in rows:
            try:
                ra, dec = float(t["RA"]), float(t["Dec"])
            except (KeyError, ValueError):
                continue
            d = np.hypot((ra0 - ra) * math.cos(math.radians(dec)), de0 - dec) * 3600.0
            k = int(np.argmin(d))
            if d[k] < 10.0:
                g = near[k]
                out.append(dict(tic=int(t["TICID"]), sector=s, gaia=g["source_id"], sep=round(float(d[k]), 2),
                                dist_pc=round(1000.0 / float(g["parallax"]), 2), g=float(g["phot_g_mean_mag"]),
                                bp_rp=float(g["bp_rp"] or "nan")))
    json.dump(out, open(os.path.join(tess20_HERE, "glint_targets.json"), "w"))
    print("%d light curves of %d stars within 30 pc" % (len(out), len({o["gaia"] for o in out})))


def tess20_fetch_one(t):
    path = os.path.join(tess20_DL, "s%04d_%d_fast-lc.fits" % (t["sector"], t["tic"]))
    if os.path.exists(path):
        return True
    keys = core.spoc_keys(t["tic"], t["sector"], fast=True)
    if not keys:
        return False
    b = tess20_get(core.S3 + keys[0], timeout=300)
    if not b or not b.startswith(b"SIMPLE"):
        return False
    open(path + ".tmp", "wb").write(b)
    os.replace(path + ".tmp", path)
    return True


def tess20_fetch(threads, per_star, share=None):
    os.makedirs(tess20_DL, exist_ok=True)
    ts = json.load(open(share or os.path.join(tess20_HERE, "glint_targets.json")))
    if per_star and not share:
        by = {}
        for t in sorted(ts, key=lambda t: t["sector"]):
            by.setdefault(t["gaia"], []).append(t)
        ts = [t for v in by.values() for t in v[:per_star]]
    with ThreadPoolExecutor(threads) as ex:
        ok = list(ex.map(tess20_fetch_one, ts))
    print("%d of %d light curves on disk" % (sum(ok), len(ts)))


def tess20_flashes(t, f, q):
    good = (q == 0) & np.isfinite(f) & np.isfinite(t)
    t, f = t[good], f[good]
    if len(f) < 1000:
        return None
    from scipy.ndimage import median_filter
    base = median_filter(f, size=31, mode="nearest")
    r = f / base - 1.0
    sig = 1.4826 * np.median(np.abs(r - np.median(r)))
    idx = np.nonzero(r > 5 * sig)[0]
    out, seen = [], set()
    for i in idx:
        if i in seen:
            continue
        j = i
        while j + 1 < len(r) and r[j + 1] > 5 * sig and t[j + 1] - t[j] < 30 / 86400.0:
            j += 1
        seen.update(range(i, j + 1))
        if j - i >= 3 or i < 4 or j > len(r) - 5:
            continue
        before, after = r[i - 4:i - 1], r[j + 2:j + 5]
        if np.all(np.abs(before) < 2 * sig) and np.all(np.abs(after) < 2 * sig) and \
                t[i] - t[i - 4] < 120 / 86400.0 and t[j + 4] - t[j] < 120 / 86400.0:
            k = i + int(np.argmax(r[i:j + 1]))
            out.append((float(t[k]), float(r[k] / sig), j - i + 1))
    return out, t, sig


def tess20_best_period(times, tol=40.0 / 86400.0):
    times = np.sort(np.asarray(times))
    best = (0, None)
    for a in range(len(times)):
        for b in range(a + 1, len(times)):
            for k in range(1, 6):
                P = (times[b] - times[a]) / k
                if P < 0.01:
                    continue
                ph = (times - times[a]) / P
                n = int(np.sum(np.abs(ph - np.round(ph)) * P < tol))
                if n > best[0]:
                    best = (n, P)
    return best


def tess20_one_curve(p):
    """Flashes in one light-curve file: (name, flashes, good times) or None."""
    from astropy.io import fits
    try:
        with fits.open(os.path.join(tess20_DL, p)) as h:
            d = h[1].data
            res = tess20_flashes(np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float),
                          np.asarray(d["QUALITY"], int))
    except Exception:
        return None
    if res is None:
        return None
    return p, res[0], res[1].astype(np.float32)


def tess20_search(share=None, out="glints.json", procs=1):
    ts = {(t["sector"], t["tic"]): t for t in json.load(open(share or os.path.join(tess20_HERE, "glint_targets.json")))}
    files = [p for p in sorted(os.listdir(tess20_DL)) if p.endswith("_fast-lc.fits")
             and (int(p.split("_")[0][1:]), int(p.split("_")[1])) in ts]
    if procs > 1:
        from multiprocessing import Pool
        with Pool(procs) as pool:
            results = pool.map(tess20_one_curve, files, chunksize=4)
    else:
        results = [tess20_one_curve(p) for p in files]
    per_star = {}
    stats = dict(curves=0, flashes=0, hours=0.0)
    for item in results:
        if item is None:
            continue
        p, fl, tg = item
        s, tic = p.split("_")[0], int(p.split("_")[1])
        meta = ts.get((int(s[1:]), tic))
        stats["curves"] += 1
        stats["flashes"] += len(fl)
        stats["hours"] += len(tg) * 20.0 / 3600.0
        key = meta["gaia"] if meta else str(tic)
        e = per_star.setdefault(key, dict(meta=meta, flashes=[], good_times=[]))
        e["flashes"] += [dict(sector=int(s[1:]), t=x[0], snr=round(x[1], 1), n=x[2]) for x in fl]
        e["good_times"].append(tg)
    print("%d light curves, %.0f star-hours at 20 s, %d short symmetric flashes (%.2f per star-day)"
          % (stats["curves"], stats["hours"], stats["flashes"], stats["flashes"] / max(stats["hours"] / 24.0, 1e-9)))
    rng = np.random.default_rng(3)
    report = []
    for key, e in per_star.items():
        # the 60 strongest flashes carry any periodic family
        times = [x["t"] for x in sorted(e["flashes"], key=lambda x: -x["snr"])[:60]]
        if len(times) < 3:
            continue
        n, P = tess20_best_period(times)
        allt = np.concatenate(e["good_times"])
        sims = []
        for _ in range(200):
            sims.append(tess20_best_period(rng.choice(allt, size=len(times), replace=False))[0])
        p = (1 + sum(1 for v in sims if v >= n)) / 201.0
        report.append(dict(gaia=key, meta=e["meta"], n_flashes=len(times), best_n=n, period_d=P, p=p,
                           flashes=e["flashes"]))
    report.sort(key=lambda r: r["p"])
    stars_all = [dict(gaia=k, meta=e["meta"], n_flashes=len(e["flashes"]), flashes=e["flashes"]) for k, e in per_star.items()]
    json.dump(dict(stats=stats, stars=report, all=stars_all), open(os.path.join(tess20_HERE, out), "w"), default=float)
    print("%d stars with three or more flashes" % len(report))
    for r in report[:12]:
        m = r["meta"] or {}
        print("  Gaia DR3 %s (%.1f pc, G %.1f, BP-RP %.2f): %d flashes, %d on one period of %.4f d, p %.3f"
              % (r["gaia"], m.get("dist_pc", 0), m.get("g", 0), m.get("bp_rp", 0), r["n_flashes"], r["best_n"],
                 r["period_d"] or 0, r["p"]))


def tess20_merge():
    """Merge shares, compare periodicity p-values with uniform, record one row per star."""
    stats = dict(curves=0, flashes=0, hours=0.0)
    report, every = [], {}
    import glob
    for path in sorted(glob.glob(os.path.join(tess20_HERE, "glints_*.json"))):
        d = json.load(open(path))
        if "stats" not in d:
            continue
        for k in stats:
            stats[k] += d["stats"][k]
        report += d["stars"]
        for s in d["all"]:
            e = every.setdefault(s["gaia"], dict(n=0))
            e["n"] += s["n_flashes"]
    ps = np.array([r["p"] for r in report])
    null = {str(c): dict(n=int((ps <= c + 1e-9).sum()), expected=round(c * len(ps), 1)) for c in (0.005, 0.01, 0.02, 0.05)}
    print("%d light curves, %.0f star-hours, %d flashes (%.2f per star-day); %d stars with three or more flashes"
          % (stats["curves"], stats["hours"], stats["flashes"], stats["flashes"] / max(stats["hours"] / 24, 1e-9), len(ps)))
    for c, v in null.items():
        print("  p <= %s: %d stars against %.1f for uniform p" % (c, v["n"], v["expected"]))
    best = {r["gaia"]: r for r in report}
    rows = [dict(target="gaia_dr3:" + k, stat=best[k]["best_n"] if k in best else None,
                 p=best[k]["p"] if k in best else None,
                 note="%d flashes%s" % (e["n"], ", period %.5f d" % best[k]["period_d"] if k in best and best[k]["period_d"] else ""))
            for k, e in every.items()]
    db = core.connect()
    sid = "tess20_glints"
    core.record_search(sid, "Specular glints, TESS 20 s", archive="MAST TESS SPOC fast-lc (STScI AWS mirror)",
                           release="sectors 27 on", selection="20 s targets matched to Gaia DR3 stars within 30 pc; the "
                           "shares in share_glint_<part>.json", code=os.path.abspath(__file__), entry="tess20",
                           params=dict(flash_sigma=5, max_cadences=3, quiet_sigma=2, period_tol_s=40, draws=200,
                                       strongest=60),
                           n_targets=len(rows), null_summary=dict(stats=stats, p=null),
                           result="no star's flashes recur at a period beyond chance", ledger="Specular glints, TESS 20 s",
                           db=db)
    core.record_targets(sid, rows, db=db)
    db.close()
    print("recorded %d stars under %s" % (len(rows), sid))


def tess20_main():
    p = argparse.ArgumentParser()
    p.add_argument("mode", choices=("targets", "fetch", "search", "merge"))
    p.add_argument("--threads", type=int, default=8)
    p.add_argument("--per-star", type=int, default=0, help="light curves per star, earliest sectors first (0: all)")
    p.add_argument("--list", default=None, help="JSON list of target records to fetch or search (one share)")
    p.add_argument("--out", default="glints.json")
    p.add_argument("--procs", type=int, default=1)
    a = p.parse_args()
    {"targets": tess20_targets, "fetch": lambda: tess20_fetch(a.threads, a.per_star, a.list),
     "search": lambda: tess20_search(a.list, a.out, a.procs), "merge": tess20_merge}[a.mode]()


def tess20_cli():
    quiet_warnings()
    tess20_main()


# --------------------------------------------------------------------------------------------------------------

toi_DOC = r"""Glints at TOI periods in SPOC two-minute light curves: the busiest 2.5-cadence phase window against 400 draws.

    python -I seti.py flashes toi targets
    python -I seti.py flashes toi split N                       (shares toi_share<i>.json, balanced by sector count)
    python -I seti.py flashes toi stream --list SHARE.json --out toi_glints_<part>.jsonl [--procs N]
    python -I seti.py flashes toi report toi_glints_*.jsonl
    python -I seti.py flashes toi record toi_glints_*.jsonl"""

toi_ROOT = os.environ.get("TOI_ROOT", core.data("toi_glints"))
toi_HERE = core.RESULTS
toi_TOI_CSV = "https://exofop.ipac.caltech.edu/tess/download_toi.php?sort=toi&output=csv"
toi_CAD = 120.0 / 86400.0
toi_BTJD = 2457000.0


def toi_targets():
    os.makedirs(toi_ROOT, exist_ok=True)
    raw = core.get(toi_TOI_CSV, timeout=300, retries=3).decode("utf-8", "replace")
    by = {}
    for r in csv.DictReader(io.StringIO(raw)):
        if (r.get("TFOPWG Disposition") or "").strip() in ("FP", "FA"):
            continue
        P, ep, dur = num(r.get("Period (days)")), num(r.get("Epoch (BJD)")), num(r.get("Duration (hours)"))
        if not P or P <= 0 or not ep:
            continue
        tic = int(r["TIC ID"])
        x = by.setdefault(tic, dict(tic=tic, tmag=num(r.get("TESS Mag")), sectors=set(), tois=[]))
        x["sectors"] |= {int(s) for s in re.findall(r"\d+", r.get("Sectors") or "")}
        x["tois"].append(dict(toi=r["TOI"], period=P, t0=ep - toi_BTJD, dur_h=dur, disp=r.get("TFOPWG Disposition")))
    out = [dict(x, sectors=sorted(x["sectors"])) for x in by.values() if x["sectors"]]
    json.dump(out, open(os.path.join(toi_ROOT, "targets.json"), "w"))
    print("%d TOI hosts (false positives and false alarms left out), %d TOIs, %d star-sectors"
          % (len(out), sum(len(x["tois"]) for x in out), sum(len(x["sectors"]) for x in out)))


def toi_split(n):
    ts = sorted(json.load(open(os.path.join(toi_ROOT, "targets.json"))), key=lambda t: -len(t["sectors"]))
    for i in range(n):
        json.dump(ts[i::n], open(os.path.join(toi_ROOT, "toi_share%d.json" % i), "w"))
    print("%d shares of %s stars" % (n, ", ".join(str(len(ts[i::n])) for i in range(n))))


def toi_one_star(t):
    quiet_warnings()
    fl, goods, used = [], [], []
    for s in t["sectors"]:
        b = core.spoc_bytes(t["tic"], s)
        if not b:
            continue
        try:
            tt, f, _ = core.read(b, columns=("PDCSAP_FLUX",), normalise=False)
        except Exception:
            continue
        if len(tt) < 1000:
            continue
        ev, _ = science.flash_events(tt, f, toi_CAD, quiet=4)
        fl += ev
        goods.append(tt)
        used.append(s)
    if not goods:
        return dict(tic=t["tic"], sectors=[], days=0.0, n_flash=0, flashes=[], tois=[])
    good_t = np.concatenate(goods)
    rng = np.random.default_rng(t["tic"] % 100000)
    times = [x[0] for x in fl]
    res = []
    for k in t["tois"]:
        if len(times) >= 3:
            n0, fap = science.phase_cluster(times, good_t, k["period"], k["t0"], 2.5 * toi_CAD, rng)
        else:
            n0, fap = len(times), 1.0
        res.append(dict(k, n_cluster=n0, fap=fap))
    return dict(tic=t["tic"], sectors=used, days=float(len(good_t) * toi_CAD), n_flash=len(fl), flashes=fl[:300], tois=res)


def toi_stream(share, out, procs):
    from concurrent.futures import ProcessPoolExecutor
    ts = json.load(open(share or os.path.join(toi_ROOT, "targets.json")))
    path = os.path.join(toi_HERE, out)
    done = set()
    if os.path.exists(path):
        done = {json.loads(ln)["tic"] for ln in open(path) if ln.strip()}
    todo = [t for t in ts if t["tic"] not in done]
    print("%d stars to search (%d done)" % (len(todo), len(done)), flush=True)
    with ProcessPoolExecutor(procs) as ex, open(path, "a") as fh:
        for r in ex.map(toi_one_star, todo, chunksize=2):
            fh.write(json.dumps(r, default=float) + "\n")
            fh.flush()


def toi_report(paths):
    seen, res = set(), []
    for p in paths:
        for ln in open(p):
            if ln.strip():
                r = json.loads(ln)
                if r["tic"] not in seen:
                    seen.add(r["tic"])
                    res.append(r)
    tested = [k for r in res for k in r["tois"] if r["n_flash"] >= 3]
    print("%d stars, %.0f days of two-minute cadence, %d flashes; %d TOIs on stars with three or more flashes"
          % (len(res), sum(r["days"] for r in res), sum(r["n_flash"] for r in res), len(tested)))
    for thr, label in ((0.01, "below 0.01"), (1 / 401.0 + 1e-9, "at the floor 1/401")):
        k = sum(1 for x in tested if x["fap"] < thr or (thr < 0.01 and x["fap"] <= thr))
        print("  TOIs with flashes clustered in phase at FAP %s: %d (at most %.1f expected)" % (label, k, thr * len(tested)))
    for r in res:
        for k in r["tois"]:
            if r["n_flash"] >= 3 and k["fap"] < 0.01:
                print("  TIC %d TOI %s P %.4f d: %d of %d flashes within 2.5 cadences of one phase, FAP %.4f"
                      % (r["tic"], k["toi"], k["period"], k["n_cluster"], r["n_flash"], k["fap"]))
    return res


def toi_record(paths):
    """Record one row per star with its lowest TOI false-alarm probability."""
    res = toi_report(paths)
    tested = [k for r in res for k in r["tois"] if r["n_flash"] >= 3]
    below = sum(1 for k in tested if k["fap"] < 0.01)
    floor = sum(1 for k in tested if k["fap"] <= 1 / 401.0 + 1e-9)
    core.record_search(
        "tess_toi_glints", "Specular glints at TESS planet-candidate periods",
        archive="TESS SPOC two-minute light curves (STScI AWS mirror)", release="sectors 1-%d" % max(
            s for r in res for s in r["sectors"]),
        selection="%d hosts of TOIs that are neither false positives nor false alarms (ExoFOP TOI table)" % len(res),
        code="seti.py flashes toi", params=dict(flash_sigma=5, max_cadences=3, clean_window=4, phase_window_cadences=2.5,
                                          null_draws=400),
        n_targets=len(res),
        null_summary="%d TOIs on stars with three or more flashes: %d below FAP 0.01 against at most %.1f, %d at the "
                     "1/401 floor against at most %.1f" % (len(tested), below, 0.01 * len(tested), floor,
                                                           len(tested) / 401.0),
        sensitivity="flashes of 5 robust deviations lasting 1-3 cadences (2-6 min); %.0f star-days, %d flashes"
                    % (sum(r["days"] for r in res), sum(r["n_flash"] for r in res)),
        result="no TOI period gathers flashes at one phase beyond chance",
        ledger="Specular glints, TESS TOI hosts")
    core.clear_targets("tess_toi_glints")
    core.record_targets("tess_toi_glints", [
        dict(target="tic:%s" % r["tic"], stat=float(r["n_flash"]), unit="flashes",
             p=min((k["fap"] for k in r["tois"]), default=None) if r["n_flash"] >= 3 else None,
             note="%.0f days in %d sectors; lowest TOI false-alarm probability over its folded periods"
                  % (r["days"], len(r["sectors"]))) for r in res])
    print("recorded")


def toi_main():
    p = argparse.ArgumentParser()
    p.add_argument("mode", choices=("targets", "split", "stream", "report", "record"))
    p.add_argument("paths", nargs="*")
    p.add_argument("--list")
    p.add_argument("--out", default="toi_glints.jsonl")
    p.add_argument("--procs", type=int, default=2)
    a = p.parse_args()
    if a.mode == "targets":
        toi_targets()
    elif a.mode == "split":
        toi_split(int(a.paths[0]) if a.paths else 2)
    elif a.mode == "stream":
        toi_stream(a.list, a.out, a.procs)
    elif a.mode == "record":
        toi_record(a.paths)
    else:
        toi_report(a.paths)


def toi_cli():
    quiet_warnings()
    toi_main()


# --------------------------------------------------------------------------------------------------------------

kepler_DOC = r"""Glints at KOI periods in Kepler short cadence: the busiest 2.5-cadence phase window against flash times drawn from good cadences.

    python -I seti.py flashes kepler targets
    python -I seti.py flashes kepler fetch [--threads 4] [--list SHARE.json]
    python -I seti.py flashes kepler search [--list SHARE.json] [--out kepler_glints.json] [--procs 2]
    python -I seti.py flashes kepler split          (two shares of the targets, share0.json and share1.json)
    python -I seti.py flashes kepler stream --list SHARE.json --out kepler_glints_<part>.jsonl [--procs 2]
    python -I seti.py flashes kepler report kepler_glints_*.jsonl"""

kepler_ROOT = os.environ.get("KEPLER_SC", core.data("kepler_sc"))
kepler_HERE = core.RESULTS
kepler_S3 = "https://stpubdata.s3.amazonaws.com/"
kepler_CAD = 58.85 / 86400.0


def kepler_targets():
    os.makedirs(kepler_ROOT, exist_ok=True)
    kois = core.exoplanets("SELECT kepid, kepoi_name, koi_disposition, koi_period, koi_time0bk, koi_duration, "
                               "koi_prad FROM cumulative WHERE koi_disposition IN ('CONFIRMED', 'CANDIDATE')")
    by = {}
    for k in kois:
        by.setdefault(str(k["kepid"]), []).append(dict(koi=k["kepoi_name"], period=float(k["koi_period"]),
                                                       t0=float(k["koi_time0bk"]), dur_h=float(k["koi_duration"] or 0),
                                                       prad=k["koi_prad"]))
    from concurrent.futures import ThreadPoolExecutor

    def listing(kic):
        k9 = "%09d" % int(kic)
        b = core.get(kepler_S3 + "?list-type=2&prefix=kepler/public/lightcurves/%s/%s/" % (k9[:4], k9), timeout=60,
                    retries=3, soft=True, pause=1)
        return kic, re.findall(r"<Key>([^<]*_slc\.fits)</Key>", b.decode()) if b else []
    out = []
    with ThreadPoolExecutor(8) as ex:
        for kic, keys in ex.map(listing, sorted(by)):
            if keys:
                out.append(dict(kic=kic, kois=by[kic], files=keys))
    json.dump(out, open(os.path.join(kepler_ROOT, "targets.json"), "w"))
    print("%d KOIs on %d stars; %d stars with short cadence, %d short-cadence files"
          % (len(kois), len(by), len(out), sum(len(t["files"]) for t in out)))


def kepler_fetch(threads, share=None):
    from concurrent.futures import ThreadPoolExecutor
    ts = json.load(open(share or os.path.join(kepler_ROOT, "targets.json")))
    keys = [k for t in ts for k in t["files"]]
    os.makedirs(os.path.join(kepler_ROOT, "slc"), exist_ok=True)

    def one(k):
        p = os.path.join(kepler_ROOT, "slc", os.path.basename(k))
        if os.path.exists(p):
            return 1
        b = core.get(kepler_S3 + k, timeout=300, retries=3, soft=True, pause=2)
        if not b or not b.startswith(b"SIMPLE"):
            return 0
        open(p + ".tmp", "wb").write(b)
        os.replace(p + ".tmp", p)
        return 1
    n = 0
    with ThreadPoolExecutor(threads) as ex:
        for i, r in enumerate(ex.map(one, keys)):
            n += r
            if i % 500 == 0:
                print("%d of %d files" % (n, len(keys)), flush=True)
    print("%d of %d short-cadence files on disk" % (n, len(keys)))


def kepler_flashes(t, f, q):
    good = (q == 0) & np.isfinite(f) & np.isfinite(t)
    t, f = t[good], f[good]
    if len(f) < 1000:
        return [], t
    from scipy.ndimage import median_filter
    r = f / median_filter(f, size=31, mode="nearest") - 1.0
    sig = 1.4826 * np.median(np.abs(r - np.median(r)))
    out, seen = [], set()
    for i in np.nonzero(r > 5 * sig)[0]:
        if i in seen:
            continue
        j = i
        while j + 1 < len(r) and r[j + 1] > 5 * sig and t[j + 1] - t[j] < 1.5 * kepler_CAD:
            j += 1
        seen.update(range(i, j + 1))
        if j - i >= 3 or i < 5 or j > len(r) - 6:
            continue
        before, after = r[i - 5:i - 1], r[j + 2:j + 6]
        if np.all(np.abs(before) < 2 * sig) and np.all(np.abs(after) < 2 * sig) and \
                t[i] - t[i - 5] < 6 * kepler_CAD and t[j + 5] - t[j] < 6 * kepler_CAD:
            k = i + int(np.argmax(r[i:j + 1]))
            out.append((float(t[k]), float(r[k] / sig), int(j - i + 1)))
    return out, t


def kepler_phase_cluster(times, good_t, P, t0, rng, ntrial=400):
    """Largest flash count in a 2.5-cadence phase window and its false-alarm probability."""
    w = 2.5 * kepler_CAD / P

    def best(ts):
        ph = np.sort(((np.asarray(ts) - t0) / P) % 1.0)
        if len(ph) < 2:
            return len(ph)
        ext = np.concatenate([ph, ph + 1.0])
        j = np.searchsorted(ext, ext[:len(ph)] + w, side="right")
        return int(np.max(j - np.arange(len(ph))))
    n0 = best(times)
    null = [best(rng.choice(good_t, size=len(times), replace=False)) for _ in range(ntrial)]
    return n0, (1 + sum(x >= n0 for x in null)) / (ntrial + 1.0)


def kepler_one_star(t, slc_dir=None):
    quiet_warnings()
    from astropy.io import fits
    fl, goods = [], []
    for k in t["files"]:
        p = os.path.join(slc_dir or os.path.join(kepler_ROOT, "slc"), os.path.basename(k))
        if not os.path.exists(p):
            continue
        try:
            with fits.open(p) as h:
                d = h[1].data
                f, g = kepler_flashes(np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float),
                               np.asarray(d["SAP_QUALITY"] if "SAP_QUALITY" in d.columns.names else d["QUALITY"], int))
        except Exception:
            continue
        fl += f
        goods.append(g)
    if not goods:
        return None
    good_t = np.concatenate(goods)
    rng = np.random.default_rng(int(t["kic"]) % 100000)
    res = []
    times = [x[0] for x in fl]
    for k in t["kois"]:
        if len(times) >= 3 and k["period"] > 0:
            n0, fap = kepler_phase_cluster(times, good_t, k["period"], k["t0"], rng)
        else:
            n0, fap = len(times), 1.0
        res.append(dict(k, n_cluster=n0, fap=fap))
    return dict(kic=t["kic"], days=float(len(good_t) * kepler_CAD), n_flash=len(fl), flashes=fl[:200], kois=res)


def kepler_summary(res):
    """Counts against the null: KOIs on stars with three or more flashes; p floor 1/401."""
    allk = [k for r in res for k in r["kois"]]
    tested = [k for r in res for k in r["kois"] if r["n_flash"] >= 3 and k["period"] > 0]
    print("%d stars, %.0f days of short cadence, %d flashes; %d KOIs folded, %d on stars with three or more flashes" % (
        len(res), sum(r["days"] for r in res), sum(r["n_flash"] for r in res), len(allk), len(tested)))
    for thr in (0.01, 1 / 401.0 + 1e-9):
        print("  KOIs with flashes clustered in phase at FAP %s %.4f: %d (at most %.1f expected)"
              % ("below" if thr == 0.01 else "at the floor", thr, sum(1 for k in tested if k["fap"] < thr or
                                                                    (thr < 0.01 and k["fap"] <= thr)), thr * len(tested)))


def kepler_search(share=None, out="kepler_glints.json", procs=2):
    from concurrent.futures import ProcessPoolExecutor
    ts = json.load(open(share or os.path.join(kepler_ROOT, "targets.json")))
    with ProcessPoolExecutor(procs) as ex:
        res = [r for r in ex.map(kepler_one_star, ts, chunksize=2) if r]
    kepler_summary(res)
    for r in res:
        for k in r["kois"]:
            if k["fap"] < 0.01:
                print("  KIC %s %s P %.4f d: %d of %d flashes within 2.5 cadences of one phase, FAP %.4f"
                      % (r["kic"], k["koi"], k["period"], k["n_cluster"], r["n_flash"], k["fap"]))
    json.dump(res, open(os.path.join(kepler_HERE, out), "w"))


def kepler_stream_star(t):
    """Fetch, search and delete one star's short-cadence files; returns its JSONL line."""
    import shutil
    tmp = os.path.join(kepler_ROOT, "tmp_%s" % t["kic"])
    os.makedirs(tmp, exist_ok=True)
    local = []
    for k in t["files"]:
        p = os.path.join(tmp, os.path.basename(k))
        if not os.path.exists(p):
            b = core.get(kepler_S3 + k, timeout=300, retries=3, soft=True, pause=2)
            if not b or not b.startswith(b"SIMPLE"):
                continue
            open(p, "wb").write(b)
        local.append(p)
    r = kepler_one_star(dict(t, files=local), slc_dir=tmp)
    shutil.rmtree(tmp, ignore_errors=True)
    return r


def kepler_stream(share, out, procs):
    """Stream every star of the list into out (JSONL), skipping stars already there."""
    from concurrent.futures import ProcessPoolExecutor
    ts = json.load(open(share or os.path.join(kepler_ROOT, "targets.json")))
    path = os.path.join(kepler_HERE, out)
    done = set()
    if os.path.exists(path):
        done = {json.loads(ln)["kic"] for ln in open(path) if ln.strip()}
    todo = [t for t in ts if t["kic"] not in done]
    print("%d stars to search (%d done)" % (len(todo), len(done)), flush=True)
    with ProcessPoolExecutor(procs) as ex, open(path, "a") as fh:
        for r in ex.map(kepler_stream_star, todo):
            if r:
                fh.write(json.dumps(r, default=float) + "\n")
                fh.flush()


def kepler_report(paths):
    # a star counts once when two runs overlap
    seen, res = set(), []
    for p in paths:
        for ln in open(p):
            if ln.strip():
                r = json.loads(ln)
                if r["kic"] not in seen:
                    seen.add(r["kic"])
                    res.append(r)
    kepler_summary(res)
    for r in res:
        for k in r["kois"]:
            if k["fap"] < 0.01:
                print("  KIC %s %s P %.4f d: %d of %d flashes within 2.5 cadences of one phase, FAP %.4f"
                      % (r["kic"], k["koi"], k["period"], k["n_cluster"], r["n_flash"], k["fap"]))
    return res


def kepler_record(paths):
    """Record one row per star with its lowest KOI false-alarm probability."""
    res = kepler_report(paths)
    tested = [k for r in res for k in r["kois"] if r["n_flash"] >= 3 and k["period"] > 0]
    below = sum(1 for k in tested if k["fap"] < 0.01)
    floor = sum(1 for k in tested if k["fap"] <= 1 / 401.0 + 1e-9)
    core.record_search(
        "kepler_sc_glints", "Specular glints at KOI periods, Kepler short cadence",
        archive="MAST Kepler short-cadence light curves (STScI AWS mirror)", release="DR25 light curves",
        selection="%d hosts of confirmed and candidate KOIs with short cadence" % len(res), code="seti.py flashes kepler",
        params=dict(flash_sigma=5, max_cadences=3, clean_window=4, phase_window_cadences=2.5, null_draws=400),
        n_targets=len(res),
        null_summary="%d KOIs on stars with three or more flashes: %d below FAP 0.01 against at most %.1f, %d at the "
                     "1/401 floor against at most %.1f" % (len(tested), below, 0.01 * len(tested), floor,
                                                           len(tested) / 401.0),
        sensitivity="flashes of 5 robust deviations lasting 1-3 cadences (59-177 s); %.0f star-days, %d flashes"
                    % (sum(r["days"] for r in res), sum(r["n_flash"] for r in res)),
        result="no KOI period gathers flashes at one phase beyond chance",
        ledger="Specular glints, Kepler short cadence")
    core.clear_targets("kepler_sc_glints")
    core.record_targets("kepler_sc_glints", [
        dict(target="kic:%s" % r["kic"], stat=float(r["n_flash"]), unit="flashes",
             p=min((k["fap"] for k in r["kois"]), default=None),
             note="%.0f days; lowest KOI false-alarm probability over its folded periods" % r["days"]) for r in res])


def kepler_main():
    p = argparse.ArgumentParser()
    p.add_argument("mode", choices=("targets", "fetch", "search", "stream", "split", "report", "record"))
    p.add_argument("paths", nargs="*")
    p.add_argument("--threads", type=int, default=4)
    p.add_argument("--list")
    p.add_argument("--out", default="kepler_glints.json")
    p.add_argument("--procs", type=int, default=2)
    a = p.parse_args()
    if a.mode == "targets":
        kepler_targets()
    elif a.mode == "fetch":
        kepler_fetch(a.threads, a.list)
    elif a.mode == "stream":
        kepler_stream(a.list, a.out, a.procs)
    elif a.mode == "split":
        # alternate stars into two shares by short-cadence volume
        ts = sorted(json.load(open(os.path.join(kepler_ROOT, "targets.json"))), key=lambda t: -len(t["files"]))
        for i in range(2):
            json.dump(ts[i::2], open(os.path.join(kepler_ROOT, "share%d.json" % i), "w"))
        print("two shares of %d and %d stars" % (len(ts[0::2]), len(ts[1::2])))
    elif a.mode == "report":
        kepler_report(a.paths)
    elif a.mode == "record":
        kepler_record(a.paths)
    else:
        kepler_search(a.list, a.out, a.procs)


def kepler_cli():
    quiet_warnings()
    kepler_main()


# --------------------------------------------------------------------------------------------------------------

asteroids_DOC = r"""Asteroid glints in ALCDEF: spikes above the rotational model recurring in phase, against draws and fades as null; injection recovery.

    python -I seti.py flashes asteroids parse
    python -I seti.py flashes asteroids inject [--groups 300]
    python -I seti.py flashes asteroids search [--procs 2] [--out alcdef_glints.json]"""

asteroids_ROOT = os.environ.get("ALCDEF_ROOT", core.data("alcdef"))
asteroids_LCDB = os.environ.get("LCDB", core.data("lcdb", "lc_summary_pub.txt"))
asteroids_HERE = core.RESULTS
# fractional period uncertainty per LCDB quality code
asteroids_FRAC = {"3": 1e-4, "3-": 3e-4, "2+": 1e-3, "2": 2e-3}


def asteroids_data_rows(path):
    """Row index of the first data line after '# Data' in a PDS4 CSV."""
    with open(path, encoding="latin-1") as f:
        for i, line in enumerate(f):
            if line.startswith("# Data"):
                return i + 1
    raise ValueError(path)


def asteroids_parse():
    import pandas as pd
    base = os.path.join(asteroids_ROOT, "gbo.ast.alcdef-database_V1_0", "data")
    pts = []
    for p in sorted(glob.glob(os.path.join(base, "*", "alcdef_lcdata-*.csv"))):
        d = pd.read_csv(p, skiprows=asteroids_data_rows(p), header=None, names=["num", "name", "mdid", "jd", "mag", "err"],
                        dtype={"num": str, "name": str}, encoding="latin-1", low_memory=False)
        pts.append(d[["mdid", "jd", "mag", "err"]])
    d = pd.concat(pts, ignore_index=True)
    d = d.sort_values(["mdid", "jd"], kind="stable")
    np.savez(os.path.join(asteroids_ROOT, "alcdef.npz"), mdid=d["mdid"].to_numpy(np.int64), jd=d["jd"].to_numpy(np.float64),
             mag=d["mag"].to_numpy(np.float32), err=d["err"].to_numpy(np.float32))
    cols = ["ID", "Submitter", "ObjNumber", "ObjName", "Desig", "Facility", "MPCCode", "Instrument", "Detector",
            "ObsLong", "ObsLat", "SessionDateTime", "ObjRA", "ObjDec", "Epoch", "Phase", "PABL", "PABB", "Filter",
            "MagBand", "CIBand", "CIObject", "CICorr", "DiffMags", "DiffZeroMag", "MagStd", "ExpJD", "Exposure"]
    meta = {}
    for p in sorted(glob.glob(os.path.join(base, "*", "alcdef_metadata-*.csv"))):
        m = pd.read_csv(p, skiprows=asteroids_data_rows(p), header=None, dtype=str, encoding="latin-1", low_memory=False)
        m = m.iloc[:, :len(cols)]
        m.columns = cols
        for r in m.itertuples(index=False):
            meta[int(r.ID)] = dict(num=r.ObjNumber, name=r.ObjName, desig=r.Desig, sub=r.Submitter, fac=r.Facility,
                                   tel=r.Instrument, when=r.SessionDateTime, phase=r.Phase, filt=r.Filter,
                                   band=r.MagBand, exp=r.Exposure, mpc=r.MPCCode)
    json.dump(meta, open(os.path.join(asteroids_ROOT, "sessions.json"), "w"))
    print("%d points in %d sessions (%d metadata records)" % (len(d), d["mdid"].nunique(), len(meta)))


def asteroids_lcdb():
    """LCDB number or name -> (period h, U code, diameter km)."""
    out = {}
    with open(asteroids_LCDB, encoding="latin-1") as f:
        lines = f.read().splitlines()
    for ln in lines[5:]:
        if len(ln) < 190:
            continue
        per, u = ln[145:158].strip(), ln[187:190].strip()
        if not per or u not in asteroids_FRAC:
            continue
        try:
            per = float(per)
        except ValueError:
            continue
        try:
            dia = float(ln[88:96])
        except ValueError:
            dia = float("nan")
        num, name = ln[0:10].strip(), ln[10:41].strip()
        rec = (per, u, dia)
        if num and num != "0":
            out["n" + num.lstrip("0")] = rec
        if name:
            out["s" + name.lower()] = rec
    return out


def asteroids_running_median(t, y, w):
    lo = np.searchsorted(t, t - w / 2)
    hi = np.searchsorted(t, t + w / 2, side="right")
    return np.array([np.median(y[a:b]) for a, b in zip(lo, hi)])


def asteroids_fourier_model(t, y, P, order):
    """Robust Fourier fit in rotational phase plus a linear term, or None."""
    ph = 2 * np.pi * (t - t[0]) / P
    cols = [np.ones_like(t), (t - t.mean())]
    for k in range(1, order + 1):
        cols += [np.cos(k * ph), np.sin(k * ph)]
    A = np.array(cols).T
    keep = np.ones(len(t), bool)
    for _ in range(5):
        if keep.sum() < A.shape[1] + 5:
            return None
        c, *_ = np.linalg.lstsq(A[keep], y[keep], rcond=None)
        r = y - A @ c
        s = 1.4826 * np.median(np.abs(r[keep] - np.median(r[keep])))
        new = np.abs(r) < 3 * max(s, 1e-4)
        if (new == keep).all():
            break
        keep = new
    return A @ c


def asteroids_events(t, flag, gap):
    """Mean times of merged runs of flagged points."""
    idx = np.nonzero(flag)[0]
    out = []
    run = []
    for i in idx:
        if run and t[i] - t[run[-1]] > gap:
            out.append(float(np.mean(t[run])))
            run = []
        run.append(i)
    if run:
        out.append(float(np.mean(t[run])))
    return out


def asteroids_session_events(t, y, e, P_d):
    """Spike and fade times of one session, its spacing and robust scatter."""
    dt = float(np.median(np.diff(t)))
    span = t[-1] - t[0]
    model = None
    if span >= 0.8 * P_d:
        order = int(min(6, max(2, len(t) // 8)))
        model = asteroids_fourier_model(t, y, P_d, order)
    if model is None:
        w = float(np.clip(P_d / 10, 5 * dt, 1 / 24))
        model = asteroids_running_median(t, y, w)
    r = y - model
    s = 1.4826 * np.median(np.abs(r - np.median(r)))
    thr = np.maximum(np.maximum(5 * s, 3 * np.where(e > 0, e, 0)), 0.05)
    spikes = asteroids_events(t, r < -thr, 2.5 * dt)
    fades = asteroids_events(t, r > thr, 2.5 * dt)
    return spikes, fades, dt, float(s)


def asteroids_max_cluster(times, P, tol):
    if len(times) < 2:
        return len(times)
    ph = np.mod(np.asarray(times) / P, 1.0)
    d = np.abs(ph[:, None] - ph[None, :])
    d = np.minimum(d, 1 - d)
    return int((d < tol / P).sum(axis=1).max())


def asteroids_group_test(args):
    """One body's session group: largest phase cluster of spikes and fades against random draws."""
    key, P_d, frac, sess, rng_seed = args
    rng = np.random.default_rng(rng_seed)
    allt = np.concatenate([s["t"] for s in sess])
    dt = float(np.median([s["dt"] for s in sess]))
    tol = max(0.75 * dt, 3 / 1440)
    out = dict(key=key, period_h=P_d * 24, n_sessions=len(sess), n_points=int(len(allt)), tol_min=tol * 1440,
               mdids=[s["mdid"] for s in sess])
    for kind in ("spikes", "fades"):
        ev = [x for s in sess for x in s[kind]]
        out["n_" + kind] = len(ev)
        best = None
        for mult in (1.0, 0.5):
            Pt = P_d * mult
            m = asteroids_max_cluster(ev, Pt, tol)
            if m < 3:
                continue
            draws = np.array([asteroids_max_cluster(rng.choice(allt, len(ev), replace=False), Pt, tol) for _ in range(2000)])
            p = float((np.sum(draws >= m) + 1) / (len(draws) + 1))
            if best is None or p < best[2]:
                best = (mult, m, p)
        if best:
            out[kind] = dict(mult=best[0], cluster=best[1], p=best[2])
    return out


def asteroids_load_sessions(keep_flux=False):
    """Sessions passing the cuts, grouped by body."""
    z = np.load(os.path.join(asteroids_ROOT, "alcdef.npz"))
    meta = json.load(open(os.path.join(asteroids_ROOT, "sessions.json")))
    per = asteroids_lcdb()
    mdid, jd, mag, err = z["mdid"], z["jd"], z["mag"], z["err"]
    cut = np.nonzero(np.diff(mdid))[0] + 1
    starts = np.concatenate([[0], cut])
    ends = np.concatenate([cut, [len(mdid)]])
    bodies = {}
    nsess = 0
    seen_t = {}
    ndup = 0
    for a, b in zip(starts, ends):
        if b - a < 20:
            continue
        m = meta.get(str(int(mdid[a])))
        if not m:
            continue
        num = str(m["num"]).strip().lstrip("0")
        rec = per.get("n" + num) if num and num not in ("-9", "-", "nan") else None
        if rec is None:
            rec = per.get("s" + str(m["name"]).strip().lower())
        if rec is None:
            continue
        t = jd[a:b]
        y = mag[a:b].astype(float)
        e = err[a:b].astype(float)
        ok = np.isfinite(t) & np.isfinite(y) & (y > -90) & (y < 90)
        t, y, e = t[ok], y[ok], e[ok]
        if len(t) < 20:
            continue
        o = np.argsort(t)
        t, y, e = t[o], y[o], e[o]
        keep = np.concatenate([[True], np.diff(t) > 0])
        t, y, e = t[keep], y[keep], e[keep]
        if len(t) < 20 or np.median(np.diff(t)) > 15 / 1440:
            continue
        # ALCDEF repeats sessions under new IDs; mostly shared observation times mark copies
        key = "n" + num if num and num not in ("-9", "-", "nan") else "s" + str(m["name"]).strip().lower()
        stamps = set(np.round(t * 86400 / 2).astype(np.int64).tolist())
        held = seen_t.setdefault(key, set())
        if len(stamps & held) > 0.5 * len(stamps):
            ndup += 1
            continue
        held |= stamps
        P_d = rec[0] / 24.0
        spikes, fades, dt, s = asteroids_session_events(t, y, e, P_d)
        nsess += 1
        rec_s = dict(mdid=int(mdid[a]), t=t, dt=dt, spikes=spikes, fades=fades, scatter=s)
        if keep_flux:
            rec_s.update(y=y, e=e)
        bodies.setdefault(key, dict(P=P_d, frac=asteroids_FRAC[rec[1]], dia=rec[2], name=str(m["name"]).strip(),
                                    sess=[]))["sess"].append(rec_s)
    print("%d sessions on %d bodies with LCDB periods (%d duplicate sessions dropped)" % (nsess, len(bodies), ndup))
    return bodies


def asteroids_make_groups(bodies):
    """Session groups at most 10 d long within the period's phase tolerance."""
    groups = []
    seed = 0
    for key, bd in bodies.items():
        ss = sorted(bd["sess"], key=lambda s: s["t"][0])
        group = []
        for s in ss:
            if group:
                span = s["t"][-1] - group[0]["t"][0]
                tol = max(0.75 * np.median([g["dt"] for g in group + [s]]), 3 / 1440)
                if span > 10 or span * bd["frac"] > tol:
                    groups.append((key, bd["P"], bd["frac"], group, seed))
                    seed += 1
                    group = []
            group.append(s)
        if group:
            groups.append((key, bd["P"], bd["frac"], group, seed))
            seed += 1
    return groups


def asteroids_target_of(key, bd):
    return "mpc:" + (key[1:] if key.startswith("n") else bd["name"])


def asteroids_search(procs, outname, sensitivity=None):
    bodies = asteroids_load_sessions()
    groups = asteroids_make_groups(bodies)
    nspk = sum(len(s["spikes"]) for bd in bodies.values() for s in bd["sess"])
    nfad = sum(len(s["fades"]) for bd in bodies.values() for s in bd["sess"])
    tasks = [x for x in groups if sum(len(s["spikes"]) for s in x[3]) >= 3 or sum(len(s["fades"]) for s in x[3]) >= 3]
    print("%d spike events and %d fade events; %d session groups hold three or more of either" % (nspk, nfad, len(tasks)))
    if procs > 1:
        from multiprocessing import Pool
        with Pool(procs) as pool:
            res = pool.map(asteroids_group_test, tasks, chunksize=4)
    else:
        res = [asteroids_group_test(x) for x in tasks]
    json.dump(res, open(os.path.join(asteroids_HERE, outname), "w"))
    summ = {}
    for kind in ("spikes", "fades"):
        ps = [r[kind]["p"] for r in res if kind in r]
        summ[kind] = dict(clusters=len(ps), p01=sum(p < 0.01 for p in ps), p001=sum(p < 0.001 for p in ps))
        print("%s: %d groups with a phase cluster of three or more; %d at p < 0.01, %d at p < 0.001"
              % (kind, len(ps), summ[kind]["p01"], summ[kind]["p001"]))
    top = sorted([r for r in res if "spikes" in r], key=lambda r: (r["spikes"]["p"], -r["spikes"]["cluster"]))[:15]
    for r in top:
        print("  %s P %.3f h: %d of %d spikes within %.1f min of one phase (x%.1f period), p %.4f; fades %s; %d sessions"
              % (r["key"], r["period_h"], r["spikes"]["cluster"], r["n_spikes"], r["tol_min"], r["spikes"]["mult"],
                 r["spikes"]["p"], r.get("fades", {}).get("p", "-"), r["n_sessions"]))
    asteroids_record(bodies, res, summ, sensitivity)


def asteroids_record(bodies, res, summ, sensitivity=None):
    """Record the run with one row per body."""
    best = {}
    for r in res:
        if "spikes" in r:
            best[r["key"]] = min(best.get(r["key"], 1.0), r["spikes"]["p"])
    rows = []
    for key, bd in bodies.items():
        n = sum(len(s["spikes"]) for s in bd["sess"])
        rows.append(dict(target=asteroids_target_of(key, bd), stat=n, p=best.get(key),
                         note="%d sessions, %d spike events, period %.4f h" % (len(bd["sess"]), n, bd["P"] * 24)))
    sid = "alcdef_glints_" + time.strftime("%Y%m%d")
    db = core.connect()
    core.record_search(sid, "Specular glints on asteroids", archive="ALCDEF (PDS SBN gbo.ast.alcdef-database)",
                           release="V1.0, 2021 snapshot; periods LCDB October 2023",
                           selection="sessions of 20 or more points at 15 min or less spacing on bodies with an LCDB "
                                     "period of U 2 or better; duplicate sessions dropped",
                           code=os.path.abspath(__file__), entry="asteroids",
                           params=dict(spike_sigma=5, spike_err=3, spike_mag=0.05, tol="max(0.75 spacing, 3 min)",
                                       group_days=10, draws=2000, folds=[1.0, 0.5]),
                           n_targets=len(rows), null_summary=summ, sensitivity=sensitivity,
                           result="no body with spikes recurring at its period at p < 0.01; fades give %d groups"
                                  % summ["fades"]["p01"],
                           ledger="Specular glints on asteroids", db=db)
    core.record_targets(sid, rows, db=db)
    db.close()
    print("recorded %d bodies under %s" % (len(rows), sid))


def asteroids_inject(n_groups, amps):
    """Recovery of a glint brightening one rotational phase every rotation."""
    bodies = asteroids_load_sessions(keep_flux=True)
    groups = asteroids_make_groups(bodies)
    rng = np.random.default_rng(7)
    ok = [g for g in groups if sum(s["t"][-1] - s["t"][0] for s in g[3]) >= 3 * g[1]]
    pick = [ok[i] for i in rng.choice(len(ok), min(n_groups, len(ok)), replace=False)]
    print("%d groups cover three or more rotations; injecting into %d" % (len(ok), len(pick)))
    out = {}
    for A in amps:
        hit = 0
        for key, P_d, frac, sess, seed in pick:
            phi0 = rng.random()
            new = []
            for s in sess:
                t, y, e = s["t"], s["y"].copy(), s["e"]
                k = np.arange(np.floor(t[0] / P_d - phi0) - 1, np.ceil(t[-1] / P_d - phi0) + 2)
                for tg in (k + phi0) * P_d:
                    j = int(np.argmin(np.abs(t - tg)))
                    if abs(t[j] - tg) < s["dt"] / 2:
                        y[j] -= A
                spikes, fades, dt, sc = asteroids_session_events(t, y, e, P_d)
                new.append(dict(mdid=s["mdid"], t=t, dt=dt, spikes=spikes, fades=fades, scatter=sc))
            r = asteroids_group_test((key, P_d, frac, new, seed))
            hit += "spikes" in r and r["spikes"]["p"] < 0.01
        out[A] = hit / len(pick)
        print("  glint of %.2f mag in one exposure per rotation: recovered at p < 0.01 in %.0f per cent"
              % (A, 100 * out[A]))
    json.dump(out, open(os.path.join(asteroids_HERE, "alcdef_inject.json"), "w"))
    return out


def asteroids_main():
    p = argparse.ArgumentParser()
    p.add_argument("mode", choices=("parse", "search", "inject"))
    p.add_argument("--procs", type=int, default=1)
    p.add_argument("--out", default="alcdef_glints.json")
    p.add_argument("--groups", type=int, default=300)
    a = p.parse_args()
    if a.mode == "parse":
        asteroids_parse()
    elif a.mode == "search":
        inj = os.path.join(asteroids_HERE, "alcdef_inject.json")
        asteroids_search(a.procs, a.out, json.load(open(inj)) if os.path.exists(inj) else None)
    else:
        asteroids_inject(a.groups, (0.05, 0.1, 0.2, 0.5))


def asteroids_cli():
    quiet_warnings()
    asteroids_main()


# --------------------------------------------------------------------------------------------------------------

tophat_DOC = r"""Top-hat brightenings in TESS 20 s light curves within 30 pc: sharp-edged boxes beating a flare template by BIC 10; common-mode and dip nulls.

    python -I seti.py flashes tophat [--procs 4]
    python -I seti.py flashes tophat record"""


def tophat_flare(t, t0, fwhm, amp):
    """Davenport et al. (2014) flare template in FWHM units."""
    x = (t - t0) / fwhm
    out = np.zeros_like(x)
    r = (x > -1) & (x <= 0)
    out[r] = 1 + 1.941 * x[r] - 0.175 * x[r] ** 2 - 2.246 * x[r] ** 3 - 1.125 * x[r] ** 4
    d = x > 0
    out[d] = 0.6890 * np.exp(-1.600 * x[d]) + 0.3030 * np.exp(-0.2783 * x[d])
    return amp * out


def tophat_one(p):
    from astropy.io import fits
    from scipy.ndimage import median_filter
    from scipy.optimize import least_squares
    try:
        with fits.open(os.path.join(tess20_DL, p)) as h:
            d = h[1].data
            t, f, q = (np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["QUALITY"], int))
            sector = int(h[0].header.get("SECTOR", 0))
    except Exception:
        return None
    good = (q == 0) & np.isfinite(f) & np.isfinite(t)
    t, f = t[good], f[good]
    if len(f) < 5000:
        return None
    base = median_filter(f, size=1081, mode="nearest")  # six hours of 20 s cadences
    r = f / base - 1.0
    sig = 1.4826 * np.median(np.abs(r - np.median(r)))
    events = []
    cs = np.concatenate([[0.0], np.cumsum(r)])
    for w in (30, 60, 90, 135, 180):  # 10 to 60 min
        box = (cs[w:] - cs[:-w]) / w
        snr = box / (sig / math.sqrt(w))
        for sign in (1, -1):
            v = sign * snr
            idx = np.nonzero((v >= 7) & (v >= np.roll(v, 1)) & (v > np.roll(v, -1)))[0]
            for i in idx:
                tc = t[i + w // 2]
                if any(abs(e["t"] - tc) < 0.03 and e["sign"] == sign for e in events):
                    continue
                events.append(dict(t=float(tc), w=w, snr=float(v[i]), sign=sign, i0=int(i)))
    out = []
    for e in sorted(events, key=lambda e: -e["snr"])[:40]:
        a, b = max(e["i0"] - 3 * e["w"], 0), min(e["i0"] + 4 * e["w"], len(t))
        tt, rr = t[a:b], e["sign"] * r[a:b]
        if len(tt) < 2 * e["w"]:
            continue
        def box_model(pp):
            t1, t2, amp, c0 = pp
            return np.where((tt >= t1) & (tt <= t2), amp, 0.0) + c0
        t1g, t2g = t[e["i0"]], t[min(e["i0"] + e["w"], len(t) - 1)]
        best_box = None
        for d1 in np.linspace(-0.5, 0.5, 11) * e["w"] * 20 / 86400:
            for d2 in np.linspace(-0.5, 0.5, 11) * e["w"] * 20 / 86400:
                pp = [t1g + d1, t2g + d2, 0.0, 0.0]
                inside = (tt >= pp[0]) & (tt <= pp[1])
                if inside.sum() < 5:
                    continue
                pp[3] = float(np.median(rr[~inside])) if (~inside).sum() > 5 else 0.0
                pp[2] = float(np.mean(rr[inside]) - pp[3])
                chi = float(np.sum((rr - box_model(pp)) ** 2) / sig ** 2)
                if best_box is None or chi < best_box[0]:
                    best_box = (chi, pp)
        k = int(np.argmax(rr))
        res = least_squares(lambda pp: (tophat_flare(tt, pp[0], abs(pp[1]) + 1e-4, pp[2]) + pp[3] - rr) / sig,
                            [tt[k], e["w"] * 20 / 86400 / 3, float(rr[k]), 0.0], max_nfev=400)
        chi_f = float(np.sum(res.fun ** 2))
        n = len(tt)
        bic_box = best_box[0] + 4 * math.log(n)
        bic_flare = chi_f + 4 * math.log(n)
        # edge sharpness: cadences from 10 to 90 per cent of amplitude
        pp = best_box[1]
        def edge(t_edge, rising):
            j = int(np.searchsorted(tt, t_edge))
            seg = rr[max(j - 6, 0):j + 6]
            if len(seg) < 6 or pp[2] <= 0:
                return 99
            lo_, hi_ = pp[3] + 0.1 * pp[2], pp[3] + 0.9 * pp[2]
            between = np.sum((seg > lo_) & (seg < hi_))
            return int(between)
        out.append(dict(t=e["t"], width_min=e["w"] / 3.0, snr=e["snr"], sign=e["sign"], amp=float(pp[2]),
                        bic_box=bic_box, bic_flare=bic_flare, edges=(edge(pp[0], True), edge(pp[1], False)),
                        dur_min=float((pp[1] - pp[0]) * 1440)))
    return dict(file=p, sector=sector, sig=float(sig), events=out)


def tophat_main(procs):
    files = sorted(os.listdir(tess20_DL))
    files = [x for x in files if x.endswith("_fast-lc.fits")]
    from concurrent.futures import ProcessPoolExecutor
    with ProcessPoolExecutor(procs) as ex:
        R = [r for r in ex.map(tophat_one, files, chunksize=8) if r]
    # common mode: events within 30 min on two or more other stars of the sector
    by = {}
    for r in R:
        for e in r["events"]:
            by.setdefault(r["sector"], []).append((e["t"], r["file"]))
    for r in R:
        for e in r["events"]:
            others = {fn for tt, fn in by.get(r["sector"], []) if abs(tt - e["t"]) < 30 / 1440 and fn != r["file"]}
            e["common"] = len(others)
    def tophat(e):
        return (e["bic_flare"] - e["bic_box"] > 10 and max(e["edges"]) <= 2 and e["common"] < 2)
    pos = [(r, e) for r in R for e in r["events"] if e["sign"] > 0]
    neg = [(r, e) for r in R for e in r["events"] if e["sign"] < 0]
    tp, tn = [x for x in pos if tophat(x[1])], [x for x in neg if tophat(x[1])]
    print("%d light curves; brightenings %d, dips %d at 7 sigma; top-hat brightenings %d, top-hat dips %d"
          % (len(R), len(pos), len(neg), len(tp), len(tn)))
    for r, e in sorted(tp, key=lambda x: -x[1]["snr"])[:25]:
        print("  %s t %.4f: %.0f min, SNR %.1f, amplitude %.4f, edges %s cadences, BIC box %.0f flare %.0f"
              % (r["file"], e["t"], e["dur_min"], e["snr"], e["amp"], e["edges"], e["bic_box"], e["bic_flare"]))
    json.dump(dict(n=len(R), pos=len(pos), neg=len(neg), tophat_pos=[dict(file=r["file"], **e) for r, e in tp],
                   tophat_neg=[dict(file=r["file"], **e) for r, e in tn]), open(os.path.join(tess20_HERE, "tophat_events.json"), "w"),
              indent=0)


def tophat_record():
    d = json.load(open(os.path.join(tess20_HERE, "tophat_events.json")))
    tg = {}
    for t in json.load(open(os.path.join(tess20_HERE, "glint_targets.json"))):
        tg[(int(t["sector"]), int(t["tic"]))] = t["gaia"]
    files = [x for x in os.listdir(tess20_DL) if x.endswith("_fast-lc.fits")]
    stars = {}
    for fn in files:
        m = re.match(r"s(\d+)_(\d+)_fast-lc\.fits", fn)
        g = tg.get((int(m.group(1)), int(m.group(2)))) if m else None
        if g:
            stars.setdefault(g, []).append(fn)
    best = {}
    for e in d["tophat_pos"]:
        m = re.match(r"s(\d+)_(\d+)_fast-lc\.fits", e["file"])
        g = tg.get((int(m.group(1)), int(m.group(2))))
        if g and (g not in best or e["snr"] > best[g]["snr"]):
            best[g] = e
    core.record_search(
        "tess20_tophat", "Top-hat flares", archive="MAST TESS SPOC 20 s light curves", selection="stars within 30 pc",
        code=os.path.abspath(__file__), entry="tophat", n_targets=len(stars), params=dict(snr=7, bic=10, max_edge_cadences=2),
        null_summary="dips of the same shape: %d" % len(d["tophat_neg"]),
        sensitivity="10-60 minute brightenings at 7 sigma of the binned noise",
        ledger="Top-hat flares",
        result="%d top-hat brightenings against %d top-hat dips among %d brightenings and %d dips at 7 sigma: the rate of "
               "instrumental steps" % (len(d["tophat_pos"]), len(d["tophat_neg"]), d["pos"], d["neg"]))
    core.clear_targets("tess20_tophat")
    core.record_targets("tess20_tophat", [dict(target="gaia_dr3:%s" % g, stat=best[g]["snr"] if g in best else None,
                                                   unit="S/N of the strongest top-hat brightening",
                                                   note=("%s t %.4f, %.0f min" % (best[g]["file"], best[g]["t"], best[g]["dur_min"]))
                                                   if g in best else None) for g in stars])
    print("recorded %d stars" % len(stars))


def tophat_cli():
    quiet_warnings()
    if len(sys.argv) > 1 and sys.argv[1] == "record":
        tophat_record()
        return
    p = argparse.ArgumentParser()
    p.add_argument("--procs", type=int, default=4)
    a = p.parse_args()
    tophat_main(a.procs)


clockwork_DOC = r"""Clockwork flares: de Jager H-test of Kepler (and TESS) flare times over 0.5-300 d against quarter-shuffled and pooled nulls, vetted for rotation, binaries and spacecraft timing.

    python -I seti.py flashes clockwork [search] [--procs 1]
    python -I seti.py flashes clockwork null [--procs 4]
    python -I seti.py flashes clockwork vet
    python -I seti.py flashes clockwork record"""

clockwork_MMAX = 20
clockwork_CAT = os.environ.get("CLOCKWORK_CAT", "kepler")
clockwork_FMAX = 5.0 if clockwork_CAT == "tess" else 2.0
clockwork_LABEL = "TIC" if clockwork_CAT == "tess" else "KIC"
clockwork_OUT = "clockwork_flares_tess.json" if clockwork_CAT == "tess" else "clockwork_flares.json"
clockwork_CADENCE = 2.0 / 1440 if clockwork_CAT == "tess" else 0.0204


def clockwork_h(t, freqs):
    """de Jager's H per frequency: max over m of Z2_m - 4m + 4, and its m."""
    n = len(t)
    best_h = np.full(len(freqs), -np.inf)
    best_m = np.zeros(len(freqs), int)
    for fb in range(0, len(freqs), 2000):
        fr = freqs[fb:fb + 2000]
        ph = 2 * np.pi * np.outer(fr, t)
        z = np.zeros(len(fr))
        bh = np.full(len(fr), -np.inf)
        bm = np.zeros(len(fr), int)
        for m in range(1, clockwork_MMAX + 1):
            z += (np.cos(m * ph).sum(axis=1) ** 2 + np.sin(m * ph).sum(axis=1) ** 2) * 2.0 / n
            h = z - 4 * m + 4
            up = h > bh
            bh[up] = h[up]
            bm[up] = m
        best_h[fb:fb + 2000] = bh
        best_m[fb:fb + 2000] = bm
    return best_h, best_m


def clockwork_data():
    if clockwork_CAT == "tess":
        p = core.data("tess_flares_yang2023.json")
        if not os.path.exists(p):
            from astroquery.vizier import Vizier
            t = Vizier(row_limit=-1, columns=["TIC", "TimeS"], timeout=900).get_catalogs("J/A+A/669/A15/table1")[0]
            json.dump([[int(r["TIC"]), float(r["TimeS"])] for r in t], open(p, "w"))
        by = {}
        for tic, ts in json.load(open(p)):
            by.setdefault(tic, []).append((ts, int((ts - 1325.29) // 13.7)))
        return by, {}
    p = core.data("kepler_flares_yang2019.json")
    if not os.path.exists(p):
        from astroquery.vizier import Vizier
        v = Vizier(row_limit=-1, columns=["**"], timeout=900)
        t2 = v.get_catalogs("J/ApJS/241/29/table2")[0]
        t1 = v.get_catalogs("J/ApJS/241/29/table1")[0]
        fl = [dict(kic=int(r["KIC"]), q=int(re.sub(r"\D", "", str(r["Q"])) or -1), begin=float(r["Begin"]), end=float(r["End"]))
              for r in t2]
        st = [dict(kic=int(r["KIC"]), prot=num(r["Prot"])) for r in t1]
        json.dump(dict(flares=fl, stars=st), open(p, "w"))
    d = json.load(open(p))
    by = {}
    for f in d["flares"]:
        by.setdefault(f["kic"], []).append((f["begin"], f["q"]))
    return by, {s["kic"]: s.get("prot") for s in d["stars"]}


def clockwork_star(args):
    kic, ev, seed = args
    ev = sorted(ev)
    t = np.array([e[0] for e in ev])
    if seed is not None:
        rng = np.random.default_rng(seed)
        q = np.array([e[1] for e in ev])
        for qq in set(q.tolist()):
            sel = q == qq
            t[sel] = rng.uniform(t[sel].min(), t[sel].max(), sel.sum())
    span = t.max() - t.min()
    freqs = np.arange(max(1 / 300.0, 3.0 / span), clockwork_FMAX, 1.0 / (4 * span))
    h, m = clockwork_h(t - t.min(), freqs)
    i = int(np.argmax(h))
    return dict(kic=kic, n=len(t), span=float(span), h=float(h[i]), period=float(1 / freqs[i]), m=int(m[i]), n_freq=len(freqs))


def clockwork_main(procs):
    by, prot = clockwork_data()
    jobs = [(k, v, None) for k, v in by.items() if len(v) >= 10]
    from concurrent.futures import ProcessPoolExecutor
    with ProcessPoolExecutor(procs) as ex:
        res = list(ex.map(clockwork_star, jobs, chunksize=8))
    res.sort(key=lambda r: -r["h"])
    top = res[:40]
    with ProcessPoolExecutor(procs) as ex:
        sh = list(ex.map(clockwork_star, [(r["kic"], by[r["kic"]], s) for r in top for s in range(10)]))
    shmax = {}
    for r in sh:
        shmax.setdefault(r["kic"], []).append(r["h"])
    for r in res:
        r["prot"] = prot.get(r["kic"])
        rot = r["prot"]
        r["rotation"] = bool(rot) and any(abs(r["period"] * k / rot - 1) < 0.05 or abs(r["period"] / (rot * k) - 1) < 0.05
                                          for k in (1, 2, 3))
        if r["kic"] in shmax:
            r["shuffled"] = sorted(shmax[r["kic"]])
            r["p_shuffled"] = float(np.mean(np.array(shmax[r["kic"]]) >= r["h"]))
    print("%d stars with 10 or more flares" % len(res))
    for r in top[:25]:
        print("  " + clockwork_LABEL + " %d: %d flares over %.0f d; H %.1f at %.3f d (%d harmonics); Prot %s%s; shuffled best H %s"
              % (r["kic"], r["n"], r["span"], r["h"], r["period"], r["m"], r["prot"], " (rotation)" if r["rotation"] else "",
                 ", ".join("%.0f" % v for v in r["shuffled"][-3:])))
    json.dump(res, open(core.results(clockwork_OUT), "w"), indent=0)


clockwork_POOL = {}


def clockwork_pool_init():
    by, _ = clockwork_data()
    pool = {}
    for kic, ev in by.items():
        for t, q in ev:
            pool.setdefault(q, []).append((t, kic))
    clockwork_POOL.update({q: (np.array([x[0] for x in v]), np.array([x[1] for x in v])) for q, v in pool.items()})
    clockwork_POOL["by"] = by


def clockwork_pooled(args):
    """Best H with each flare time replaced by another star's from the same quarter."""
    kic, seed = args
    rng = np.random.default_rng(seed)
    ev = clockwork_POOL["by"][kic]
    t = np.empty(len(ev))
    for i, (t0, q) in enumerate(ev):
        times, owners = clockwork_POOL[q]
        others = times[owners != kic]
        t[i] = others[rng.integers(len(others))] if len(others) else t0
    span = t.max() - t.min()
    freqs = np.arange(max(1 / 300.0, 3.0 / span), clockwork_FMAX, 1.0 / (4 * span))
    return kic, float(clockwork_h(t - t.min(), freqs)[0].max())


def clockwork_null(procs, n_top=40, n_real=20):
    res = json.load(open(core.results(clockwork_OUT)))
    top = res[:n_top]
    from concurrent.futures import ProcessPoolExecutor
    with ProcessPoolExecutor(procs, initializer=clockwork_pool_init) as ex:
        sh = list(ex.map(clockwork_pooled, [(r["kic"], s) for r in top for s in range(n_real)]))
    by, _ = clockwork_data()
    pooled = {}
    for k, h in sh:
        pooled.setdefault(k, []).append(h)
    for r in top:
        r["pooled"] = sorted(pooled[r["kic"]])
        r["p_pooled"] = float(np.mean(np.array(pooled[r["kic"]]) >= r["h"]))
        t = np.array(sorted(e[0] for e in by[r["kic"]]))
        ph = (t / r["period"]) % 1.0
        hist = np.histogram(ph, bins=50, range=(0, 1))[0]
        k = int(np.argmax(hist))
        d = np.abs(((ph - (k + 0.5) / 50) + 0.5) % 1.0 - 0.5)
        r["in_peak"] = int(np.sum(d < 0.02))
    print("the %d highest stars against %d draws of other stars' flare times in the same quarters:" % (n_top, n_real))
    for r in top:
        print("  " + clockwork_LABEL + " %-9d %4d flares H %7.1f at %8.3f d (%2d harmonics) %s; pooled best H %s; p %.2f; %d flares within 0.02 "
              "of the peak phase" % (r["kic"], r["n"], r["h"], r["period"], r["m"], "rotation" if r["rotation"] else "        ",
                                     ", ".join("%.0f" % v for v in r["pooled"][-3:]), r["p_pooled"], r["in_peak"]))
    json.dump(res, open(core.results(clockwork_OUT), "w"), indent=0)


clockwork_INSTRUMENT = ((("TESS orbit", 13.7), ("sector", 27.4)) if clockwork_CAT == "tess" else
                        (("Kepler orbit", 372.5), ("quarter", 93.0), ("monthly downlink", 31.0)))


def clockwork_vet():
    """Vet null-beating stars: distinct cycles in the peak, Kepler EB periods, spacecraft timing harmonics."""
    res = json.load(open(core.results(clockwork_OUT)))
    by, _ = clockwork_data()
    eb_p = core.data("tess_eb_prsa2022.json" if clockwork_CAT == "tess" else "kepler_eb_kirk2016.json")
    if not os.path.exists(eb_p):
        from astroquery.vizier import Vizier
        idc, cat = ("TIC", "J/ApJS/258/16/tess-ebs") if clockwork_CAT == "tess" else ("KIC", "J/AJ/151/68/catalog")
        t = Vizier(row_limit=-1, columns=[idc, "Per"], timeout=600).get_catalogs(cat)[0]
        json.dump({str(int(r[idc])): float(r["Per"]) for r in t if r["Per"] is not np.ma.masked}, open(eb_p, "w"))
    eb = json.load(open(eb_p))
    for r in res:
        if "p_pooled" not in r or r["p_pooled"] > 0 or r["rotation"]:
            continue
        t = np.array(sorted(e[0] for e in by[r["kic"]]))
        ph = (t / r["period"]) % 1.0
        k = int(np.argmax(np.histogram(ph, bins=50, range=(0, 1))[0]))
        d = np.abs(((ph - (k + 0.5) / 50) + 0.5) % 1.0 - 0.5)
        cyc = np.floor(t / r["period"])[d < 0.02]
        r["peak_cycles"] = int(len(set(cyc.tolist())))
        r["cycles_spanned"] = int(np.floor(t.max() / r["period"]) - np.floor(t.min() / r["period"]) + 1)
        pe = eb.get(str(r["kic"]))
        r["eb_period"] = pe
        r["eb"] = bool(pe) and any(abs(r["period"] * m / (pe * n) - 1) < 0.01 for m in (1, 2, 3) for n in (1, 2, 3))
        r["instrument"] = next((nm for nm, pi in clockwork_INSTRUMENT for n in (1, 2, 3, 4)
                                if abs(r["period"] / (pi / n) - 1) < 0.03 or abs(r["period"] / (pi * n) - 1) < 0.03), None)
        r["burst"] = r["peak_cycles"] < 4
        # period refined over +-0.2 per cent; a clock keeps the phase scatter below 1e-3
        fr = np.linspace(1 / r["period"] * (1 - 2e-3), 1 / r["period"] * (1 + 2e-3), 801)
        h = clockwork_h(t - t.min(), fr)[0]
        P = 1 / fr[int(np.argmax(h))]
        ph = ((t - t.min()) / P) % 1.0
        hist = np.histogram(ph, bins=100, range=(0, 1))[0]
        c = (int(np.argmax(np.convolve(np.r_[hist, hist[:4]], np.ones(5), "valid")[:100])) + 2.5) / 100
        dd = ((ph - c + 0.5) % 1.0) - 0.5
        r["period_refined"] = float(P)
        r["jitter"] = float(np.std(dd[np.abs(dd) < 0.03]))
        r["jitter_floor"] = clockwork_CADENCE / (P * math.sqrt(12))
        print("  " + clockwork_LABEL + " %-9d %4d flares at %8.3f d: peak fed by %d of %d cycles%s; EB %s%s; spacecraft timing %s; phase scatter "
              "%.4f (cadence floor %.4f)"
              % (r["kic"], r["n"], P, r["peak_cycles"], r["cycles_spanned"], " (burst)" if r["burst"] else "",
                 "%.4f d" % pe if pe else "no", " (match)" if r["eb"] else "", r["instrument"] or "no", r["jitter"],
                 r["jitter_floor"]))
    json.dump(res, open(core.results(clockwork_OUT), "w"), indent=0)


def clockwork_record():
    res = json.load(open(core.results(clockwork_OUT)))
    top = [r for r in res if "p_pooled" in r]
    beat = [r for r in top if r["p_pooled"] == 0.0 and not r["rotation"]]

    def vet(r):
        if r.get("rotation"):
            return "rotation period or a multiple"
        if "burst" not in r:
            return None
        if r["burst"]:
            return "a burst: the phase peak is fed by %d cycles" % r["peak_cycles"]
        if r["eb"]:
            return "eclipsing binary period"
        if r["instrument"]:
            return "spacecraft timing (%s)" % r["instrument"]
        return "phase scatter %.4f of the period, above the 1e-3 of a clock" % r["jitter"] if r["jitter"] >= 1e-3 else \
            "open: phase scatter %.4f of the period" % r["jitter"]
    clock = [r for r in beat if vet(r) and vet(r).startswith("open")]
    tess = clockwork_CAT == "tess"
    rid = "tess_clockwork_flares" if tess else "kepler_clockwork_flares"
    lab = clockwork_LABEL
    locked = [r for r in beat if vet(r) and vet(r).startswith("phase scatter")]
    jit = [r["jitter"] for r in beat if "jitter" in r]
    core.record_search(
        rid, "Clockwork flares",
        archive=("Yang et al. 2023 TESS flare catalogue (J/A+A/669/A15)" if tess else
                 "Yang and Liu 2019 Kepler flare catalogue (J/ApJS/241/29)"),
        selection="%s stars with 10 or more catalogued flares" % ("TESS" if tess else "Kepler"),
        code=os.path.abspath(__file__), entry="clockwork",
        n_targets=len(res), params=dict(harmonics=clockwork_MMAX, periods_d=[1 / clockwork_FMAX, 300], null_draws=20),
        null_summary="each of the %d highest stars' flare times replaced by other stars' flare times from the same %s, "
                     "20 times, which keeps the spacecraft's gaps" % (len(top), "TESS orbits" if tess else "quarters"),
        sensitivity="a train of flare-like events phase-locked within about 2.5 per cent of its period",
        ledger="Clockwork flares, TESS" if tess else "Clockwork flares",
        result="%d of the %d highest stars beat all 20 of their null draws away from rotation: %d burst%s, %d at spacecraft "
               "timing, %d eclipsing binaries, %d phase-locked with scatter %s of the period (%s %s)%s"
               % (len(beat), len(top), sum(1 for r in beat if r.get("burst")),
                  "" if sum(1 for r in beat if r.get("burst")) == 1 else "s",
                  sum(1 for r in beat if not r.get("burst") and r.get("instrument")),
                  sum(1 for r in beat if not r.get("burst") and r.get("eb")), len(locked),
                  "%.4f-%.4f" % (min(jit), max(jit)) if jit else "-", lab,
                  ", ".join("%d at %.3f d" % (r["kic"], r["period_refined"]) for r in locked) or "none",
                  "; none below the 1e-3 of a clock" if not clock else
                  "; below the 1e-3 of a clock: " + ", ".join("%s %d" % (lab, r["kic"]) for r in clock)))
    core.clear_targets(rid)
    core.record_targets(rid, [dict(
        target="%s:%d" % (lab.lower(), r["kic"]), stat=r["h"], p=r.get("p_pooled"),
        unit="best H over %.1f-300 d" % (1 / clockwork_FMAX), vet=vet(r),
        note="%d flares, best %.3f d with %d harmonics" % (r["n"], r.get("period_refined", r["period"]), r["m"])) for r in res])
    print("recorded %d stars" % len(res))


def clockwork_cli():
    quiet_warnings()
    if len(sys.argv) > 1 and sys.argv[1] == "record":
        clockwork_record()
        return
    p = argparse.ArgumentParser()
    p.add_argument("mode", nargs="?", choices=("search", "null", "vet"), default="search")
    p.add_argument("--procs", type=int, default=1)
    a = p.parse_args()
    if a.mode == "vet":
        clockwork_vet()
        return
    (clockwork_null if a.mode == "null" else clockwork_main)(a.procs)


levels_DOC = r"""Discrete flux levels in TESS 20 s light curves within 30 pc: sharp 7-sigma steps between three or more revisited levels; common-mode steps and flares removed.

    python -I seti.py flashes levels [--procs 4]
    python -I seti.py flashes levels record
    python -I seti.py flashes levels stream --targets FILE [--part k/n] [--procs 3]   (SPOC two-minute light curves of the star-sectors in FILE, a determinism targets list, read in memory, steps and levels into LEVELS_ROOT/levels2_<k>of<n>.jsonl)"""

levels_W = 15  # 30 min of 2-min bins
levels_SNR = 7.0
levels_DECAY = 0.5  # an up-step whose light falls by half its jump within the next window is a flare's rise
levels_DECAY_DAYS = 0.125  # steps down within three hours of a flare's rise are its decay


def levels_steps(tb, fb, w, gap, sharp_half=1):
    """Steps in a binned light curve: 7-sigma jumps between window medians, each with its sharpest transition, and the levels between them."""
    from scipy.ndimage import median_filter
    med = median_filter(fb, size=w, mode="nearest")
    delta = np.full(len(fb), np.nan)
    delta[w:-w] = med[w + w // 2:len(fb) - w + w // 2] - med[w - w // 2 - 1:len(fb) - w - w // 2 - 1]
    # no steps across data gaps
    for gi in np.nonzero(np.diff(tb) > gap)[0]:
        delta[max(gi - w, 0):gi + w + 1] = np.nan
    ok = np.isfinite(delta)
    if ok.sum() < 3 * w:
        return None
    sd = 1.4826 * np.median(np.abs(delta[ok] - np.median(delta[ok])))
    sig_bin = 1.4826 * np.median(np.abs(np.diff(fb))) / math.sqrt(2)
    steps = []
    order = np.argsort(-np.abs(np.nan_to_num(delta)))
    for i in order:
        if not ok[i] or abs(delta[i]) < levels_SNR * sd:
            break
        if any(abs(i - s["i"]) < 2 * w for s in steps):
            continue
        # sharpest consistent transition: ramps of 0-8 bins near i
        best = None
        for j in range(i - w // 2, i + w // 2 + 1):
            a = fb[max(j - w, 0):j]
            b = fb[j:j + w]
            if len(a) < 5 or len(b) < 5:
                continue
            for half in (0, 1, 2, 4, 8):
                lo, hi = j - half, j + half
                if lo - w < 0 or hi + w > len(fb):
                    continue
                pre, post = fb[lo - w:lo], fb[hi:hi + w]
                m1, m2 = float(np.median(pre)), float(np.median(post))
                ramp = np.linspace(m1, m2, hi - lo + 2)[1:-1] if hi > lo else np.array([])
                model = np.concatenate([np.full(len(pre), m1), ramp, np.full(len(post), m2)])
                seg = fb[lo - w:hi + w]
                chi = float(np.sum((seg - model) ** 2)) / sig_bin ** 2 - (len(seg) - 2 * w - 2 * half)
                if best is None or chi < best[0]:
                    best = (chi, j, half, m2 - m1)
        if best is None:
            continue
        # a flare rises sharply and decays: the light just after the transition against the window after it, over the jump
        j, k = best[1], max(3, w // 5)
        early = float(np.median(fb[j:j + k])) if j + k <= len(fb) else float("nan")
        late = float(np.median(fb[j + w:j + 2 * w])) if j + 2 * w <= len(fb) else float("nan")
        steps.append(dict(i=int(i), t=float(tb[j]), jump=float(best[3]), snr=float(abs(delta[i]) / sd),
                          half=int(best[2]), sharp=best[2] <= sharp_half,
                          decay=round((early - late) / best[3], 3) if best[3] and np.isfinite(early - late) else None))
    steps.sort(key=lambda s: s["t"])
    # segments between detected steps with their medians
    edges = [0] + [s["i"] for s in steps] + [len(fb)]
    segs = [dict(t0=float(tb[a]), t1=float(tb[b - 1]), level=float(np.median(fb[a:b])), n=int(b - a))
            for a, b in zip(edges[:-1], edges[1:]) if b - a >= 5]
    return dict(sd=float(sd), sig_bin=float(sig_bin), steps=steps, segs=segs)


def levels_one(p):
    from astropy.io import fits
    try:
        with fits.open(os.path.join(tess20_DL, p)) as h:
            d = h[1].data
            t, f, q = (np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["QUALITY"], int))
            sector = int(h[0].header.get("SECTOR", 0))
    except Exception:
        return None
    good = (q == 0) & np.isfinite(f) & np.isfinite(t)
    t, f = t[good], f[good]
    if len(f) < 5000:
        return None
    f = f / np.median(f)
    k = np.floor((t - t[0]) / (2.0 / 1440)).astype(int)
    n = np.bincount(k)
    keep = n >= 4
    tb = (np.bincount(k, t) / np.maximum(n, 1))[keep]
    fb = (np.bincount(k, f) / np.maximum(n, 1))[keep]
    r = levels_steps(tb, fb, levels_W, 10.0 / 1440)
    if r is None:
        return None
    for st in r["steps"]:
        st["ramp_min"] = 2 * st["half"] * 2.0
    return dict(file=p, sector=sector, **r)


levels_ROOT = os.environ.get("LEVELS_ROOT", core.data("levels2"))


def levels_one_spoc(x):
    """Steps and levels of one star-sector's SPOC two-minute light curve read in memory; None when it cannot be read."""
    try:
        b = core.spoc_bytes(x["tic"], x["sector"])
    except Exception:
        return None
    if not b:
        return dict(id=x["id"], status="no light curve")
    try:
        t, f, _ = core.read(b, columns=("PDCSAP_FLUX",))
    except Exception as e:
        return dict(id=x["id"], status="error", error=repr(e)[:160])
    if len(f) < 2000:
        return dict(id=x["id"], status="short")
    r = levels_steps(t, f, levels_W, 10.0 / 1440)
    if r is None:
        return dict(id=x["id"], status="short")
    for st in r["steps"]:
        st["ramp_min"] = 2 * st["half"] * 2.0
    return dict(id=x["id"], status="ok", tic=x["tic"], sector=x["sector"], cam=x.get("cam"), ccd=x.get("ccd"), plx=x.get("plx"), **r)


def levels_stream(targets_path, part=None, procs=3):
    """The star-sectors of a determinism targets list through levels_one_spoc on procs processes, one line each into ROOT/levels2_<k>of<n>.jsonl, past those any levels2 file under ROOT holds; the number left unread."""
    from concurrent.futures import ProcessPoolExecutor
    os.makedirs(levels_ROOT, exist_ok=True)
    targets = json.load(open(targets_path))
    k, n = (int(v) for v in part.split("/")) if part else (0, 1)
    done = set()
    for p in glob.glob(os.path.join(levels_ROOT, "levels2_*.jsonl")):
        for ln in open(p, encoding="utf-8", errors="replace"):
            try:
                done.add(json.loads(ln)["id"])
            except (ValueError, KeyError):
                pass
    todo = [dict(x, id="tic:%d:s%d" % (x["tic"], x["sector"])) for x in targets[k::n]]
    todo = [x for x in todo if x["id"] not in done]
    path = os.path.join(levels_ROOT, "levels2_%dof%d.jsonl" % (k, n))
    print("%d star-sectors to read, %d read under the root" % (len(todo), len(done)), flush=True)
    t0, m, miss = time.time(), 0, 0
    with ProcessPoolExecutor(procs) as ex, open(path, "a", encoding="utf-8") as fh:
        for r in ex.map(levels_one_spoc, todo, chunksize=4):
            if r is None:
                miss += 1
                continue
            fh.write(json.dumps(r, separators=(",", ":")) + "\n")
            m += 1
            if m % 200 == 0:
                fh.flush()
                print("%d of %d, %.1f a minute, %d to read again" % (m, len(todo), 60 * m / (time.time() - t0), miss), flush=True)
    print("%d star-sectors read, %d to read again" % (m, miss), flush=True)
    return miss


def levels_count(segs, real, sd, sig_bin, w, cad):
    """Levels of one light curve with the segments joined across rejected steps: (levels, levels revisited twice or more); a segment begins at a kept step when a real step's sharpest transition lies within half a window of its start."""
    # a segment starts where the window medians differ most, a real step at its sharpest transition, at most w // 2 bins away
    near = (w // 2 + 1) * cad
    times = [st["t"] for st in real]
    merged = []
    for sg in segs:
        if merged and not any(abs(sg["t0"] - t) <= near for t in times):
            m = merged[-1]
            m["level"] = (m["level"] * m["n"] + sg["level"] * sg["n"]) / (m["n"] + sg["n"])
            m["n"] += sg["n"]
            m["t1"] = sg["t1"]
        else:
            merged.append(dict(sg))
    tol = 3 * max(sd, sig_bin / math.sqrt(w))
    groups = []
    for v in sorted(m["level"] for m in merged):
        if groups and v - groups[-1][-1] < tol:
            groups[-1].append(v)
        else:
            groups.append([v])
    centres = [float(np.mean(g)) for g in groups]
    seq = [int(np.argmin([abs(m["level"] - c) for c in centres])) for m in merged]
    visits = {g: sum(1 for a, b in zip([None] + seq[:-1], seq) if b == g and a != g) for g in set(seq)}
    return len(centres), sum(1 for v in visits.values() if v >= 2)


def levels_main(procs):
    from concurrent.futures import ProcessPoolExecutor
    files = sorted(x for x in os.listdir(tess20_DL) if x.endswith("_fast-lc.fits"))
    with ProcessPoolExecutor(procs) as ex:
        res = [r for r in ex.map(levels_one, files, chunksize=4) if r]
    by_sector = {}
    for r in res:
        for s in r["steps"]:
            by_sector.setdefault(r["sector"], []).append((s["t"], r["file"]))
    out = []
    for r in res:
        for s in r["steps"]:
            others = {fn for tt, fn in by_sector[r["sector"]] if abs(tt - s["t"]) < 1.0 / 24 and fn != r["file"]}
            s["common"] = len(others) >= 2
        for s in r["steps"]:
            s["flare"] = s["jump"] > 0 and s["decay"] is not None and s["decay"] > levels_DECAY
        # the steps down within three hours after a flare's rise are its decay
        rises = [s["t"] for s in r["steps"] if s["flare"]]
        for s in r["steps"]:
            if s["jump"] < 0 and any(0 < s["t"] - t0 <= levels_DECAY_DAYS for t0 in rises):
                s["flare"] = True
        real = [s for s in r["steps"] if s["sharp"] and not s["common"] and not s["flare"]]
        nlev, revisited = levels_count(r["segs"], real, r["sd"], r["sig_bin"], levels_W, 2.0 / 1440)
        out.append(dict(file=r["file"], sector=r["sector"], n_steps=len(r["steps"]), n_sharp=sum(s["sharp"] for s in r["steps"]),
                        n_common=sum(s["common"] for s in r["steps"]), n_flare=sum(s["flare"] and s["sharp"] and not s["common"] for s in r["steps"]),
                        n_real=len(real), levels=nlev,
                        revisited=revisited, multi=revisited >= 3, steps=real[:20], sd=r["sd"]))
    n_multi = sum(o["multi"] for o in out)
    print("%d light curves; sharp steps %d, %d of them common-mode, %d of flares, rises that decay by half within the next window and the steps down after them; "
          "light curves with real sharp steps %d; with three or more levels each revisited %d"
          % (len(out), sum(o["n_sharp"] for o in out), sum(o["n_common"] for o in out), sum(o["n_flare"] for o in out),
             sum(1 for o in out if o["n_real"]), n_multi))
    up = sum(1 for o in out for s in o["steps"] if s["jump"] > 0)
    dn = sum(1 for o in out for s in o["steps"] if s["jump"] < 0)
    print("real sharp steps up %d, down %d" % (up, dn))
    for o in sorted(out, key=lambda o: (-o["revisited"], -o["n_real"]))[:25]:
        print("  %s: %d real sharp steps, %d levels, %d revisited; steps %s"
              % (o["file"][:40], o["n_real"], o["levels"], o["revisited"],
                 ", ".join("%.3f %+.4f (%.0f min)" % (s["t"], s["jump"], s["ramp_min"]) for s in o["steps"][:6])))
    json.dump(out, open(os.path.join(tess20_HERE, "levels.json"), "w"))


def levels_record():
    out = json.load(open(os.path.join(tess20_HERE, "levels.json")))
    tg = {}
    for t in json.load(open(os.path.join(tess20_HERE, "glint_targets.json"))):
        tg[(int(t["sector"]), int(t["tic"]))] = t["gaia"]
    best = {}
    for o in out:
        m = re.match(r"s(\d+)_(\d+)_fast-lc\.fits", o["file"])
        g = tg.get((int(m.group(1)), int(m.group(2)))) if m else None
        if g and (g not in best or o["revisited"] > best[g]["revisited"]):
            best[g] = o
    multi = [o for o in out if o["multi"]]
    core.record_search(
        "tess20_levels", "Flux in discrete levels", archive="MAST TESS SPOC 20 s light curves", selection="stars within 30 pc",
        code=os.path.abspath(__file__), entry="levels", n_targets=len(best),
        params=dict(bin_min=2, window_min=30, snr=levels_SNR, max_ramp_min=2, common_hours=1, level_sigma=3),
        null_summary="common-mode steps on three or more stars of a sector: %d, and flares, rises that lose half their jump within the next "
                     "half hour and the steps down within three hours after them: %d, of %d sharp steps"
                     % (sum(o["n_common"] for o in out), sum(o["n_flare"] for o in out), sum(o["n_sharp"] for o in out)),
        sensitivity="levels separated by 7 sigma of the 30-minute median noise, transitions of 2 minutes or less",
        ledger="Flux in discrete levels",
        result="%d light curves: %d with real sharp steps, %d with three or more levels each revisited%s"
               % (len(out), sum(1 for o in out if o["n_real"]), len(multi),
                  (": " + ", ".join(o["file"][:22] for o in multi[:8])) if multi else ""))
    core.clear_targets("tess20_levels")
    core.record_targets("tess20_levels", [dict(
        target="gaia_dr3:%s" % g, stat=float(o["revisited"]), unit="levels revisited at least twice",
        note="%s: %d real sharp steps, %d levels" % (o["file"], o["n_real"], o["levels"])) for g, o in best.items()])
    print("recorded %d stars" % len(best))


def levels_cli():
    quiet_warnings()
    if len(sys.argv) > 1 and sys.argv[1] == "record":
        levels_record()
        return
    p = argparse.ArgumentParser()
    p.add_argument("mode", nargs="?", choices=["stream"])
    p.add_argument("--procs", type=int, default=4)
    p.add_argument("--targets")
    p.add_argument("--part", default=None)
    a = p.parse_args()
    if a.mode == "stream":
        sys.exit(1 if levels_stream(a.targets, a.part, a.procs) else 0)
    levels_main(a.procs)


keplevels_DOC = r"""Discrete flux levels in Kepler long-cadence light curves of the DR25 stars: steps within one cadence between three or more revisited levels in a quarter; steps three other stars of the channel or twenty of the focal plane share removed.

    python -I seti.py flashes keplevels targets                    (the DR25 stellar table into ROOT/targets.json)
    python -I seti.py flashes keplevels stream [--part k/n] [--sub j/m] [--threads 4]   (each star's quarters read in memory, steps into ROOT/steps_<k>of<n>[_<j>of<m>].jsonl; exits 1 while stars remain unread)
    python -I seti.py flashes keplevels analyse                    (every part: channel-common and focal-plane steps removed, levels counted)
    python -I seti.py flashes keplevels vet                        (the stars with three or more revisited levels against the eclipsing-binary catalogue, their gravities, their planets' transits, SIMBAD and Gaia's variability classes)
    python -I seti.py flashes keplevels record"""

keplevels_ROOT = os.environ.get("KEPLEVELS_ROOT", core.data("keplevels"))
keplevels_W = 15  # cadences per window, 7.4 h
keplevels_CAD = 29.4244 / 1440  # long cadence, days
keplevels_COMMON = 3  # other stars of the channel stepping within 1.5 cadences
keplevels_PLANE = 20  # stars anywhere on the focal plane stepping within 1.5 cadences, three times the chance median of 6


def keplevels_targets():
    """DR25 stellar table: every KIC star with Kepler data, into ROOT/targets.json."""
    os.makedirs(keplevels_ROOT, exist_ok=True)
    rows = core.exoplanets("SELECT kepid, kepmag, teff, logg, radius FROM keplerstellar WHERE st_delivname = 'q1_q17_dr25_stellar'")
    json.dump([dict(kepid=int(r["kepid"]), kepmag=num(r["kepmag"]), teff=num(r["teff"]), logg=num(r["logg"]), radius=num(r["radius"]))
               for r in rows if num(r["kepid"])], open(os.path.join(keplevels_ROOT, "targets.json"), "w"))
    print("%d DR25 stars" % len(rows))


def keplevels_star(x):
    """One star's quarters read in memory: steps and levels of each quarter's PDCSAP light curve with its channel; None when the listing or any quarter fails, so the star is read again on the next pass."""
    from astropy.io import fits
    out = []
    keys = core.kepler_keys(x["kepid"], short=False)
    if not keys:
        return None
    for key in keys:
        b = core.get(core.S3 + key, timeout=300, retries=3, soft=True, pause=2)
        if not b or not b.startswith(b"SIMPLE"):
            return None
        try:
            with fits.open(io.BytesIO(b)) as h:
                d = h[1].data
                t, f, q = (np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["SAP_QUALITY"], int))
                quarter, channel = int(h[0].header.get("QUARTER", -1)), int(h[0].header.get("CHANNEL", -1))
        except Exception:
            continue
        good = (q == 0) & np.isfinite(f) & np.isfinite(t)
        t, f = t[good], f[good]
        if len(f) < 500:
            continue
        r = levels_steps(t, f / np.median(f), keplevels_W, 2.5 * keplevels_CAD)
        if r is None:
            continue
        out.append(dict(q=quarter, ch=channel, sd=round(r["sd"], 7), sig=round(r["sig_bin"], 7),
                        steps=[[round(st["t"], 5), round(st["jump"], 6), round(st["snr"], 1), st["half"]] for st in r["steps"]],
                        segs=[[round(sg["t0"], 4), round(sg["t1"], 4), round(sg["level"], 6), sg["n"]] for sg in r["segs"]]))
    return dict(kepid=x["kepid"], kepmag=x["kepmag"], quarters=out)


def keplevels_stream(part=None, threads=4, sub=None):
    """Stream a part of the DR25 stars (and a sub-part j/m of it), appending one line per star to ROOT/steps_<k>of<n>[_<j>of<m>].jsonl, past every star any steps file under ROOT holds with quarters; the number of stars left unread."""
    targets = json.load(open(os.path.join(keplevels_ROOT, "targets.json")))
    k, n = (int(v) for v in part.split("/")) if part else (0, 1)
    j, m_ = (int(v) for v in sub.split("/")) if sub else (0, 1)
    path = os.path.join(keplevels_ROOT, "steps_%dof%d%s.jsonl" % (k, n, "_%dof%d" % (j, m_) if sub else ""))
    done = set()
    for p in glob.glob(os.path.join(keplevels_ROOT, "steps_*of*.jsonl")):
        for ln in open(p, encoding="utf-8", errors="replace"):
            try:
                r = json.loads(ln)
            except ValueError:
                continue
            if r.get("quarters"):
                done.add(r["kepid"])
    todo = [x for x in targets[k::n][j::m_] if x["kepid"] not in done]
    print("%d stars to read, %d read before under the root" % (len(todo), len(done)), flush=True)
    t0, m, miss = time.time(), 0, 0
    with ThreadPoolExecutor(threads) as ex, open(path, "a", encoding="utf-8") as fh:
        for r in ex.map(keplevels_star, todo):
            if r is None:
                miss += 1
                continue
            fh.write(json.dumps(r, separators=(",", ":")) + "\n")
            m += 1
            if m % 200 == 0:
                fh.flush()
                print("%d of %d, %.1f stars a minute, %d to read again" % (m, len(todo), 60 * m / (time.time() - t0), miss), flush=True)
    print("%d stars read, %d to read again" % (m, miss), flush=True)
    return miss


def keplevels_analyse():
    """Every part's steps: steps three other stars of the channel share within 1.5 cadences removed, levels counted per quarter."""
    best_row = {}
    for p in sorted(glob.glob(os.path.join(keplevels_ROOT, "steps_*of*.jsonl"))):
        for ln in open(p, encoding="utf-8", errors="replace"):
            try:
                r = json.loads(ln)
            except ValueError:
                continue
            if r["kepid"] not in best_row or len(r["quarters"]) > len(best_row[r["kepid"]]["quarters"]):
                best_row[r["kepid"]] = r  # one row a star, the fullest where a star was read twice
    rows = list(best_row.values())
    index, plane = {}, {}
    for r in rows:
        for qd in r["quarters"]:
            for st in qd["steps"]:
                index.setdefault((qd["q"], qd["ch"]), []).append((st[0], r["kepid"]))
                if st[3] <= 1:
                    plane.setdefault(qd["q"], []).append(st[0])
    for v in index.values():
        v.sort()
    plane = {q: np.sort(np.array(v)) for q, v in plane.items()}
    out, n_steps, n_sharp, n_common, n_plane = [], 0, 0, 0, 0
    for r in rows:
        best = None
        for qd in r["quarters"]:
            ts = index.get((qd["q"], qd["ch"]), [])
            real = []
            for t, jump, snr, half in qd["steps"]:
                n_steps += 1
                if half > 1:
                    continue
                n_sharp += 1
                lo = np.searchsorted([x[0] for x in ts], t - 1.5 * keplevels_CAD)
                others = {kid for tt, kid in ts[lo:] if tt <= t + 1.5 * keplevels_CAD and kid != r["kepid"]}
                if len(others) >= keplevels_COMMON:
                    n_common += 1
                    continue
                pq = plane[qd["q"]]
                # the spacecraft's events step many stars across the focal plane at once
                if np.searchsorted(pq, t + 1.5 * keplevels_CAD, side="right") - np.searchsorted(pq, t - 1.5 * keplevels_CAD) - 1 >= keplevels_PLANE:
                    n_plane += 1
                    continue
                real.append(dict(t=t, jump=jump, snr=snr))
            segs = [dict(t0=a, t1=b, level=lv, n=nn) for a, b, lv, nn in qd["segs"]]
            nlev, revisited = levels_count(segs, real, qd["sd"], qd["sig"], keplevels_W, keplevels_CAD)
            row = dict(q=qd["q"], ch=qd["ch"], n_real=len(real), levels=nlev, revisited=revisited, steps=real[:12])
            if best is None or (revisited, len(real)) > (best["revisited"], best["n_real"]):
                best = row
        if best:
            out.append(dict(kepid=r["kepid"], kepmag=r["kepmag"], n_quarters=len(r["quarters"]), **best))
    multi = [o for o in out if o["revisited"] >= 3]
    up = sum(1 for o in out for st in o["steps"] if st["jump"] > 0)
    print("%d stars, %d quarters; steps %d, sharp %d, %d of those shared on the channel and %d by %d or more stars of the focal plane; "
          "stars with real sharp steps %d; with three or more levels each revisited in a quarter %d; real steps up %d, down %d"
          % (len(out), sum(o["n_quarters"] for o in out), n_steps, n_sharp, n_common, n_plane, keplevels_PLANE,
             sum(1 for o in out if o["n_real"]), len(multi), up, sum(1 for o in out for st in o["steps"]) - up))
    for o in sorted(multi, key=lambda o: (-o["revisited"], -o["n_real"]))[:25]:
        print("  KIC %d (Kp %s) Q%d ch%d: %d real sharp steps, %d levels, %d revisited; %s" % (
            o["kepid"], o["kepmag"], o["q"], o["ch"], o["n_real"], o["levels"], o["revisited"],
            ", ".join("%.3f %+.4f" % (st["t"], st["jump"]) for st in o["steps"][:6])))
    json.dump(dict(n_stars=len(out), n_steps=n_steps, n_sharp=n_sharp, n_common=n_common, n_plane=n_plane, rows=out,
                   params=dict(window=keplevels_W, snr=levels_SNR, common=keplevels_COMMON, plane=keplevels_PLANE)),
              open(core.results("keplevels.json"), "w"))


keplevels_GIANT_LOGG = 3.5  # giants' oscillations and granulation move the window medians in steps


def keplevels_vet():
    """Each star with three or more revisited levels against the Kepler eclipsing-binary catalogue (Kirk et al. 2016), the DR25 gravity, its KOIs' transit times, its SIMBAD type and Gaia DR3's variability class, into results/keplevels_vet.json."""
    from astroquery.vizier import Vizier
    d = json.load(open(core.results("keplevels.json")))
    multi = [o for o in d["rows"] if o["revisited"] >= 3]
    stars = {x["kepid"]: x for x in json.load(open(os.path.join(keplevels_ROOT, "targets.json")))}
    eb = set()
    for t in Vizier(columns=["KIC"], row_limit=-1).get_catalogs("J/AJ/151/68"):
        if "KIC" in t.colnames:
            eb |= {int(x) for x in t["KIC"]}
    ids = ",".join(str(o["kepid"]) for o in multi)
    kois = core.exoplanets("SELECT kepid, kepoi_name, kepler_name, koi_period, koi_time0bk, koi_duration FROM cumulative "
                           "WHERE koi_disposition IN ('CANDIDATE', 'CONFIRMED') AND kepid IN (%s)" % ids)
    sim = core.simbad("SELECT i.id AS kic, b.oid, b.main_id, b.otype FROM ident i JOIN basic b ON b.oid = i.oidref WHERE i.id IN (%s)"
                      % ",".join("'KIC %d'" % o["kepid"] for o in multi))
    by_kic = {int(x["kic"].split()[1]): x for x in sim}
    gaia_ids = {x["oidref"]: x["id"].split()[-1] for x in core.simbad(
        "SELECT oidref, id FROM ident WHERE oidref IN (%s) AND id LIKE 'Gaia DR3%%'" % ",".join(str(x["oid"]) for x in sim))} if sim else {}
    classes = {str(x["source_id"]): (x["best_class_name"], float(x["best_class_score"])) for x in core.gaia(
        "SELECT source_id, best_class_name, best_class_score FROM gaiadr3.vari_classifier_result WHERE source_id IN (%s)"
        % ",".join(gaia_ids.values()))} if gaia_ids else {}
    out = {}
    for o in multi:
        k, why, vet = o["kepid"], [], []
        x = by_kic.get(k)
        name = x["main_id"] if x else "KIC %d" % k
        if k in eb:
            vet.append("eclipsing-binary")
            why.append("in the Kepler eclipsing-binary catalogue")
        logg = stars.get(k, {}).get("logg")
        if logg is not None and logg < keplevels_GIANT_LOGG:
            vet.append("evolved")
            why.append("a giant (log g %.2f), whose oscillations and granulation step the window medians" % logg)
        for kr in (r for r in kois if int(r["kepid"]) == k):
            per, t0, dur = num(kr["koi_period"]), num(kr["koi_time0bk"]), num(kr["koi_duration"])
            if not (per and t0 and dur):
                continue
            near = [abs(((st["t"] - t0 + per / 2) % per) - per / 2) <= dur / 48 + 1.5 * keplevels_CAD for st in o["steps"]]
            if sum(near) >= len(near) / 2:
                vet.append("natural-source")
                why.append("the transits of %s at %.4f d" % (kr["kepler_name"] or kr["kepoi_name"], per))
        if x and x["otype"] in ("CV*", "DN*", "No*", "NL*"):
            vet.append("natural-type")
            why.append("the cataclysmic variable %s" % name)
        cl = classes.get(gaia_ids.get(x["oid"]) if x else None)
        if cl and cl[1] >= 0.5:
            vet.append("variable-star")
            why.append("Gaia DR3 classes it %s (%.2f)" % cl)
        elif x and x["otype"] in ("dS*", "gD*", "RR*", "Ce*", "Pu*", "LP*", "Mi*", "sr*"):
            vet.append("variable-star")
            why.append("a pulsating star (%s) in SIMBAD" % x["otype"])
        out[str(k)] = dict(name=name, vet=";".join(dict.fromkeys(vet)) or None, why="; ".join(why) or None, q=o["q"], ch=o["ch"],
                           n_real=o["n_real"], levels=o["levels"], revisited=o["revisited"], kepmag=o["kepmag"], logg=logg,
                           jumps=[round(st["jump"], 4) for st in o["steps"][:8]])
    left = [v for v in out.values() if not v["vet"]]
    print("%d stars with three or more revisited levels: %d eclipsing binaries, %d giants, %d planets' transits, %d cataclysmic variables, "
          "%d pulsators; %d left" % (len(out), sum("eclipsing-binary" in (v["vet"] or "") for v in out.values()),
                                     sum("evolved" in (v["vet"] or "") for v in out.values()),
                                     sum("natural-source" in (v["vet"] or "") for v in out.values()),
                                     sum("natural-type" in (v["vet"] or "") for v in out.values()),
                                     sum("variable-star" in (v["vet"] or "") for v in out.values()), len(left)))
    for v in left:
        print("  %s (Kp %s, log g %s) Q%d ch%d: %d real sharp steps, %d levels, %d revisited; jumps %s" % (
            v["name"], v["kepmag"], v["logg"], v["q"], v["ch"], v["n_real"], v["levels"], v["revisited"], v["jumps"]))
    json.dump(out, open(core.results("keplevels_vet.json"), "w"), indent=1)


def keplevels_record():
    d = json.load(open(core.results("keplevels.json")))
    vets = json.load(open(core.results("keplevels_vet.json")))
    multi = [o for o in d["rows"] if o["revisited"] >= 3]
    left = [o for o in multi if not vets[str(o["kepid"])]["vet"]]
    core.record_search(
        "kepler_levels", "Flux in discrete levels, Kepler", archive="Kepler DR25 long-cadence light curves (STScI AWS mirror)",
        release="DR25", selection="the DR25 stellar table: %d stars with light curves" % d["n_stars"],
        code=os.path.abspath(__file__), entry="keplevels", n_targets=d["n_stars"], n_unit="stars", params=d["params"],
        outputs=[core.results("keplevels.json"), core.results("keplevels_vet.json")],
        null_summary="steps that three or more other stars of the CCD channel share within 1.5 cadences: %d, and that %d or more stars of the "
                     "focal plane share: %d, of %d sharp steps" % (d["n_common"], d["params"]["plane"], d["n_plane"], d["n_sharp"]),
        sensitivity="levels separated by 7 sigma of the 7.4-hour median noise, transitions within one cadence",
        ledger="Flux in discrete levels, Kepler",
        result="%d stars: %d with real sharp steps, %d with three or more levels each revisited in a quarter, %d of them unexplained%s" % (
            d["n_stars"], sum(1 for o in d["rows"] if o["n_real"]), len(multi), len(left),
            (": " + ", ".join("KIC %d Q%d" % (o["kepid"], o["q"]) for o in left[:8])) if left else ""))
    core.clear_targets("kepler_levels")

    def vet(o):
        v = vets.get(str(o["kepid"]))
        return (v["vet"], v["why"]) if v else (None, None)
    core.record_targets("kepler_levels", [dict(target="kic:%d" % o["kepid"], stat=float(o["revisited"]), unit="levels revisited at least twice in a quarter",
                                                vet=vet(o)[0], note="Q%d channel %d: %d real sharp steps, %d levels%s" % (
                                                    o["q"], o["ch"], o["n_real"], o["levels"], "; " + vet(o)[1] if vet(o)[1] else ""))
                                           for o in d["rows"] if o["n_real"]])
    print("recorded %d stars, rows for the %d with real sharp steps" % (d["n_stars"], sum(1 for o in d["rows"] if o["n_real"])))


def keplevels_cli():
    quiet_warnings()
    p = argparse.ArgumentParser()
    p.add_argument("mode", choices=["targets", "stream", "analyse", "vet", "record"])
    p.add_argument("--part", default=None)
    p.add_argument("--threads", type=int, default=4)
    p.add_argument("--sub", default=None)
    a = p.parse_args()
    if a.mode == "stream":
        sys.exit(1 if keplevels_stream(a.part, a.threads, a.sub) else 0)
    else:
        {"targets": keplevels_targets, "analyse": keplevels_analyse, "vet": keplevels_vet, "record": keplevels_record}[a.mode]()


determinism_DOC = r"""Determinism in light curves: each segment's residual, once the slow trend, up to three periodic signals and flares are removed, against IAAFT surrogates of the same power spectrum and flux distribution, at the native cadence and binned eightfold: the compressed length of its quantile symbols, its nonlinear prediction error and its time asymmetry; a candidate is more compressible or more predictable than its surrogates by 5 sigma, its asymmetry read beside it (flares carry asymmetry without predictability, chaos carries both), unshared by the stars of its sector and camera; a chaotic signal at the noise's level reaches 14 sigma in prediction binned, one at a third of it is not found.

    python -I seti.py flashes determinism targets --lists DIR [--plx 10]   (TESS two-minute star-sectors of the Gaia nearby stars beyond parallax plx, nearest first, into ROOT/targets.json; DIR holds the all_targets_S<nnn> lists as s<nnn>.csv)
    python -I seti.py flashes determinism stream [--part k/n] [--procs 8] [--gpu]   (SPOC two-minute light curves read in memory, one line a star-sector into ROOT/stats_<k>of<n>.jsonl)
    python -I seti.py flashes determinism local DIR [--procs 6] [--tag NAME]   (every FITS light curve under DIR, 20 s or Kepler short cadence, into ROOT/stats_<NAME>.jsonl)
    python -I seti.py flashes determinism kepler_targets   (every KOI host with a candidate or confirmed planet into ROOT/targets_kepler.json)
    python -I seti.py flashes determinism kepler [--procs 10]   (each host's long-cadence quarters read in memory, every segment on the CPU, into ROOT/stats_kepler.jsonl; exits 1 while stars remain unread)
    python -I seti.py flashes determinism check   (white and red noise, a chaotic signal, a coded on-off signal and flares through the same statistics)
    python -I seti.py flashes determinism report"""

determinism_ROOT = os.environ.get("DETERMINISM_ROOT", core.data("determinism"))
determinism_NSUR = 99  # surrogates a segment and level
determinism_ITERS = 40  # IAAFT iterations
determinism_LEVELS = 16  # quantile symbols for the compressed length
determinism_EMBED = 4  # embedding dimension of the nonlinear prediction
determinism_K = 4  # neighbours a prediction
determinism_QUERY = 3000  # predicted samples a series
determinism_LAGS = (1, 2, 4, 8, 16, 32)
determinism_BIN = 8  # cadences a sample at the coarse level
determinism_MIN_DAYS = 2.0
determinism_GAP = 0.25  # days; a longer gap splits a segment
determinism_FLARE = 5.0  # robust sigma


def determinism_segments(t, f, cad, seed=0):
    """Uniform segments of a light curve's residual: [(residual, info)] with the slow trend (half-day block medians), up to three periodic signals (each folded at its Lomb-Scargle peak and its phase-binned median subtracted) and flares (5 sigma brightenings and the half hour after) removed, flare and missing cadences refilled with noise of the residual's scatter."""
    from astropy.timeseries import LombScargle
    o = np.argsort(t)
    t, f = t[o], f[o]
    cut = np.nonzero(np.diff(t) > determinism_GAP)[0] + 1
    out = []
    rng = np.random.default_rng(seed)
    for a, b in zip(np.r_[0, cut], np.r_[cut, len(t)]):
        tt, ff = t[a:b], f[a:b]
        if len(tt) < 500 or tt[-1] - tt[0] < determinism_MIN_DAYS:
            continue
        pos = np.round((tt - tt[0]) / cad).astype(np.int64)
        n = int(pos[-1]) + 1
        x = np.full(n, np.nan)
        x[pos] = ff / np.median(ff) - 1.0
        fill = 1.0 - len(np.unique(pos)) / n
        if fill > 0.3:
            continue
        x = x - science.block_median(x, max(5, int(0.5 / cad)))
        grid = tt[0] + cad * np.arange(n)
        ok = np.isfinite(x)
        periods = []
        for _ in range(3):
            ts, xs = grid[ok], x[ok]
            if len(xs) > 20000:
                k = max(1, len(xs) // 20000)
                ts, xs = ts[::k], xs[::k]
            fmin, fmax = 2.0 / (grid[-1] - grid[0]), min(0.5 / cad, 100.0)
            freq = np.linspace(fmin, fmax, int(5 * (grid[-1] - grid[0]) * (fmax - fmin)))
            pw = LombScargle(ts, xs).power(freq, method="fast")
            j = int(np.argmax(pw))
            if not pw[j] > 30 * np.median(pw):
                break
            P = 1.0 / freq[j]
            ph = ((grid - grid[0]) / P) % 1.0
            nb = int(np.clip(ok.sum() / 50, 20, 200))
            k = (ph * nb).astype(int)
            med = np.array([np.nanmedian(x[k == i]) if np.any(ok & (k == i)) else 0.0 for i in range(nb)])
            x = x - np.interp(ph, (np.arange(nb) + 0.5) / nb, med, period=1.0)
            periods.append([round(float(P), 6), round(float(pw[j]), 4)])
        s = core.robust_sigma(x[ok])
        hot = ok & (x > determinism_FLARE * s)
        after = max(3, int(round(0.02 / cad)))
        mask = np.convolve(hot.astype(float), np.ones(after + 2), mode="full")[1:n + 1] > 0
        events = int(np.sum(np.diff(np.r_[0, hot.astype(int)]) == 1))
        bad = mask | ~ok
        x[bad] = rng.normal(0.0, s, int(bad.sum()))
        out.append((x, dict(t0=round(float(grid[0]), 4), t1=round(float(grid[-1]), 4), n=n, fill=round(float(fill), 4),
                            masked=round(float(mask.mean()), 4), periods=periods, flares=events, sigma=float(s))))
    return out


def determinism_iaaft(x, n, iters, rng):
    """n IAAFT surrogates of x: the amplitude spectrum imposed and the values rank-remapped in turn, ending on the exact values."""
    N = len(x)
    amp = np.abs(np.fft.rfft(x))
    sx = np.sort(x)
    Y = np.array([rng.permutation(x) for _ in range(n)])
    for _ in range(iters):
        F = np.fft.rfft(Y, axis=1)
        Y = np.fft.irfft(amp[None, :] * F / np.maximum(np.abs(F), 1e-30), N, axis=1)
        np.put_along_axis(Y, np.argsort(Y, axis=1), np.broadcast_to(sx, Y.shape), axis=1)
    return Y


def determinism_iaaft_torch(x, n, iters, seed, dev):
    """determinism_iaaft on a torch device."""
    import torch
    N = len(x)
    xt = torch.as_tensor(x, dtype=torch.float32, device=dev)
    amp = torch.abs(torch.fft.rfft(xt))
    sx = torch.sort(xt).values.expand(n, N).contiguous()
    g = torch.Generator(device=dev).manual_seed(seed)
    Y = xt[torch.argsort(torch.rand(n, N, generator=g, device=dev), dim=1)]
    for _ in range(iters):
        F = torch.fft.rfft(Y, dim=1)
        Y = torch.fft.irfft(amp[None, :] * F / F.abs().clamp_min(1e-30), N, dim=1)
        Y = torch.empty_like(Y).scatter_(1, torch.argsort(Y, dim=1), sx)
    return Y


def determinism_window(x):
    """Theiler window: the first lag where the autocorrelation falls below 1/e, within 5-50 samples."""
    y = x - x.mean()
    n = 1 << int(np.ceil(np.log2(2 * len(y))))
    acf = np.fft.irfft(np.abs(np.fft.rfft(y, n)) ** 2, n)[:200]
    acf = acf / max(acf[0], 1e-30)
    below = np.nonzero(acf < 1 / math.e)[0]
    return int(np.clip(below[0] if len(below) else 50, 5, 50))


def determinism_asym(X):
    """Time asymmetry of each row: the root sum of squares over lags of <d^3>/<d^2>^1.5, d the lagged difference."""
    rows = []
    for L in determinism_LAGS:
        if L >= X.shape[1] // 10:
            break
        d = X[:, L:] - X[:, :-L]
        rows.append(np.mean(d ** 3, axis=1) / np.maximum(np.mean(d ** 2, axis=1), 1e-30) ** 1.5)
    return np.sqrt(np.sum(np.array(rows) ** 2, axis=0))


def determinism_compress(X, edges, threads=4):
    """zlib-compressed length (bytes) of each row's quantile symbols."""
    import zlib
    S = np.searchsorted(edges, X).astype(np.uint8)
    with ThreadPoolExecutor(threads) as ex:
        return np.array(list(ex.map(lambda s: len(zlib.compress(s.tobytes(), 9)), S)), float)


def determinism_npe(X, qidx, w):
    """Nonlinear prediction error of each row: the next value from the mean of the K nearest delay vectors more than w samples away, its RMS error over the row's scatter at the samples qidx."""
    from scipy.spatial import cKDTree
    m, k = determinism_EMBED, determinism_K
    out = []
    for x in X:
        N = len(x)
        E = np.stack([x[m - 1 - j:N - 1 - j] for j in range(m)], axis=1)
        target = x[m:]
        _, nb = cKDTree(E).query(E[qidx], k=k + 2 * w + 1)
        far = np.abs(nb - qidx[:, None]) > w
        rank = np.cumsum(far, axis=1)
        use = far & (rank <= k)
        cnt = use.sum(1)
        pred = np.where(use, target[np.minimum(nb, len(target) - 1)], 0.0).sum(1) / np.maximum(cnt, 1)
        good = cnt > 0
        out.append(math.sqrt(float(np.mean((target[qidx][good] - pred[good]) ** 2))) / max(float(np.std(x)), 1e-30))
    return np.array(out)


def determinism_npe_torch(X, qidx, w, dev, chunk=32):
    """determinism_npe on a torch device for every row at once: distances by chunks of queries, near-in-time vectors set to infinity."""
    import torch
    m, k = determinism_EMBED, determinism_K
    B, N = X.shape
    E = torch.stack([X[:, m - 1 - j:N - 1 - j] for j in range(m)], dim=2)
    target = X[:, m:]
    idx = torch.arange(E.shape[1], device=dev)
    qi = torch.as_tensor(qidx, device=dev)
    se = torch.zeros(B, device=dev, dtype=torch.float64)
    for c in range(0, len(qi), chunk):
        q = qi[c:c + chunk]
        d = torch.cdist(E[:, q], E)
        d.masked_fill_(((idx[None, :] - q[:, None]).abs() <= w)[None], float("inf"))
        nb = torch.topk(d, k, dim=2, largest=False).indices
        pred = torch.gather(target, 1, nb.reshape(B, -1)).reshape(B, len(q), k).mean(2)
        se += ((target[:, q] - pred) ** 2).sum(1).double()
    return (torch.sqrt(se / len(qi)) / X.std(dim=1, correction=0).double().clamp_min(1e-30)).cpu().numpy()


def determinism_level(x, seed, dev=None):
    """One series against its surrogates: (value, surrogate mean, surrogate scatter, z) of the compressed length, the prediction error and the asymmetry."""
    rng = np.random.default_rng(seed)
    w = determinism_window(x)
    N = len(x)
    qidx = np.sort(rng.choice(N - determinism_EMBED - 1, min(determinism_QUERY, N - determinism_EMBED - 1), replace=False))
    edges = np.quantile(x, np.arange(1, determinism_LEVELS) / determinism_LEVELS)
    if dev is not None:
        import torch
        Y = determinism_iaaft_torch(x, determinism_NSUR, determinism_ITERS, seed, dev)
        Xt = torch.cat([torch.as_tensor(x, dtype=torch.float32, device=dev)[None, :], Y])
        npe = determinism_npe_torch(Xt, qidx, w, dev)
        X = Xt.cpu().numpy().astype(np.float64)
    else:
        X = np.vstack([x[None, :], determinism_iaaft(x, determinism_NSUR, determinism_ITERS, rng)])
        npe = determinism_npe(X, qidx, w)
    stats = dict(comp=determinism_compress(X, edges), npe=npe, asym=determinism_asym(X))
    out = dict(w=w)
    for key, v in stats.items():
        mu, sd = float(np.mean(v[1:])), float(np.std(v[1:], ddof=1))
        out[key] = [round(float(v[0]), 6), round(mu, 6), round(sd, 6), round((float(v[0]) - mu) / max(sd, 1e-12), 2)]
    return out


def determinism_curve(t, f, cad, seed=0, dev=None):
    """Every segment of a light curve at the native cadence and binned determinism_BIN-fold, with z combined over segments."""
    segs = []
    for i, (x, info) in enumerate(determinism_segments(t, f, cad, seed)):
        info["native"] = determinism_level(x, seed + 101 * i, dev)
        nb = len(x) // determinism_BIN
        if nb >= 500:
            info["coarse"] = determinism_level(x[:nb * determinism_BIN].reshape(nb, determinism_BIN).mean(1), seed + 101 * i + 7, dev)
        segs.append(info)
    Z = {}
    for lev in ("native", "coarse"):
        zs = [s[lev] for s in segs if lev in s]
        if zs:
            Z[lev] = [round(sum(z[key][3] for z in zs) / math.sqrt(len(zs)), 2) for key in ("comp", "npe", "asym")]
    return dict(cad_s=round(cad * 86400, 2), segs=segs, Z=Z)


def determinism_lists(dirpath):
    """{(tic, sector): (camera, ccd, tmag, ra, dec)} from the two-minute target lists s<nnn>.csv under dirpath."""
    out = {}
    for p in sorted(glob.glob(os.path.join(dirpath, "s[0-9][0-9][0-9].csv"))):
        sector = int(os.path.basename(p)[1:4])
        for line in open(p, encoding="utf-8", errors="replace"):
            v = line.strip().split(",")
            if len(v) == 6 and v[0].isdigit():
                out[(int(v[0]), sector)] = (int(v[1]), int(v[2]), num(v[3]), float(v[4]), float(v[5]))
    return out


def determinism_targets(lists, plx_min=10.0):
    """TESS two-minute star-sectors of the Gaia Catalogue of Nearby Stars' sources beyond parallax plx_min, matched within 3 arcsec at the nearest of epochs 2000, 2016 and 2021 and within 2.5 mag of G, nearest first, into ROOT/targets.json."""
    from scipy.spatial import cKDTree
    os.makedirs(determinism_ROOT, exist_ok=True)
    rows = catalogues.gcns("source_id, ra, dec, pmra, pmdec, parallax, phot_g_mean_mag", "parallax > %g" % plx_min, maxrec=600000, timeout=1800)
    print("%d nearby stars beyond parallax %g mas" % (len(rows), plx_min), flush=True)
    L = determinism_lists(lists)
    print("%d two-minute star-sectors in %s" % (len(L), lists), flush=True)
    pos = {}
    for (tic, sector), v in L.items():
        pos.setdefault(tic, (v[3], v[4], v[2]))
    tics = np.array(list(pos))
    radec = np.array([pos[x][:2] for x in tics])
    tree = cKDTree(science.unit_vector(radec[:, 0], radec[:, 1]).T)
    match = {}
    for r in rows:
        d, j = min(tree.query(science.unit_vector(*core.star_at(float(r["ra"]), float(r["dec"]), num(r["pmra"], 0.0),
                                                                num(r["pmdec"], 0.0), 2016.0, at)), k=1)
                   for at in (2000.0, 2016.0, 2021.0))
        if d < math.radians(3.0 / 3600) and abs((pos[tics[j]][2] or 99) - num(r["phot_g_mean_mag"], 99)) < 2.5:
            tic = int(tics[j])
            if tic not in match or float(r["parallax"]) > match[tic][1]:
                match[tic] = (r["source_id"], float(r["parallax"]), num(r["phot_g_mean_mag"]))
    out = [dict(tic=tic, sector=sector, gaia=match[tic][0], plx=round(match[tic][1], 3), G=match[tic][2], cam=v[0], ccd=v[1])
           for (tic, sector), v in L.items() if tic in match]
    out.sort(key=lambda r: (-r["plx"], r["tic"], r["sector"]))
    json.dump(out, open(os.path.join(determinism_ROOT, "targets.json"), "w"))
    print("%d star-sectors of %d stars" % (len(out), len({r["tic"] for r in out})), flush=True)


def determinism_one_spoc(x):
    """One star-sector's SPOC two-minute light curve read in memory and its statistics on the CPU; None when unreadable."""
    try:
        b = core.spoc_bytes(x["tic"], x["sector"])
        if not b:
            return dict(x, status="no light curve")
        t, f, _ = core.read(b, columns=("PDCSAP_FLUX",))
        if len(t) < 1000:
            return dict(x, status="short")
        cad = float(np.median(np.diff(t)))
        if _DET_DEV.get("gpu"):
            return dict(x, status="segments", cad=cad, segs=[(s.astype(np.float32), info) for s, info in
                                                              determinism_segments(t, f, cad, seed=x["tic"] % 100000)])
        return dict(x, status="ok", **determinism_curve(t, f, cad, seed=x["tic"] % 100000))
    except Exception as e:
        return dict(x, status="error", error=repr(e)[:200])


_DET_DEV = {}


def determinism_pool_init(gpu):
    os.environ.setdefault("OMP_NUM_THREADS", "1")
    _DET_DEV["gpu"] = gpu


def determinism_finish_gpu(r, dev):
    """Statistics on the GPU for a star-sector whose segments a worker prepared."""
    segs = []
    seed = r["tic"] % 100000
    for i, (x, info) in enumerate(r.pop("segs")):
        x = x.astype(np.float64)
        info["native"] = determinism_level(x, seed + 101 * i, dev)
        nb = len(x) // determinism_BIN
        if nb >= 500:
            info["coarse"] = determinism_level(x[:nb * determinism_BIN].reshape(nb, determinism_BIN).mean(1), seed + 101 * i + 7, dev)
        segs.append(info)
    Z = {}
    for lev in ("native", "coarse"):
        zs = [s[lev] for s in segs if lev in s]
        if zs:
            Z[lev] = [round(sum(z[key][3] for z in zs) / math.sqrt(len(zs)), 2) for key in ("comp", "npe", "asym")]
    r.update(status="ok", cad_s=round(r.pop("cad") * 86400, 2), segs=segs, Z=Z)
    return r


def determinism_done():
    done = set()
    for p in glob.glob(os.path.join(determinism_ROOT, "stats_*.jsonl")):
        for ln in open(p, encoding="utf-8", errors="replace"):
            try:
                r = json.loads(ln)
            except ValueError:
                continue
            if r.get("status") in ("ok", "no light curve", "short"):
                done.add(r["id"])
    return done


def determinism_stream(part=None, procs=8, gpu=False):
    """Stream a part of the target star-sectors: workers read each light curve in memory and prepare its segments (and on the CPU its statistics); with gpu the surrogates and predictions run in this process on the GPU, the workers started by spawn so that none inherits its CUDA context."""
    import multiprocessing
    from concurrent.futures import ProcessPoolExecutor
    targets = json.load(open(os.path.join(determinism_ROOT, "targets.json")))
    k, n = (int(v) for v in part.split("/")) if part else (0, 1)
    done = determinism_done()
    todo = [dict(x, id="tic:%d:s%d" % (x["tic"], x["sector"])) for x in targets[k::n]]
    todo = [x for x in todo if x["id"] not in done]
    dev = None
    if gpu:
        import torch
        dev = torch.device("cuda")
        torch.cuda.set_per_process_memory_fraction(0.25)
    path = os.path.join(determinism_ROOT, "stats_%dof%d.jsonl" % (k, n))
    print("%d star-sectors to read, %d done under the root, statistics on %s" % (len(todo), len(done), dev or "the CPU"), flush=True)
    t0, m = time.time(), 0
    with ProcessPoolExecutor(procs, mp_context=multiprocessing.get_context("spawn"), initializer=determinism_pool_init,
                             initargs=(gpu,)) as ex, open(path, "a", encoding="utf-8") as fh:
        for r in ex.map(determinism_one_spoc, todo, chunksize=1):
            if r.get("status") == "segments":
                r = determinism_finish_gpu(r, dev)
            fh.write(json.dumps(r, separators=(",", ":")) + "\n")
            m += 1
            if m % 50 == 0:
                fh.flush()
                print("%d of %d, %.1f a minute" % (m, len(todo), 60 * m / (time.time() - t0)), flush=True)
    print("%d star-sectors written" % m, flush=True)


def determinism_one_file(path):
    """One cached FITS light curve's statistics on the CPU."""
    name = os.path.basename(path)
    try:
        t, f, _ = core.read(open(path, "rb").read(), columns=("PDCSAP_FLUX",))
        if len(t) < 1000:
            return dict(id=name, status="short")
        cad = float(np.median(np.diff(t)))
        seed = int(re.sub(r"\D", "", name)[-5:] or 0)
        return dict(id=name, status="ok", **determinism_curve(t, f, cad, seed=seed))
    except Exception as e:
        return dict(id=name, status="error", error=repr(e)[:200])


def determinism_local(directory, procs=6, tag="local"):
    """Every FITS light curve under a directory through determinism_curve on procs processes, into ROOT/stats_<tag>.jsonl."""
    from concurrent.futures import ProcessPoolExecutor
    os.makedirs(determinism_ROOT, exist_ok=True)
    files = sorted(glob.glob(os.path.join(directory, "**", "*.fits"), recursive=True))
    path = os.path.join(determinism_ROOT, "stats_%s.jsonl" % tag)
    done = set()
    if os.path.exists(path):
        for ln in open(path, encoding="utf-8", errors="replace"):
            try:
                done.add(json.loads(ln)["id"])
            except (ValueError, KeyError):
                pass
    todo = [p for p in files if os.path.basename(p) not in done]
    print("%d light curves under %s, %d to do" % (len(files), directory, len(todo)), flush=True)
    t0, m = time.time(), 0
    with ProcessPoolExecutor(procs, initializer=determinism_pool_init, initargs=(False,)) as ex, open(path, "a", encoding="utf-8") as fh:
        for r in ex.map(determinism_one_file, todo, chunksize=1):
            fh.write(json.dumps(r, separators=(",", ":")) + "\n")
            m += 1
            if m % 20 == 0:
                fh.flush()
                print("%d of %d, %.2f a minute" % (m, len(todo), 60 * m / (time.time() - t0)), flush=True)
    print("%d light curves written" % m, flush=True)


def determinism_kepler_targets():
    """Every KOI host with a candidate or confirmed planet, brightest first, into ROOT/targets_kepler.json."""
    os.makedirs(determinism_ROOT, exist_ok=True)
    rows = core.exoplanets("SELECT kepid, koi_kepmag, koi_disposition FROM cumulative")
    best = {}
    for r in rows:
        if r["koi_disposition"] in ("CANDIDATE", "CONFIRMED") and num(r["kepid"]):
            best[int(r["kepid"])] = num(r["koi_kepmag"], 99.0)
    out = [dict(kepid=k, kepmag=v) for k, v in sorted(best.items(), key=lambda kv: kv[1])]
    json.dump(out, open(os.path.join(determinism_ROOT, "targets_kepler.json"), "w"))
    print("%d KOI hosts" % len(out))


def determinism_one_kepler(x):
    """One Kepler star's long-cadence quarters read in memory, every segment of every quarter through the statistics on the CPU, z combined over them; None when the listing or a quarter fails, so the star is read again."""
    keys = core.kepler_keys(x["kepid"], short=False)
    if not keys:
        return None
    segs = []
    for i, key in enumerate(keys):
        b = core.get(core.S3 + key, timeout=300, retries=3, soft=True, pause=2)
        if not b or not b.startswith(b"SIMPLE"):
            return None
        try:
            t, f, _ = core.read(b, columns=("PDCSAP_FLUX",))
        except Exception:
            continue
        if len(t) < 1000:
            continue
        r = determinism_curve(t, f, float(np.median(np.diff(t))), seed=(x["kepid"] + 7919 * i) % 100000)
        for s in r["segs"]:
            s["file"] = os.path.basename(key)
        segs += r["segs"]
    Z = {}
    for lev in ("native", "coarse"):
        zs = [s[lev] for s in segs if lev in s]
        if zs:
            Z[lev] = [round(sum(z[k][3] for z in zs) / math.sqrt(len(zs)), 2) for k in ("comp", "npe", "asym")]
    return dict(id="kic:%d" % x["kepid"], kepid=x["kepid"], kepmag=x["kepmag"], status="ok", cad_s=1765.5, segs=segs, Z=Z)


def determinism_kepler(procs=10):
    """Every KOI host of ROOT/targets_kepler.json through determinism_one_kepler, one line a star into ROOT/stats_kepler.jsonl."""
    from concurrent.futures import ProcessPoolExecutor
    targets = json.load(open(os.path.join(determinism_ROOT, "targets_kepler.json")))
    done = determinism_done()
    todo = [x for x in targets if "kic:%d" % x["kepid"] not in done]
    path = os.path.join(determinism_ROOT, "stats_kepler.jsonl")
    print("%d KOI hosts to read, %d done" % (len(todo), len(targets) - len(todo)), flush=True)
    t0, m, miss = time.time(), 0, 0
    with ProcessPoolExecutor(procs, initializer=determinism_pool_init, initargs=(False,)) as ex, open(path, "a", encoding="utf-8") as fh:
        for r in ex.map(determinism_one_kepler, todo, chunksize=1):
            if r is None:
                miss += 1
                continue
            fh.write(json.dumps(r, separators=(",", ":")) + "\n")
            m += 1
            if m % 20 == 0:
                fh.flush()
                print("%d of %d, %.1f stars a minute, %d to read again" % (m, len(todo), 60 * m / (time.time() - t0), miss), flush=True)
    print("%d stars written, %d to read again" % (m, miss), flush=True)
    return miss


def determinism_check(gpu=False):
    """The statistics on synthetic segments of two-minute cadence: white noise and red noise (AR(1)) as the null, a chaotic signal (the logistic map's orbit, held for ten cadences a step) and a coded on-off signal (a repeating 63-step maximal-length sequence) at a tenth of the noise and a third of it, and flares on red noise."""
    dev = None
    if gpu:
        import torch
        dev = torch.device("cuda")
    rng = np.random.default_rng(5)
    N = 9000
    ar = np.zeros(N)
    e = rng.normal(0, 1, N)
    for i in range(1, N):
        ar[i] = 0.9 * ar[i - 1] + e[i]
    ar /= ar.std()
    lm = np.empty(N // 10 + 1)
    lm[0] = 0.4
    for i in range(1, len(lm)):
        lm[i] = 3.99 * lm[i - 1] * (1 - lm[i - 1])
    chaos = np.repeat(lm - lm.mean(), 10)[:N] / lm.std()
    reg = [1, 0, 0, 0, 0, 0]
    seq = []
    for _ in range(63):
        seq.append(reg[-1])
        reg = [reg[0] ^ reg[-1]] + reg[:-1]
    code = np.tile(np.repeat(np.array(seq, float) * 2 - 1, 8), N // (63 * 8) + 1)[:N]
    flares = ar.copy()
    for t0 in rng.choice(N - 60, 12, replace=False):
        flares[t0:t0 + 60] += 8 * np.exp(-np.arange(60) / 8.0)
    cases = [("white noise", rng.normal(0, 1, N)), ("red noise", ar)]
    for amp in (0.33, 1.0, 3.0):
        cases += [("chaos at %.2f of the noise" % amp, rng.normal(0, 1, N) + amp * chaos),
                  ("code at %.2f of the noise" % amp, rng.normal(0, 1, N) + amp * code)]
    cases.append(("flares on red noise", flares))
    for name, x in cases:
        r = determinism_level(x, 11, dev)
        nb = len(x) // determinism_BIN
        c = determinism_level(x[:nb * determinism_BIN].reshape(nb, determinism_BIN).mean(1), 12, dev)
        print("%-27s z compressed, prediction, asymmetry: native %+7.2f %+7.2f %+7.2f; binned %+7.2f %+7.2f %+7.2f" % (
            name, r["comp"][3], r["npe"][3], r["asym"][3], c["comp"][3], c["npe"][3], c["asym"][3]), flush=True)


def determinism_report():
    """Every star-sector's combined z at both levels: the null's spread from the population, the candidates more compressible or more predictable by 5 sigma with their asymmetry, and the stars of the same sector and camera that share it."""
    rows = {}
    for p in sorted(glob.glob(os.path.join(determinism_ROOT, "stats_*.jsonl"))):
        for ln in open(p, encoding="utf-8", errors="replace"):
            try:
                r = json.loads(ln)
            except ValueError:
                continue
            if r.get("status") == "ok" and r.get("Z"):
                rows[r["id"]] = r
    R = list(rows.values())
    print("%d star-sectors with statistics" % len(R))
    out = []
    for lev in ("native", "coarse"):
        Z = np.array([r["Z"][lev] for r in R if lev in r["Z"]])
        if not len(Z):
            continue
        print("%s: %d; z median and robust scatter compressed %+.2f %.2f, prediction %+.2f %.2f, asymmetry %+.2f %.2f" % (
            lev, len(Z), np.median(Z[:, 0]), core.robust_sigma(Z[:, 0]), np.median(Z[:, 1]), core.robust_sigma(Z[:, 1]),
            np.median(Z[:, 2]), core.robust_sigma(Z[:, 2])))
        for thr in (3, 5, 8):
            print("   below -%d: compressed %d, prediction %d; asymmetry above %d: %d" % (
                thr, np.sum(Z[:, 0] < -thr), np.sum(Z[:, 1] < -thr), thr, np.sum(Z[:, 2] > thr)))
        for r in R:
            z = r["Z"].get(lev)
            if z and min(z[0], z[1]) < -5:
                out.append(dict(id=r["id"], level=lev, z=z, gaia=r.get("gaia"), plx=r.get("plx"), G=r.get("G"), cam=r.get("cam"),
                                sector=r.get("sector"), periods=[p for s in r["segs"] for p in s["periods"]],
                                flares=sum(s["flares"] for s in r["segs"])))
    for c in out:
        if c.get("sector") is not None:
            c["shared"] = sum(1 for o in out if o["id"] != c["id"] and o["level"] == c["level"] and o.get("sector") == c["sector"]
                              and o.get("cam") == c["cam"])
    out.sort(key=lambda c: min(c["z"][0], c["z"][1]))
    for c in out[:40]:
        print("  %-28s %-6s z %s plx %s G %s cam %s, %s others in its sector and camera; periods %s; flares %d" % (
            c["id"], c["level"], c["z"], c["plx"], c["G"], c["cam"], c.get("shared"), c["periods"][:3], c["flares"]))
    json.dump(dict(n=len(R), candidates=out), open(os.path.join(determinism_ROOT, "report.json"), "w"))


def determinism_cli():
    quiet_warnings()
    p = argparse.ArgumentParser()
    p.add_argument("mode", choices=["targets", "stream", "local", "kepler_targets", "kepler", "check", "report"])
    p.add_argument("dir", nargs="?")
    p.add_argument("--lists")
    p.add_argument("--plx", type=float, default=10.0)
    p.add_argument("--part", default=None)
    p.add_argument("--procs", type=int, default=8)
    p.add_argument("--gpu", action="store_true")
    p.add_argument("--tag", default="local")
    a = p.parse_args()
    if a.mode == "targets":
        determinism_targets(a.lists, a.plx)
    elif a.mode == "stream":
        determinism_stream(a.part, a.procs, a.gpu)
    elif a.mode == "local":
        determinism_local(a.dir, a.procs, a.tag)
    elif a.mode == "kepler_targets":
        determinism_kepler_targets()
    elif a.mode == "kepler":
        sys.exit(1 if determinism_kepler(a.procs) else 0)
    elif a.mode == "check":
        determinism_check(a.gpu)
    else:
        determinism_report()


psfflash_DOC = r"""PSF-shaped flashes in Kepler's cosmic-ray corrections: the flux the pipeline removed from single cadences of each target pixel file (the COSMIC_RAYS column), in cadences where three or more pixels of the optimal aperture were corrected together, correlated with the star's own pixel image; a flash from the star follows that image (correlation 0.9 or more, 0.2 above the image shifted by one pixel), a cosmic ray or its track does not; the null is the shifted image and the coincidences each pixel's own correction rate gives, events shared by other stars of the CCD channel at the same cadence are the spacecraft's, and light left in the next cadences marks a flare.

    python -I seti.py flashes psfflash targets   (every KOI host with a candidate or confirmed planet, brightest first, into ROOT/targets.json)
    python -I seti.py flashes psfflash stream [--procs 3]   (each star's long-cadence target pixel files read in memory, one line a star into ROOT/events.jsonl; exits 1 while stars remain unread)
    python -I seti.py flashes psfflash report
    python -I seti.py flashes psfflash record   (the PSF-shaped rate against the shifted-image null by stellar temperature; each star's PSF-shaped events its flares when the star is cooler than 5300 K, carries three or more or shows light in the next cadence, else cosmic rays shaped like its image by chance)"""

psfflash_ROOT = os.environ.get("PSFFLASH_ROOT", core.data("psfflash"))
psfflash_MIN_PIX = 3
psfflash_R = 0.9
psfflash_MARGIN = 0.2
psfflash_SHIFTS = ((1, 0), (-1, 0), (0, 1), (0, -1))
psfflash_TEFF_BINS = ((2500, 4000), (4000, 4800), (4800, 5300), (5300, 5800), (5800, 6300), (6300, 12000))
psfflash_FLARE_TEFF = 5300  # below it the PSF-shaped events outrun the null: cool dwarfs' flares
psfflash_FLARE_MIN = 3  # PSF-shaped events on one star, against about half an event the null gives a star over its quarters
psfflash_AFTER = 3.0  # light in the next cadence, in the light curve's sigma


def psfflash_targets():
    """Every KOI host with a candidate or confirmed planet, brightest first, into ROOT/targets.json."""
    os.makedirs(psfflash_ROOT, exist_ok=True)
    rows = core.exoplanets("SELECT kepid, koi_kepmag, koi_disposition FROM cumulative")
    best = {}
    for r in rows:
        if r["koi_disposition"] in ("CANDIDATE", "CONFIRMED") and num(r["kepid"]):
            best[int(r["kepid"])] = num(r["koi_kepmag"], 99.0)
    out = [dict(kepid=k, kepmag=v) for k, v in sorted(best.items(), key=lambda kv: kv[1])]
    json.dump(out, open(os.path.join(psfflash_ROOT, "targets.json"), "w"))
    print("%d KOI hosts" % len(out))


def psfflash_corr(a, b):
    a, b = a - a.mean(), b - b.mean()
    d = math.sqrt(float(np.dot(a, a) * np.dot(b, b)))
    return float(np.dot(a, b) / d) if d > 0 else float("nan")


def psfflash_quarter(raw):
    """Multi-pixel corrections of one gzipped long-cadence target pixel file: cadences where psfflash_MIN_PIX or more optimal-aperture pixels were corrected, each with its correlation to the median image and to the image shifted by one pixel, its share of the star's flux and the aperture flux in the next two cadences over the scatter; with the counts the per-pixel rates expect."""
    import gzip
    from astropy.io import fits
    with fits.open(io.BytesIO(gzip.decompress(raw))) as h:
        d = h[1].data
        quarter, channel = int(h[0].header.get("QUARTER", -1)), int(h[0].header.get("CHANNEL", -1))
        t, cad, q = np.asarray(d["TIME"], float), np.asarray(d["CADENCENO"], int), np.asarray(d["QUALITY"], int)
        F, C = np.asarray(d["FLUX"], float), np.asarray(d["COSMIC_RAYS"], float)
        ap = np.asarray(h[2].data, int)
    # cadences whose only flags are a cosmic ray in the optimal aperture (128), an impulsive outlier (1024) or a cosmic ray
    # in the collateral data (2048): every other flag marks the spacecraft's events
    good = ((q & ~(128 | 1024 | 2048)) == 0) & np.isfinite(t)
    t, cad, F, C = t[good], cad[good], F[good], np.nan_to_num(C[good])
    if len(t) < 500:
        return None
    with np.errstate(all="ignore"):
        M = np.nan_to_num(np.nanmedian(F, axis=0)).clip(0)
    core_ = ((ap & 2) > 0) & (M > 0)
    if core_.sum() < psfflash_MIN_PIX + 1:
        return None
    m = M[core_]
    total = float(m.sum())
    lc = np.nansum(np.where(core_[None, :, :], F, 0.0), axis=(1, 2))
    base = science.block_median(lc, 48)
    scat = core.robust_sigma(lc - base)
    hit = (C > 0) & core_[None, :, :]
    npix = hit.sum(axis=(1, 2))
    rate = hit.mean(axis=0)[core_]
    # cadences with psfflash_MIN_PIX or more corrected pixels if the pixels were hit independently at their own rates
    p = rate.clip(0, 1)
    dist = np.zeros(len(p) + 1)
    dist[0] = 1.0
    for x in p:
        dist[1:] = dist[1:] * (1 - x) + dist[:-1] * x
        dist[0] *= 1 - x
    expect = float(len(t) * dist[psfflash_MIN_PIX:].sum())
    events = []
    shifted = [np.roll(M, s, axis=(0, 1))[core_] for s in psfflash_SHIFTS]
    for i in np.nonzero(npix >= psfflash_MIN_PIX)[0]:
        c = C[i][core_]
        r = psfflash_corr(c, m)
        rs = max(psfflash_corr(c, s) for s in shifted)
        after = [round(float((lc[j] - base[j]) / scat), 1) if j < len(lc) and scat > 0 else None for j in (i + 1, i + 2)]
        events.append(dict(cad=int(cad[i]), t=round(float(t[i]), 5), npix=int(npix[i]), r=round(r, 3), r_shift=round(rs, 3),
                           frac=round(float(c.sum()) / total, 6), after=after))
    return dict(q=quarter, ch=channel, n=int(len(t)), npix_ap=int(core_.sum()), n_hits=int(hit.sum()), n_multi=len(events),
                expect=round(expect, 3), scatter=round(float(scat / total), 7), events=events)


def psfflash_star(x):
    """One star's long-cadence target pixel files read in memory through psfflash_quarter; None when the listing or a file fails."""
    k9 = "%09d" % x["kepid"]
    keys = core.s3_keys("kepler/public/target_pixel_files/%s/%s/" % (k9[:4], k9), r"_lpd-targ\.fits\.gz$")
    if not keys:
        return None
    out = []
    for key in keys:
        b = core.get(core.S3 + key, timeout=300, retries=3, soft=True, pause=2)
        if not b:
            return None
        try:
            r = psfflash_quarter(b)
        except Exception:
            continue
        if r:
            out.append(r)
    return dict(kepid=x["kepid"], kepmag=x["kepmag"], quarters=out)


def psfflash_stream(procs=3):
    """Every target star through psfflash_star on procs processes, one line a star into ROOT/events.jsonl, past those read."""
    from concurrent.futures import ProcessPoolExecutor
    targets = json.load(open(os.path.join(psfflash_ROOT, "targets.json")))
    path = os.path.join(psfflash_ROOT, "events.jsonl")
    done = set()
    if os.path.exists(path):
        for ln in open(path, encoding="utf-8", errors="replace"):
            try:
                done.add(json.loads(ln)["kepid"])
            except (ValueError, KeyError):
                pass
    todo = [x for x in targets if x["kepid"] not in done]
    print("%d stars to read, %d read" % (len(todo), len(done)), flush=True)
    t0, m, miss = time.time(), 0, 0
    with ProcessPoolExecutor(procs) as ex, open(path, "a", encoding="utf-8") as fh:
        for r in ex.map(psfflash_star, todo, chunksize=1):
            if r is None:
                miss += 1
                continue
            fh.write(json.dumps(r, separators=(",", ":")) + "\n")
            m += 1
            if m % 10 == 0:
                fh.flush()
                print("%d of %d, %.1f stars a minute, %d to read again" % (m, len(todo), 60 * m / (time.time() - t0), miss), flush=True)
    print("%d stars read, %d to read again" % (m, miss), flush=True)
    return miss


def psfflash_report():
    """Multi-pixel corrections against the per-pixel expectation, PSF-shaped ones against the shifted image, shared cadences of a channel, and the flashes left with their after-light."""
    rows = [json.loads(ln) for ln in open(os.path.join(psfflash_ROOT, "events.jsonl"), encoding="utf-8")]
    nq = sum(len(r["quarters"]) for r in rows)
    multi = sum(qd["n_multi"] for r in rows for qd in r["quarters"])
    expect = sum(qd["expect"] for r in rows for qd in r["quarters"])
    ev = [dict(e, kepid=r["kepid"], kepmag=r["kepmag"], q=qd["q"], ch=qd["ch"]) for r in rows for qd in r["quarters"] for e in qd["events"]]
    shape = [e for e in ev if e["r"] >= psfflash_R and e["r"] - e["r_shift"] >= psfflash_MARGIN]
    null = [e for e in ev if e["r_shift"] >= psfflash_R and e["r_shift"] - e["r"] >= psfflash_MARGIN]
    by = {}
    for e in ev:
        by.setdefault((e["q"], e["ch"], e["cad"]), set()).add(e["kepid"])
    for e in shape:
        e["shared"] = len(by[(e["q"], e["ch"], e["cad"])]) - 1
    lone = [e for e in shape if e["shared"] == 0]
    print("%d stars, %d quarters; cadences with %d or more corrected aperture pixels %d against %.1f from the pixels' own rates; "
          "PSF-shaped %d, shaped like the shifted image instead %d; PSF-shaped and unshared on the channel %d"
          % (len(rows), nq, psfflash_MIN_PIX, multi, expect, len(shape), len(null), len(lone)))
    for e in sorted(lone, key=lambda e: -e["frac"])[:30]:
        print("  KIC %d (Kp %.1f) Q%d ch%d cadence %d BKJD %.4f: %d pixels, r %.3f against %.3f shifted, %.0f ppm of the star; "
              "next cadences %s sigma" % (e["kepid"], e["kepmag"], e["q"], e["ch"], e["cad"], e["t"], e["npix"], e["r"],
                                          e["r_shift"], e["frac"] * 1e6, e["after"]))
    json.dump(dict(n_stars=len(rows), n_quarters=nq, multi=multi, expect=expect, shaped=shape, null=len(null)),
              open(os.path.join(psfflash_ROOT, "report.json"), "w"))


def psfflash_teff(kepids):
    """DR25 stellar-table temperatures of the stars: the copy the Kepler levels search keeps, else the archive's table by id."""
    path = os.path.join(keplevels_ROOT, "targets.json")
    if os.path.exists(path):
        return {x["kepid"]: x["teff"] for x in json.load(open(path)) if x.get("teff")}
    teff = {}
    left = sorted(set(kepids))
    for i in range(0, len(left), 400):
        for r in core.exoplanets("SELECT kepid, teff FROM keplerstellar WHERE st_delivname = 'q1_q17_dr25_stellar' AND kepid IN (%s)"
                                 % ",".join(str(k) for k in left[i:i + 400])):
            if num(r["teff"]):
                teff[int(r["kepid"])] = num(r["teff"])
    return teff


def psfflash_record():
    """The PSF-shaped and shifted-image rates by stellar temperature, and each star's unshared PSF-shaped events judged as its flares or as cosmic rays shaped like its image."""
    rows = [json.loads(ln) for ln in open(os.path.join(psfflash_ROOT, "events.jsonl"), encoding="utf-8")]
    rep = json.load(open(os.path.join(psfflash_ROOT, "report.json")))
    teff = psfflash_teff([r["kepid"] for r in rows])
    nq = {r["kepid"]: len(r["quarters"]) for r in rows}
    ev = [dict(e, kepid=r["kepid"], kepmag=r["kepmag"], q=qd["q"], ch=qd["ch"]) for r in rows for qd in r["quarters"] for e in qd["events"]]
    shape = [e for e in ev if e["r"] >= psfflash_R and e["r"] - e["r_shift"] >= psfflash_MARGIN]
    null = [e for e in ev if e["r_shift"] >= psfflash_R and e["r_shift"] - e["r"] >= psfflash_MARGIN]
    ratios = []
    for lo, hi in psfflash_TEFF_BINS:
        ks = {k for k in nq if lo <= teff.get(k, -1) < hi}
        a = sum(1 for e in shape if e["kepid"] in ks)
        b = sum(1 for e in null if e["kepid"] in ks) / len(psfflash_SHIFTS)
        ratios.append("%d-%d K %.1f" % (lo, hi, a / b if b else float("nan")))
    by = {}
    for e in ev:
        by.setdefault((e["q"], e["ch"], e["cad"]), set()).add(e["kepid"])
    lone = [e for e in shape if len(by[(e["q"], e["ch"], e["cad"])]) == 1]
    per = {}
    for e in lone:
        per.setdefault(e["kepid"], []).append(e)
    out = []
    for k, es in per.items():
        t = teff.get(k)
        after = sum(1 for e in es if any(a is not None and a > psfflash_AFTER for a in e["after"]))
        flare = (t is not None and t < psfflash_FLARE_TEFF) or len(es) >= psfflash_FLARE_MIN or after > 0
        big = max(es, key=lambda e: e["frac"])
        out.append(dict(target="kic:%d" % k, stat=float(len(es)), unit="unshared PSF-shaped single-cadence corrections",
                        vet="flare" if flare else "cosmic-ray",
                        note="%s; %d of them with light in the next cadence; the largest %.0f ppm of the star, Q%d cadence %d; %s" % (
                            "Teff %d K" % t if t else "no Teff", after, big["frac"] * 1e6, big["q"], big["cad"],
                            "the star's flares" if flare else "a cosmic ray shaped like the star's image by chance, at the shifted-image rate")))
    n_flare = sum(1 for o in out if o["vet"] == "flare")
    core.record_search(
        "kepler_psf_flashes", "PSF-shaped flashes in pixel data",
        archive="Kepler DR25 long-cadence target pixel files (STScI AWS mirror)", release="DR25",
        selection="every KOI host with a candidate or confirmed planet: %d stars, %d quarters" % (rep["n_stars"], rep["n_quarters"]),
        code=os.path.abspath(__file__), entry="psfflash", n_targets=rep["n_stars"], n_unit="stars",
        outputs=[os.path.join(psfflash_ROOT, "events.jsonl"), os.path.join(psfflash_ROOT, "report.json")],
        params=dict(min_pixels=psfflash_MIN_PIX, r=psfflash_R, margin=psfflash_MARGIN, flare_teff=psfflash_FLARE_TEFF,
                    flare_min=psfflash_FLARE_MIN, after_sigma=psfflash_AFTER),
        null_summary="corrections shaped like the star's image shifted by one pixel, the most like of four shifts: %d, against %d PSF-shaped; "
                     "PSF-shaped over a quarter of the shifted-image count by temperature: %s" % (len(null), len(shape), ", ".join(ratios)),
        sensitivity="a brightening in one 29.4-minute cadence over three or more pixels of the optimal aperture that the pipeline removed as a cosmic ray",
        ledger="PSF-shaped flashes in pixel data",
        result="%d cadences with three or more corrected aperture pixels against %.0f from the pixels' own rates; %d PSF-shaped, %d of them "
               "unshared on the channel, on %d stars: %d stars' flares, %d at the chance rate" % (
                   rep["multi"], rep["expect"], len(shape), len(lone), len(out), n_flare, len(out) - n_flare))
    core.clear_targets("kepler_psf_flashes")
    core.record_targets("kepler_psf_flashes", out)
    print("recorded %d stars with unshared PSF-shaped events: %d flaring, %d at the chance rate; PSF-shaped over the null by temperature: %s"
          % (len(out), n_flare, len(out) - n_flare, ", ".join(ratios)))


def psfflash_cli():
    quiet_warnings()
    p = argparse.ArgumentParser()
    p.add_argument("mode", choices=["targets", "stream", "report", "record"])
    p.add_argument("--procs", type=int, default=3)
    a = p.parse_args()
    if a.mode == "targets":
        psfflash_targets()
    elif a.mode == "stream":
        sys.exit(1 if psfflash_stream(a.procs) else 0)
    elif a.mode == "record":
        psfflash_record()
    else:
        psfflash_report()


etz_DOC = r"""Earth-transit-zone stars at opposition in TESS sectors 42-46: flashes within 6.5 h of opposition against the rest and off-ecliptic controls.

    python -I seti.py flashes etz [--threads 8]
    python -I seti.py flashes etz record"""

etz_ROOT = os.environ.get("ETZ_ROOT", core.data("etz"))
etz_SECTORS = tuple(int(x) for x in os.environ.get("ETZ_SECTORS", "42,43,44,45,46").split(","))
etz_RUN = "tess_etz_opposition" + ("" if etz_SECTORS == (42, 43, 44, 45, 46) else "_%d_%d" % (etz_SECTORS[0], etz_SECTORS[-1]))


def etz_sun_lon(jd):
    T = (jd - 2451545.0) / 36525
    L = (280.46646 + 36000.76983 * T) % 360
    M = math.radians((357.52911 + 35999.05029 * T) % 360)
    return (L + (1.914602 - 0.004817 * T) * math.sin(M) + 0.019993 * math.sin(2 * M)) % 360


def etz_opposition(lon, t0, t1):
    """BTJD in [t0, t1] when the Sun's longitude is the star's plus 180, or None."""
    ts = np.arange(t0, t1, 0.01)
    d = np.array([((etz_sun_lon(t + 2457000.0) - (lon + 180.0)) + 180) % 360 - 180 for t in ts])
    k = np.nonzero((d[:-1] <= 0) & (d[1:] > 0))[0]
    if not len(k):
        return None
    i = k[0]
    return float(ts[i] + 0.01 * (-d[i]) / (d[i + 1] - d[i]))


def etz_tasks():
    from scipy.spatial import cKDTree
    os.makedirs(etz_ROOT, exist_ok=True)
    gp = os.path.join(etz_ROOT, "gaia100pc_ecl5.json")
    if not os.path.exists(gp):
        from astroquery.gaia import Gaia
        Gaia.ROW_LIMIT = -1
        t = Gaia.launch_job_async("SELECT source_id, ra, dec, ecl_lon, ecl_lat, parallax, pmra, pmdec, phot_g_mean_mag FROM "
                                  "gaiadr3.gaia_source WHERE parallax > 10 AND parallax_over_error > 10 AND ABS(ecl_lat) < 5 "
                                  "AND phot_g_mean_mag < 15").get_results()
        json.dump([[str(r["source_id"]), float(r["ra"]), float(r["dec"]), float(r["ecl_lon"]), float(r["ecl_lat"]),
                    float(r["parallax"]), num(r["pmra"]) or 0.0, num(r["pmdec"]) or 0.0, float(r["phot_g_mean_mag"])] for r in t],
                  open(gp, "w"))
    g = json.load(open(gp))
    ra = np.array([x[1] for x in g])
    de = np.array([x[2] for x in g])
    ra0 = ra - np.array([x[6] for x in g]) / 3.6e6 * 16 / np.cos(np.radians(de))  # back to the TIC epoch 2000
    de0 = de - np.array([x[7] for x in g]) / 3.6e6 * 16

    def xyz(r, d):
        r, d = np.radians(r), np.radians(d)
        return np.vstack([np.cos(d) * np.cos(r), np.cos(d) * np.sin(r), np.sin(d)]).T
    tree = cKDTree(xyz(ra0, de0))
    out = []
    for s in etz_SECTORS:
        tl = core.get("https://tess.mit.edu/public/target_lists/2m/all_targets_S%03d_v1.csv" % s, timeout=300, soft=True)
        if tl is None:
            tl = core.get("https://tess.mit.edu/wp-content/uploads/all_targets_S%03d_v1.csv" % s, timeout=300)
        rows = list(csv.reader([ln for ln in (tl.decode("latin-1") if isinstance(tl, bytes) else tl).splitlines()
                                if ln and not ln.startswith("#")]))
        ix = {h.strip(): i for i, h in enumerate(rows[0])}
        sc = core.get("https://archive.stsci.edu/missions/tess/download_scripts/sector/tesscurl_sector_%d_lc.sh" % s, timeout=600)
        urls = {int(m.group(2)): m.group(1) for m in re.finditer(r"(https://\S+-s%04d-(\d{16})-\d{4}-s_lc\.fits)" % s,
                                                                 sc.decode() if isinstance(sc, bytes) else sc)}
        tic = np.array([int(r[ix["TICID"]]) for r in rows[1:]])
        d, i = tree.query(xyz(np.array([float(r[ix["RA"]]) for r in rows[1:]]), np.array([float(r[ix["Dec"]]) for r in rows[1:]])),
                          distance_upper_bound=np.radians(20 / 3600.0))
        for k in np.nonzero(np.isfinite(d))[0]:
            x = g[i[k]]
            if int(tic[k]) in urls and (abs(x[4]) < 0.27 or 1 < abs(x[4]) < 5):
                out.append(dict(sector=s, tic=int(tic[k]), gaia=x[0], ecl_lon=x[3], ecl_lat=x[4], g=x[8],
                                url=urls[int(tic[k])], etz=abs(x[4]) < 0.27))
    import random
    random.seed(4)
    ctl = [o for o in out if not o["etz"]]
    return [o for o in out if o["etz"]] + random.sample(ctl, min(1500, len(ctl)))


def etz_one(task):
    from astropy.io import fits
    from scipy.ndimage import median_filter
    p = os.path.join(etz_ROOT, "%d_%d.json" % (task["sector"], task["tic"]))
    if os.path.exists(p):
        return json.load(open(p))
    try:
        b = core.get(task["url"], timeout=300)
        with fits.open(io.BytesIO(b)) as h:
            d = h[1].data
            t, f, q = np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["QUALITY"], int)
    except Exception as e:
        return dict(task, error=str(e)[:100])
    good = (q == 0) & np.isfinite(f) & np.isfinite(t)
    t, f = t[good], f[good]
    if len(t) < 2000:
        return dict(task, error="short")
    r = f / median_filter(f, size=61, mode="nearest") - 1.0
    sig = 1.4826 * np.median(np.abs(r - np.median(r)))
    lone = (r > 5 * sig) & (np.r_[r[1:], 0] < 2 * sig) & (np.r_[0, r[:-1]] < 2 * sig)
    multi = (r > 5 * sig) & ~lone
    to = etz_opposition(task["ecl_lon"], t.min(), t.max())
    win = np.abs(t - to) < 6.5 / 24 if to is not None else np.zeros(len(t), bool)
    res = dict(task, t_opp=to, n=int(len(t)), n_win=int(win.sum()), sig=float(sig), lone=int(lone.sum()),
               lone_win=int((lone & win).sum()), multi=int(multi.sum()), multi_win=int((multi & win).sum()),
               max_win=float(np.max(r[win]) / sig) if win.any() else None, lone_times=[float(x) for x in t[lone & win]])
    json.dump(res, open(p, "w"))
    return res


def etz_summary(res):
    out = {}
    for lab, sel in (("etz", True), ("control", False)):
        rs = [r for r in res if "error" not in r and r["etz"] == sel and r["t_opp"] is not None and r["n_win"] > 100]
        out[lab] = dict(windows=len(rs), stars=len({r["gaia"] for r in rs}), lone_win=sum(r["lone_win"] for r in rs),
                        n_win=sum(r["n_win"] for r in rs), lone_out=sum(r["lone"] - r["lone_win"] for r in rs),
                        n_out=sum(r["n"] - r["n_win"] for r in rs), multi_win=sum(r["multi_win"] for r in rs),
                        multi_out=sum(r["multi"] - r["multi_win"] for r in rs), rows=rs)
    return out


def etz_main(threads):
    tasks = etz_tasks()
    print("%d light curves: %d transit-zone, %d control" % (len(tasks), sum(t["etz"] for t in tasks), sum(not t["etz"] for t in tasks)),
          flush=True)
    with ThreadPoolExecutor(threads) as ex:
        res = list(ex.map(etz_one, tasks))
    s = etz_summary(res)
    for lab in ("etz", "control"):
        x = s[lab]
        print("%s: %d windows on %d stars; lone flashes %d in %d window cadences against %d in %d outside (%.2f expected); "
              "brightenings %d in the windows against %.2f expected"
              % (lab, x["windows"], x["stars"], x["lone_win"], x["n_win"], x["lone_out"], x["n_out"],
                 x["lone_out"] / max(x["n_out"], 1) * x["n_win"], x["multi_win"], x["multi_out"] / max(x["n_out"], 1) * x["n_win"]))
    json.dump(res, open(core.results("etz_opposition.json"), "w"))


def etz_record():
    res = json.load(open(core.results("etz_opposition.json")))
    s = etz_summary(res)
    e, c = s["etz"], s["control"]
    exp_l = e["lone_out"] / max(e["n_out"], 1) * e["n_win"]
    exp_m = e["multi_out"] / max(e["n_out"], 1) * e["n_win"]
    core.record_search(
        etz_RUN, "Earth-transit-zone stars at opposition", archive="MAST TESS SPOC 2-minute light curves",
        release="sectors %d-%d" % (etz_SECTORS[0], etz_SECTORS[-1]), selection="Gaia DR3 stars within 100 pc and 0.27 deg of the ecliptic, G below 15, with "
                                           "2-minute data and their opposition inside the sector",
        code=os.path.abspath(__file__), entry="etz", n_targets=e["stars"], n_unit="stars",
        params=dict(window_h=6.5, lone_sigma=5, median_cadences=61, etz_deg=0.27, control_deg=[1, 5]),
        null_summary="the same stars outside the window (%.2f lone flashes and %.2f brightenings expected in it); control stars "
                     "1-5 deg off the ecliptic: %d lone flashes in %d window cadences" % (exp_l, exp_m, c["lone_win"], c["n_win"]),
        sensitivity="a one-cadence flash at 5 sigma of each light curve's 2-minute scatter",
        ledger="Earth-transit-zone stars at opposition",
        result="%d transit windows on %d stars: %d lone flashes against %.2f expected and %d brightenings against %.2f"
               % (e["windows"], e["stars"], e["lone_win"], exp_l, e["multi_win"], exp_m))
    core.clear_targets(etz_RUN)
    core.record_targets(etz_RUN, [dict(
        target="gaia_dr3:%s" % r["gaia"], stat=float(r["lone_win"]), unit="lone flashes within 6.5 h of opposition",
        note="sector %d, TIC %d, opposition BTJD %.3f, %d window cadences, peak %.1f sigma" % (r["sector"], r["tic"], r["t_opp"],
                                                                                            r["n_win"], r["max_win"] or 0))
        for r in e["rows"]])
    print("recorded")


def etz_cli():
    quiet_warnings()
    if len(sys.argv) > 1 and sys.argv[1] == "record":
        etz_record()
        return
    p = argparse.ArgumentParser()
    p.add_argument("--threads", type=int, default=8)
    a = p.parse_args()
    etz_main(a.threads)


nova_DOC = r"""Nova-ellipsoid timing: TESS flashes of stars within 100 pc at their arrival dates keyed to T CrB 1866 and 1946 (or another eruption), against the rest of their light curves.

    python -I seti.py flashes nova [--threads 8]
    python -I seti.py flashes nova record"""

nova_ROOT = os.environ.get("NOVA_ROOT", core.data("novae"))
# name -> (RA, Dec deg, parallax mas, [(shell, epoch yr)])
nova_EVENTS = {
    "tcrb": (239.87565, 25.92026, 1.091953, [("1946", 1946.108), ("1866", 1866.363)]),        # Gaia DR3 1220110705972528512
    "gkper": (52.80000, 43.90417, 2.26, [("gkper_1901", 1901.140)]),
    "v603aql": (282.22750, 0.58417, 3.17, [("v603aql_1918", 1918.433)]),
    "rrpic": (98.90042, -62.64000, 2.0, [("rrpic_1925", 1925.395)]),
    "dqher": (271.87625, 45.85917, 2.0, [("dqher_1934", 1934.945)]),
    "cppup": (122.94208, -35.35139, 1.27, [("cppup_1942", 1942.852)]),
    "v1500cyg": (317.90250, 48.15056, 0.67, [("v1500cyg_1975", 1975.658)]),
    "sn1572": (6.33958, 64.14083, 0.4, [("sn1572", 1572.877)]),
    "sn1604": (262.64992, -21.48228, 0.2, [("sn1604", 1604.773)]),
}
nova_EVENT = os.environ.get("NOVA_EVENT", "tcrb")
nova_TCRB = nova_EVENTS[nova_EVENT][:3]
nova_ERUPTIONS = tuple(nova_EVENTS[nova_EVENT][3])
nova_TAG = "" if nova_EVENT == "tcrb" else "_" + nova_EVENT


def nova_gaia():
    """Gaia DR3 stars within 100 pc, G below 13, parallax at 20 sigma, cached."""
    p = os.path.join(nova_ROOT, "gaia_100pc_g13.json")
    if os.path.exists(p):
        return json.load(open(p))
    rows = []
    for lo, hi in ((10, 11.5), (11.5, 13), (13, 15), (15, 18), (18, 25), (25, 1000)):
        rows += core.datalab("SELECT source_id, ra, dec, parallax, parallax_error, phot_g_mean_mag FROM gaia_dr3.gaia_source "
                                 "WHERE parallax >= %g AND parallax < %g AND parallax_over_error > 20 AND phot_g_mean_mag < 13" % (lo, hi))
    out = [[str(r["source_id"]), float(r["ra"]), float(r["dec"]), float(r["parallax"]), float(r["parallax_error"]),
            float(r["phot_g_mean_mag"])] for r in rows]
    os.makedirs(nova_ROOT, exist_ok=True)
    json.dump(out, open(p, "w"))
    return out


def nova_targets():
    p = os.path.join(nova_ROOT, "nova_targets%s.json" % nova_TAG)
    if os.path.exists(p):
        return json.load(open(p))
    from astropy.table import Table
    from astroquery.xmatch import XMatch
    import astropy.units as u
    G = nova_gaia()
    sid = np.array([int(x[0]) for x in G], dtype=np.int64)
    ra, de = np.radians(np.array([x[1] for x in G])), np.radians(np.array([x[2] for x in G]))
    gmag = np.array([x[5] for x in G])
    plx, plx_err = np.array([x[3] for x in G]), np.array([x[4] for x in G])
    r = 1000 / plx
    sr = r * plx_err / plx

    def uv(a, d):
        return np.array([np.cos(d) * np.cos(a), np.cos(d) * np.sin(a), np.sin(d)])
    dn = 1000 / nova_TCRB[2]
    un = uv(math.radians(nova_TCRB[0]), math.radians(nova_TCRB[1]))
    xs = (uv(ra, de) * r).T
    D = np.linalg.norm(xs - dn * un, axis=1) + r - dn
    cos = (xs @ un) / r
    rows = []
    for name, t0 in nova_ERUPTIONS:
        arr = t0 + D * 3.26156
        sig = np.abs(1 - cos) * sr * 3.26156
        for k in np.nonzero((arr > 2018.55) & (arr < 2026.5))[0]:
            rows.append((name, str(sid[k]), float(np.degrees(ra[k])), float(np.degrees(de[k])), float(gmag[k]),
                         float(arr[k]), float(sig[k])))
    if not rows:
        os.makedirs(nova_ROOT, exist_ok=True)
        json.dump([], open(p, "w"))
        return []
    t = Table(rows=[(i, x[2], x[3]) for i, x in enumerate(rows)], names=("idx", "ra", "dec"))
    m = XMatch.query(cat1=t, cat2="vizier:IV/39/tic82", max_distance=6 * u.arcsec, colRA1="ra", colDec1="dec")
    best = {}
    for x in m:
        i = int(x["idx"])
        dg = abs(float(x["Gmag"]) - rows[i][4]) if not np.ma.is_masked(x["Gmag"]) else 0.5
        if i not in best or (dg, float(x["angDist"])) < best[i][0]:
            best[i] = ((dg, float(x["angDist"])), int(x["TIC"]))
    out = [dict(shell=x[0], gaia=x[1], ra=x[2], dec=x[3], g=x[4], arrival=x[5], sig=x[6], tic=best[i][1])
           for i, x in enumerate(rows) if i in best and best[i][0][0] < 1.0]
    os.makedirs(nova_ROOT, exist_ok=True)
    json.dump(out, open(p, "w"))
    return out


def nova_year(stamp):
    import datetime
    d = datetime.datetime.strptime(stamp[:7], "%Y%j") + datetime.timedelta(hours=int(stamp[7:9]), minutes=int(stamp[9:11]))
    y0 = datetime.datetime(d.year, 1, 1)
    return d.year + (d - y0).total_seconds() / (datetime.datetime(d.year + 1, 1, 1) - y0).total_seconds()


def nova_one(t):
    from astropy.io import fits
    from scipy.ndimage import median_filter
    p = os.path.join(nova_ROOT, "lc", "%s_%d_%d.json" % (t["shell"], t["sector"], t["tic"]))
    if os.path.exists(p):
        return json.load(open(p))
    try:
        with fits.open(io.BytesIO(core.get(t["url"], timeout=300))) as h:
            d = h[1].data
            tt, f, q = np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["QUALITY"], int)
    except Exception as e:
        return dict(t, error=str(e)[:80])
    ok = (q == 0) & np.isfinite(f) & np.isfinite(tt)
    tt, f = tt[ok], f[ok]
    if len(tt) < 2000:
        return dict(t, error="short")
    r = f / median_filter(f, size=61, mode="nearest") - 1.0
    sig = 1.4826 * np.median(np.abs(r - np.median(r)))
    lone = (r > 5 * sig) & (np.r_[r[1:], 0] < 2 * sig) & (np.r_[0, r[:-1]] < 2 * sig)
    yr = 2014.0 + (tt + 2457000.0 - 2456658.5) / 365.25
    win = np.abs(yr - t["arrival"]) <= 2 * t["sig"]
    res = dict(t, n=int(len(tt)), n_win=int(win.sum()), lone=int(lone.sum()), lone_win=int((lone & win).sum()),
               lone_times=[float(x) for x in tt[lone & win]], max_win=float(np.max(r[win]) / sig) if win.any() else None)
    os.makedirs(os.path.dirname(p), exist_ok=True)
    json.dump(res, open(p, "w"))
    return res


def nova_main(threads):
    targets = nova_targets()
    urls, start = {}, {}
    for s in range(1, 100):
        p = os.path.join(nova_ROOT, "curl_%03d.sh" % s)
        if not os.path.exists(p):
            try:
                b = core.get("https://archive.stsci.edu/missions/tess/download_scripts/sector/tesscurl_sector_%d_lc.sh" % s, timeout=600)
            except Exception:
                continue
            open(p, "wb").write(b if isinstance(b, bytes) else b.encode("utf-8"))
        txt = open(p, encoding="utf-8", errors="replace").read()
        for m in re.finditer(r"(https://\S+/(tess(\d{13})-s%04d-(\d{16})-\d{4}-s_lc\.fits))" % s, txt):
            urls[(s, int(m.group(4)))] = m.group(1)
            start.setdefault(s, nova_year(m.group(3)))
    sect = sorted(start)
    end = {s: (start[sect[i + 1]] if i + 1 < len(sect) else start[s] + 27.4 / 365.25) for i, s in enumerate(sect)}
    tasks = []
    for t in targets:
        for s in sect:
            if (s, t["tic"]) in urls:
                tasks.append(dict(t, sector=s, url=urls[(s, t["tic"])], t0=start[s], t1=end[s],
                                  in_window=bool(start[s] - 2 * t["sig"] <= t["arrival"] <= end[s] + 2 * t["sig"])))
    keep = {(t["shell"], t["tic"]) for t in tasks if t["in_window"]}
    tasks = [t for t in tasks if (t["shell"], t["tic"]) in keep]
    print("%d sectors; %d star-sectors of %d stars with a sector over their arrival" % (len(sect), len(tasks), len(keep)), flush=True)
    with ThreadPoolExecutor(threads) as ex:
        res = [r for r in ex.map(nova_one, tasks) if "error" not in r]
    json.dump(res, open(core.results("nova_ellipsoid%s.json" % nova_TAG), "w"))
    for name, _ in nova_ERUPTIONS:
        s = nova_summary(res, name)
        print("%s eruption: %d stars (G 6 or fainter) observed at arrival in %d light curves; capped lone flashes %d against %.1f "
              "from the same sectors; light curves with a flash %d against %.1f" % (name, s["stars"], s["lcs"], s["lone_win"],
                                                                                   s["expected"], s["any_obs"], s["any_exp"]))


def nova_summary(res, name, gmin=6.0, cap=3):
    """Window flashes against the out-of-window rate; G below 6 excluded, counts capped at 3 per light curve."""
    import collections
    rs = [r for r in res if r["shell"] == name]
    raw = dict(n=sum(r["lone_win"] for r in rs))
    rs = [r for r in rs if r["g"] >= gmin]
    n_s, l_s = collections.Counter(), collections.Counter()
    for r in rs:
        n_s[r["sector"]] += r["n"] - r["n_win"]
        l_s[r["sector"]] += min(r["lone"] - r["lone_win"], cap * max(1, round((r["n"] - r["n_win"]) / 15000)))
    win = [r for r in rs if r["n_win"] > 0]
    rate = {s_: (l_s[s_] / n_s[s_] if n_s[s_] else 0.0) for s_ in n_s}
    obs = sum(min(r["lone_win"], cap) for r in win)
    exp = sum(r["n_win"] * rate.get(r["sector"], 0.0) for r in win)
    any_obs = sum(1 for r in win if r["lone_win"] > 0)
    any_exp = sum(1 - math.exp(-r["n_win"] * rate.get(r["sector"], 0.0)) for r in win)
    return dict(stars=len({r["tic"] for r in win}), lcs=len(win), lone_win=obs, expected=exp, any_obs=any_obs, any_exp=any_exp,
                raw=raw["n"], rows=win)


def nova_record():
    res = json.load(open(core.results("nova_ellipsoid%s.json" % nova_TAG)))
    S = [(name, nova_summary(res, name)) for name, _ in nova_ERUPTIONS]
    run = "tcrb_nova_ellipsoid" if nova_EVENT == "tcrb" else "nova_ellipsoid_" + nova_EVENT
    core.record_search(
        run, "Nova ellipsoids", archive="MAST TESS SPOC 2-minute light curves",
        release="sectors 1-99", selection="Gaia DR3 stars within 100 pc, G below 13, on the %s ellipsoid during the TESS "
                                          "years, with 2-minute data over their arrival" % " or ".join(n for n, _ in nova_ERUPTIONS),
        code=os.path.abspath(__file__), entry="nova", n_targets=sum(s["stars"] for _, s in S),
        params=dict(window_sigma=2, lone_sigma=5, event=nova_EVENT, ra=nova_TCRB[0], dec=nova_TCRB[1], parallax_mas=nova_TCRB[2],
                    eruptions=[e for _, e in nova_ERUPTIONS]),
        null_summary="the same sectors' light curves outside their windows give the rate; stars brighter than G 6 are left out "
                     "and each light curve's count is capped at 3; with them, the raw window counts are %s"
                     % ", ".join("%d" % s["raw"] for _, s in S),
        sensitivity="a one-cadence flash at 5 sigma of each light curve's 2-minute scatter within the arrival window",
        ledger="Nova ellipsoids",
        result="; ".join("%s: %d stars at arrival, %d capped lone flashes against %.1f and %d light curves with a flash against %.1f"
                         % (name, s["stars"], s["lone_win"], s["expected"], s["any_obs"], s["any_exp"]) for name, s in S))
    core.clear_targets(run)
    core.record_targets(run, [dict(
        target="gaia_dr3:%s" % r["gaia"], stat=float(r["lone_win"]), unit="lone flashes within 2 sigma of the arrival",
        note="%s, arrival %.3f +- %.3f yr, sector %d, %d window cadences" % (r["shell"], r["arrival"], r["sig"], r["sector"],
                                                                           r["n_win"]))
        for _, s in S for r in s["rows"] if r["n_win"] > 0])
    print("recorded %s" % run)


def nova_cli():
    quiet_warnings()
    if len(sys.argv) > 1 and sys.argv[1] == "record":
        nova_record()
        return
    p = argparse.ArgumentParser()
    p.add_argument("--threads", type=int, default=8)
    a = p.parse_args()
    nova_main(a.threads)


clocks_DOC = r"""Shared clocks: ZTF periodic variables pairing in period within a fractional tolerance (counted as well at a tenth and three tenths of it) and in phase at a reference epoch, against the phases shuffled among the stars.

    python -I seti.py flashes clocks [record] [--eps 1e-5] [--phase 0.01] [--tref 58700] [--min-sep-arcsec 10] [--shuffles 20]"""


def clocks_main(record=False, eps=1e-5, phase=0.01, tref=58700.0, min_sep_arcsec=10.0, shuffles=20):
    from astropy.table import Table
    p = core.data("ztf_var", "ztf_periodic.fits")
    t = Table.read(p)
    P = np.array(np.ma.filled(t["Per"].astype(float), np.nan))
    T0 = np.array(np.ma.filled(t["T0"].astype(float), np.nan))
    ra = np.radians(np.array(t["RAJ2000"], float))
    de = np.radians(np.array(t["DEJ2000"], float))
    ok = np.isfinite(P) & np.isfinite(T0) & (P > 0)
    alias = np.zeros(len(P), bool)
    for base in (1.0, 0.99726957, 29.530589, 365.25):
        for n in range(1, 11):
            for k in range(1, 5):
                alias |= np.abs(P / (base * n / k) - 1) < 1e-3
    keep = np.nonzero(ok & ~alias)[0]
    lp = np.log(P[keep])
    o = np.argsort(lp)
    lp, idx = lp[o], keep[o]
    xyz = np.vstack([np.cos(de) * np.cos(ra), np.cos(de) * np.sin(ra), np.sin(de)]).T

    def pairs(eps):
        out = []
        j = 0
        for i in range(len(lp)):
            j = max(j, i + 1)
            while j < len(lp) and lp[j] - lp[i] < eps:
                j += 1
            for k in range(i + 1, j):
                out.append((idx[i], idx[k]))
        return out
    counts = {}
    sep10 = 2 * math.sin(math.radians(min_sep_arcsec / 3600.0) / 2)
    for tol in (eps / 10, eps * 0.3, eps):
        pr = pairs(tol)
        a = np.array(pr, dtype=np.int64).reshape(-1, 2)
        far = np.linalg.norm(xyz[a[:, 0]] - xyz[a[:, 1]], axis=1) > sep10 if len(a) else np.zeros(0, bool)
        counts[tol] = int(far.sum())
        if tol == eps:
            base = a[far]
    print("%d variables with periods off the aliases (of %d); pairs within the fractional period tolerance: %s"
          % (len(keep), len(P), ", ".join("%.0e: %d (%.3g per unit)" % (e, c, c / e) for e, c in counts.items())), flush=True)
    ph = ((tref - T0) / P) % 1.0

    def coherent(phases):
        d = np.abs(phases[base[:, 0]] - phases[base[:, 1]])
        d = np.minimum(d, 1 - d)
        return base[d < phase]

    def groups(cp):
        parent = {}

        def find(x):
            parent.setdefault(x, x)
            while parent[x] != x:
                parent[x] = parent[parent[x]]
                x = parent[x]
            return x
        for a_, b_ in cp:
            parent[find(int(a_))] = find(int(b_))
        g = {}
        for a_, b_ in cp:
            for x in (int(a_), int(b_)):
                g.setdefault(find(x), set()).add(x)
        return [sorted(v) for v in g.values()]
    cp = coherent(ph)
    gr = groups(cp)
    rng = np.random.default_rng(5)
    nulls = []
    for k in range(shuffles):
        sh = ph.copy()
        sh[keep] = rng.permutation(ph[keep])
        c2 = coherent(sh)
        g2 = groups(c2)
        nulls.append((len(c2), sum(1 for g in g2 if len(g) >= 3), max([len(g) for g in g2] or [0])))
    big = sorted([g for g in gr if len(g) >= 3], key=len, reverse=True)
    # an exactly repeated catalogue period is a grid, not a measurement
    from collections import Counter
    reps = Counter(np.round(P[keep], 6).tolist())
    grid = {round(float(P[g[0]]), 6) for g in big if reps[round(float(P[g[0]]), 6)] >= 10}
    print("in phase within %g at MJD %.0f: %d pairs, %d groups of 3 or more (largest %d); phases shuffled: pairs %s, groups %s, "
          "largest %s" % (phase, tref, len(cp), len(big), len(big[0]) if big else 0, [n[0] for n in nulls[:8]], [n[1] for n in nulls[:8]],
                          [n[2] for n in nulls[:8]]))
    for g in big[:10]:
        print("  group of %d at P %.6f d: %s" % (len(g), P[g[0]], ", ".join("%s (%s)" % (t["ID"][i], t["Type"][i]) for i in g[:6])))
    nongrid = [g for g in big if round(float(P[g[0]]), 6) not in grid]
    print("groups at a period the catalogue repeats exactly (its grid): %d; the largest of the rest %d, against the shuffled "
          "largest %d" % (len(big) - len(nongrid), len(nongrid[0]) if nongrid else 0, max(n[2] for n in nulls)))
    res = dict(n=len(keep), counts={str(k): v for k, v in counts.items()}, pairs=len(cp), groups=[[int(i) for i in g] for g in big],
               null_pairs=[n[0] for n in nulls], null_groups=[n[1] for n in nulls], null_largest=[n[2] for n in nulls])
    json.dump(res, open(core.results("shared_clocks.json"), "w"))
    if record:
        core.record_search(
            "ztf_shared_clocks", "Shared clocks", archive="ZTF periodic variables (Chen et al. 2020, J/ApJS/249/18)",
            selection="%d variables with periods off the n/k-day, sidereal-day, lunar and annual aliases" % len(keep),
            code=os.path.abspath(__file__), entry="clocks", n_targets=len(keep),
            params=dict(eps=eps, phase=phase, tref=tref, min_sep_arcsec=min_sep_arcsec, shuffles=shuffles),
            null_summary="phases shuffled among the stars, %d times: %.0f +- %.0f pairs in phase, groups of 3 or more %.1f, largest "
                         "%d" % (shuffles, np.mean([n[0] for n in nulls]), np.std([n[0] for n in nulls]), np.mean([n[1] for n in nulls]),
                                 max(n[2] for n in nulls)),
            sensitivity="three or more variables sharing a period to %g and a phase to %g cycle" % (eps, phase),
            ledger="Shared clocks",
            result="pairs within %s in period scale with the tolerance (%s per unit); in phase within %g: %d "
                   "pairs and %d groups of 3 or more, as shuffled phases give; groups above the shuffled largest (%d) sit at "
                   "periods the catalogue repeats exactly, its grid for long-period variables; the largest of the rest %d"
                   % (", ".join("%.3g" % e for e in counts), ", ".join("%.3g" % (c / e) for e, c in counts.items()), phase, len(cp), len(big),
                      max(n[2] for n in nulls), len(nongrid[0]) if nongrid else 0))
        core.clear_targets("ztf_shared_clocks")
        core.record_targets("ztf_shared_clocks", [dict(
            target="ztf:%s" % t["ID"][i], stat=float(len(g)), unit="variables sharing its period and phase",
            note="P %.6f d, %s" % (P[i], t["Type"][i])) for g in big for i in g])
        print("recorded")


superflare_DOC = r"""Superflare deficit among TESS planet hosts: fraction of flares above 1e34 erg against matched non-hosts, 2000 draws.

    python -I seti.py flashes superflare analyse
    python -I seti.py flashes superflare record"""

superflare_ROOT = os.environ.get("SUPERFLARE_ROOT", core.data("superflare"))
superflare_OUT = "superflare_deficit.json"
superflare_E1, superflare_E2, superflare_NMATCH = 1e33, 1e34, 10


def superflare_load():
    from astropy.table import Table
    os.makedirs(superflare_ROOT, exist_ok=True)
    hp = os.path.join(superflare_ROOT, "hosts.json")
    if not os.path.exists(hp):
        json.dump(dict(toi=core.exoplanets("SELECT tid, toi, tfopwg_disp, pl_orbper, pl_rade, st_teff, st_dist FROM toi"),
                       ps=core.exoplanets("SELECT tic_id, hostname, pl_name, pl_orbper, pl_rade, st_teff, sy_dist, discoverymethod FROM pscomppars")), open(hp, "w"))
    h = json.load(open(hp))
    hosts = {int(r["tid"]) for r in h["toi"] if r["tfopwg_disp"] in ("PC", "CP", "KP") and r["tid"]}
    hosts |= {int(str(r["tic_id"]).replace("TIC", "").strip()) for r in h["ps"] if r["tic_id"] and str(r["tic_id"]).replace("TIC", "").strip().isdigit()}
    for cat, tb in (("J/A+A/669/A15", "table1"), ("J/A+A/669/A15", "table2")):
        p = os.path.join(superflare_ROOT, "yang2023_%s.ecsv" % tb)
        if not os.path.exists(p):
            from astroquery.vizier import Vizier
            Vizier(row_limit=-1, columns=["**"], timeout=900).get_catalogs("%s/%s" % (cat, tb))[0].write(p, format="ascii.ecsv", overwrite=True)
    stars = {}
    for r in Table.read(os.path.join(superflare_ROOT, "yang2023_table2.ecsv"), format="ascii.ecsv"):
        stars[int(r["TIC"])] = dict(tic=int(r["TIC"]), teff=num(r["Teff"]), rad=num(r["Rad"]), dist=num(r["DistG"]) or num(r["DistT"]), nsec=num(r["NTESS"]),
                                    flare=str(r["Flare"]).strip().lower().startswith("y"), host=int(r["TIC"]) in hosts, E=[])
    for r in Table.read(os.path.join(superflare_ROOT, "yang2023_table1.ecsv"), format="ascii.ecsv"):
        tic, e = int(r["TIC"]), num(r["E"])
        if tic in stars and e and e > 0:
            stars[tic]["E"].append(e)
    return stars


def superflare_analyse():
    stars = superflare_load()
    flaring = [s for s in stars.values() if s["E"] and s["teff"] and s["rad"] and s["dist"]]
    hosts = [s for s in flaring if s["host"]]
    pool = [s for s in flaring if not s["host"]]
    print("%d stars, %d flaring with energies, %d of them planet hosts (of %d hosts in the sample); %d flaring non-hosts" % (
        len(stars), len(flaring), len(hosts), sum(1 for s in stars.values() if s["host"]), len(pool)))

    def key(s):
        return np.array([s["teff"] / 300.0, math.log10(s["rad"]) / 0.1, math.log10(s["dist"]) / 0.15, (s["nsec"] or 1) / 4.0])
    from scipy.spatial import cKDTree
    pk = np.array([key(s) for s in pool])
    tree = cKDTree(pk)
    matches = [tree.query(key(h), k=superflare_NMATCH)[1] for h in hosts]

    def fractions(group):
        e = np.concatenate([np.array(s["E"]) for s in group]) if group else np.array([])
        e = e[e >= superflare_E1]
        return (float(np.mean(e >= superflare_E2)) if len(e) else float("nan"), int(len(e)), float(np.mean([max(s["E"]) >= superflare_E2 for s in group])) if group else float("nan"))
    fh, nh, ph = fractions(hosts)
    rng = np.random.default_rng(4)
    draws, draws_star = [], []
    for _ in range(2000):
        ctrl = [pool[int(rng.choice(m))] for m in matches]
        f, _, p = fractions(ctrl)
        draws.append(f)
        draws_star.append(p)
    draws, draws_star = np.array(draws), np.array(draws_star)
    from scipy.stats import ks_2samp
    eh = np.concatenate([np.array(s["E"]) for s in hosts])
    eh = np.log10(eh[eh >= superflare_E1])
    ctrl_all = [pool[int(i)] for m in matches for i in m]
    ec = np.concatenate([np.array(s["E"]) for s in ctrl_all])
    ec = np.log10(ec[ec >= superflare_E1])
    ks = ks_2samp(eh, ec)
    print("flares above %.0e erg among those above %.0e: hosts %.3f (%d flares) against matched non-hosts %.3f +- %.3f (p of a deficit %.3f); stars with a flare above %.0e: "
          "hosts %.3f against %.3f +- %.3f (p %.3f); energies above %.0e erg: KS statistic %.3f, p %.3f, medians %.2f and %.2f" % (
              superflare_E2, superflare_E1, fh, nh, draws.mean(), draws.std(), float(np.mean(draws <= fh)), superflare_E2, ph, draws_star.mean(), draws_star.std(),
              float(np.mean(draws_star <= ph)), superflare_E1, ks.statistic, ks.pvalue, float(np.median(eh)), float(np.median(ec))))
    # hosts with their largest flares
    for s in sorted(hosts, key=lambda s: -max(s["E"]))[:8]:
        print("  TIC %d: Teff %.0f K, R %.2f Rsun, %.0f pc, %d flares, largest %.1e erg" % (s["tic"], s["teff"], s["rad"], s["dist"], len(s["E"]), max(s["E"])))
    json.dump(dict(n_stars=len(stars), n_flaring=len(flaring), n_hosts=len(hosts), n_pool=len(pool), frac_hosts=fh, n_flares_hosts=nh, frac_ctrl=float(draws.mean()), sd_ctrl=float(draws.std()),
                   p_deficit=float(np.mean(draws <= fh)), star_hosts=ph, star_ctrl=float(draws_star.mean()), star_sd=float(draws_star.std()), p_star=float(np.mean(draws_star <= ph)),
                   ks=float(ks.statistic), ks_p=float(ks.pvalue), params=dict(e1=superflare_E1, e2=superflare_E2, nmatch=superflare_NMATCH),
                   hosts=[dict(tic=s["tic"], teff=s["teff"], rad=s["rad"], dist=s["dist"], n=len(s["E"]), emax=max(s["E"]), frac=float(np.mean(np.array(s["E"])[np.array(s["E"]) >= superflare_E1] >= superflare_E2)) if any(e >= superflare_E1 for e in s["E"]) else None) for s in hosts]),
              open(core.results(superflare_OUT), "w"), indent=0)


def superflare_record():
    d = json.load(open(core.results(superflare_OUT)))
    core.record_search(
        "superflare_deficit", "Superflare deficit among planet hosts", archive="Yang et al. (2023) TESS flare catalogue (J/A+A/669/A15); TESS objects of interest and confirmed planets (NASA Exoplanet Archive)",
        selection="flaring TESS two-minute targets with flare energies (%d), the %d planet hosts among them against %d flaring non-hosts" % (d["n_flaring"], d["n_hosts"], d["n_pool"]),
        code=os.path.abspath(__file__), entry="superflare", n_targets=d["n_hosts"], params=d["params"],
        null_summary="each host's %d nearest non-hosts in temperature, radius, distance and sectors, drawn 2000 times: fraction of flares above %.0e among those above %.0e %.3f +- %.3f, stars with a flare above %.0e %.3f +- %.3f" % (
            d["params"]["nmatch"], d["params"]["e2"], d["params"]["e1"], d["frac_ctrl"], d["sd_ctrl"], d["params"]["e2"], d["star_ctrl"], d["star_sd"]),
        sensitivity="a halving of the superflare fraction at about 3 sigma for %d hosts with %d flares above %.0e erg" % (d["n_hosts"], d["n_flares_hosts"], d["params"]["e1"]),
        ledger="Superflare deficit among planet hosts",
        result="hosts' fraction of flares above %.0e erg %.3f (%d flares) against %.3f +- %.3f matched (p of a deficit %.2f); stars with a superflare %.3f against %.3f +- %.3f (p %.2f); KS on the energies %.3f (p %.2f)" % (
            d["params"]["e2"], d["frac_hosts"], d["n_flares_hosts"], d["frac_ctrl"], d["sd_ctrl"], d["p_deficit"], d["star_hosts"], d["star_ctrl"], d["star_sd"], d["p_star"], d["ks"], d["ks_p"]))
    core.clear_targets("superflare_deficit")
    core.record_targets("superflare_deficit", [dict(target="tic:%d" % h["tic"], stat=h["frac"], unit="fraction of the star's flares above 1e34 erg among those above 1e33",
                                                         note="Teff %.0f K, R %.2f Rsun, %.0f pc, %d flares, largest %.1e erg" % (h["teff"], h["rad"], h["dist"], h["n"], h["emax"])) for h in d["hosts"]])
    print("recorded %d hosts" % d["n_hosts"])


def superflare_cli():
    mode = sys.argv[1] if len(sys.argv) > 1 else "analyse"
    {"analyse": superflare_analyse, "record": superflare_record}[mode]()


neliota_DOC = r"""NELIOTA lunar flashes: R-I colour temperature against impact range, duration against 0.5 s, and positional recurrence against permuted longitudes.

    python -I seti.py flashes neliota fetch
    python -I seti.py flashes neliota analyse
    python -I seti.py flashes neliota record"""

neliota_ROOT = os.environ.get("NELIOTA_ROOT", core.data("neliota"))
neliota_OUT = "neliota_flashes.json"
neliota_LAM_R, neliota_LAM_I = 0.64, 0.80  # Cousins R and I (um)
neliota_TMIN, neliota_TMAX, neliota_DURATION, neliota_PAIR = 1000.0, 8000.0, 0.5, 1.5
neliota_HEAD = dict(core.UA, **{"Accept": "*/*"})


def neliota_mag(s):
    """Magnitude m.mm(ee) as (value, error); (None, None) when empty."""
    m = re.match(r"\s*([\d.]+)\s*\((\d+)\)", s or "")
    if not m:
        return None, None
    v, e = m.group(1), m.group(2)
    dec = len(v.split(".")[1]) if "." in v else 0
    return float(v), float(e) * 10 ** (-dec)


def neliota_fetch():
    """NELIOTA flash tables (2020 PDF, 2024 HTML appendix): id, date, class, duration, R, I, latitude, longitude."""
    import fitz
    import html as htmlmod
    os.makedirs(neliota_ROOT, exist_ok=True)
    rows, seen = [], set()
    p = os.path.join(neliota_ROOT, "arxiv_1911.06101.pdf")
    if not os.path.exists(p):
        core.download("https://arxiv.org/pdf/1911.06101", p, headers=neliota_HEAD)
    text = re.sub(r"\s*\n\s*", " ", "".join(pg.get_text() for pg in fitz.open(p)))
    pat = re.compile(r"(?<![\d.])(\d{1,3}) (20\d\d \d\d \d\d) (\d\d:\d\d:\d\d\.\d+) (V|SC1|SC2) (\d+) ([\d.]+\(\d+\))(?: ([\d.]+\(\d+\)))? ([+−-]?\d+\.\d) ([+−-]?\d+\.\d)")
    for m in pat.finditer(text):
        fid, date, ut, val, dt, m1, m2, lat, lon = m.groups()
        if int(fid) in seen:
            continue
        seen.add(int(fid))
        if m2:
            (mR, eR), (mI, eI) = neliota_mag(m1), neliota_mag(m2)
        elif val == "SC2":
            (mR, eR), (mI, eI) = (None, None), neliota_mag(m1)
        else:
            (mR, eR), (mI, eI) = neliota_mag(m1), (None, None)
        rows.append(dict(id=int(fid), date=date + " " + ut, val=val, dur_ms=float(dt), mR=mR, eR=eR, mI=mI, eI=eI,
                         lat=float(lat.replace("−", "-")), lon=float(lon.replace("−", "-")), source="Liakos et al. 2020"))
    p = os.path.join(neliota_ROOT, "arxiv_2403.19613.html")
    if not os.path.exists(p):
        open(p, "wb").write(core.get("https://arxiv.org/html/2403.19613", headers=neliota_HEAD, timeout=120))
    page = open(p, encoding="utf-8", errors="replace").read()

    def cells(row):
        return [re.sub(r"\s+", " ", htmlmod.unescape(re.sub(r"<[^>]+>", " ", c))).strip() for c in re.findall(r"<t[dh][^>]*>(.*?)</t[dh]>", row, re.S)]

    def number(s):
        m = re.match(r"^[−-]?\s*[\d.]+", (s or "").replace("−", "-"))
        return float(m.group(0).replace(" ", "")) if m else None

    for tb in re.findall(r"<table.*?</table>", page, re.S):
        trs = re.findall(r"<tr[^>]*>(.*?)</tr>", tb, re.S)
        if not trs or "Date" not in trs[0] or "Lat" not in trs[0]:
            continue
        for tr in trs[1:]:
            c = cells(tr)
            for off in (0, 8):
                if len(c) >= off + 8 and re.match(r"^\d+$", c[off]) and int(c[off]) not in seen:
                    fid, date, val, dt, m1, m2, lat, lon = c[off:off + 8]
                    (mR, eR), (mI, eI) = neliota_mag(m1), neliota_mag(m2)
                    rows.append(dict(id=int(fid), date=date, val="V" if val.startswith("Val") else val, dur_ms=number(dt), mR=mR, eR=eR, mI=mI, eI=eI,
                                     lat=number(lat), lon=number(lon), source="Liakos et al. 2024"))
                    seen.add(int(fid))
    rows.sort(key=lambda r: r["id"])
    json.dump(rows, open(os.path.join(neliota_ROOT, "flashes.json"), "w"), indent=0)
    print("%d flashes, %d validated, %d with both bands, ids %d-%d" % (len(rows), sum(r["val"] == "V" for r in rows), sum(1 for r in rows if r["mR"] is not None and r["mI"] is not None),
                                                                       rows[0]["id"], rows[-1]["id"]))


def neliota_planck(lam_um, T):
    h, c, k = 6.626e-27, 2.998e10, 1.381e-16
    lam = lam_um * 1e-4
    return 2 * h * c ** 2 / lam ** 5 / math.expm1(h * c / (lam * k * T))


def neliota_temperature(ri):
    """Black-body temperature with R - I = ri (Vega as 9602 K); None outside 500-50000 K."""
    def colour(T):
        v = neliota_planck(neliota_LAM_R, 9602.0) / neliota_planck(neliota_LAM_I, 9602.0)
        return -2.5 * math.log10(neliota_planck(neliota_LAM_R, T) / neliota_planck(neliota_LAM_I, T) / v)
    lo, hi = 500.0, 50000.0
    if ri <= colour(hi) or ri >= colour(lo):
        return None
    for _ in range(60):
        mid = 0.5 * (lo + hi)
        if colour(mid) > ri:
            lo = mid
        else:
            hi = mid
    return 0.5 * (lo + hi)


def neliota_pairs(lat, lon, limit_deg):
    """Index pairs of flashes within limit_deg on the lunar sphere."""
    la, lo = np.radians(lat), np.radians(lon)
    cosd = np.sin(la)[:, None] * np.sin(la)[None, :] + np.cos(la)[:, None] * np.cos(la)[None, :] * np.cos(lo[:, None] - lo[None, :])
    ang = np.degrees(np.arccos(np.clip(cosd, -1, 1)))
    i, j = np.where(np.triu(ang < limit_deg, 1))
    return list(zip(i.tolist(), j.tolist(), ang[i, j].tolist()))


def neliota_analyse():
    rows = json.load(open(os.path.join(neliota_ROOT, "flashes.json")))
    for r in rows:
        r["T"] = r["T_lo3"] = r["T_hi3"] = None
        if r["mR"] is not None and r["mI"] is not None:
            ri = r["mR"] - r["mI"]
            e = math.hypot(r["eR"] or 0.3, r["eI"] or 0.3)
            r["T"], r["T_hi3"], r["T_lo3"] = neliota_temperature(ri), neliota_temperature(ri - 3 * e), neliota_temperature(ri + 3 * e)
        r["hot"] = r["T_lo3"] is not None and r["T_lo3"] > neliota_TMAX
        r["cold"] = r["T_hi3"] is not None and r["T_hi3"] < neliota_TMIN
        r["long"] = (r["dur_ms"] or 0) > neliota_DURATION * 1000
    withT = [r for r in rows if r["T"]]
    T = np.array([r["T"] for r in withT])
    pos = [r for r in rows if r["lat"] is not None and r["lon"] is not None]
    lat, lon = np.array([r["lat"] for r in pos]), np.array([r["lon"] for r in pos])
    pairs = neliota_pairs(lat, lon, neliota_PAIR)
    rng = np.random.default_rng(4)
    null = np.array([len(neliota_pairs(lat, rng.permutation(lon), neliota_PAIR)) for _ in range(2000)])
    p_pairs = float((np.sum(null >= len(pairs)) + 1) / (len(null) + 1))
    hot, cold, long_ = [r for r in rows if r["hot"]], [r for r in rows if r["cold"]], [r for r in rows if r["long"]]
    print("%d flashes (%d validated), %d with colour temperatures: median %.0f K, 5th-95th %.0f-%.0f K; %d hotter than %.0f K at 3 sigma, %d cooler than %.0f, %d longer than %.1f s (longest %.0f ms)" % (
        len(rows), sum(r["val"] == "V" for r in rows), len(withT), np.median(T), np.percentile(T, 5), np.percentile(T, 95), len(hot), neliota_TMAX, len(cold), neliota_TMIN, len(long_), neliota_DURATION,
        max(r["dur_ms"] or 0 for r in rows)))
    print("pairs within %.1f deg on the Moon: %d against %.1f +- %.1f with the longitudes permuted (p %.3f)" % (neliota_PAIR, len(pairs), null.mean(), null.std(), p_pairs))
    for i, j, ang in sorted(pairs, key=lambda x: x[2]):
        a, b = pos[i], pos[j]
        print("  flashes %d and %d: %.2f deg apart at (%+.1f, %+.1f), %s and %s, %s/%s, T %s/%s K" % (
            a["id"], b["id"], ang, a["lat"], a["lon"], a["date"][:10], b["date"][:10], a["val"], b["val"], "%.0f" % a["T"] if a["T"] else "-", "%.0f" % b["T"] if b["T"] else "-"))
    for r in hot + cold:
        print("  flash %d (%s, %s): R %s I %s, T %.0f K (3 sigma range %s-%s), %s ms" % (r["id"], r["date"][:10], r["val"], r["mR"], r["mI"], r["T"], "%.0f" % r["T_lo3"] if r["T_lo3"] else "-",
                                                                                   "%.0f" % r["T_hi3"] if r["T_hi3"] else "-", r["dur_ms"]))
    json.dump(dict(n=len(rows), n_T=len(withT), T_median=float(np.median(T)), hot=[r["id"] for r in hot], cold=[r["id"] for r in cold], long=[r["id"] for r in long_],
                   pairs=[(pos[i]["id"], pos[j]["id"], ang) for i, j, ang in pairs], pairs_null=float(null.mean()), pairs_null_sd=float(null.std()), p_pairs=p_pairs, rows=rows,
                   params=dict(tmin=neliota_TMIN, tmax=neliota_TMAX, duration_s=neliota_DURATION, pair_deg=neliota_PAIR)), open(core.results(neliota_OUT), "w"), indent=0)


def neliota_record():
    d = json.load(open(core.results(neliota_OUT)))
    core.record_search(
        "neliota_flashes", "Non-thermal or recurrent lunar impact flashes", archive="NELIOTA flash lists of Liakos et al. (2020, 2024), from the papers' tables", release="2017-2023",
        selection="%d flashes, %d with R and I" % (d["n"], d["n_T"]), code=os.path.abspath(__file__), entry="neliota", n_targets=d["n"], params=d["params"],
        null_summary="pairs within %.1f deg against %.1f +- %.1f with the longitudes permuted among the flashes 2000 times" % (d["params"]["pair_deg"], d["pairs_null"], d["pairs_null_sd"]),
        sensitivity="a colour temperature outside %.0f-%.0f K at 3 sigma, a duration above %.1f s, or two flashes within %.1f deg" % (d["params"]["tmin"], d["params"]["tmax"], d["params"]["duration_s"], d["params"]["pair_deg"]),
        ledger="Non-thermal or recurrent lunar impact flashes",
        result="colour temperatures median %.0f K; %d hotter than the range at 3 sigma, %d cooler, %d longer than %.1f s; %d pairs within %.1f deg against %.1f (p %.2f)" % (
            d["T_median"], len(d["hot"]), len(d["cold"]), len(d["long"]), d["params"]["duration_s"], len(d["pairs"]), d["params"]["pair_deg"], d["pairs_null"], d["p_pairs"]))
    core.clear_targets("neliota_flashes")
    core.record_targets("neliota_flashes", [dict(target="neliota:%d" % r["id"], stat=r["T"], unit="colour temperature from R - I, K",
                                                     note="%s, %s, %s ms, (%s, %s)" % (r["date"][:10], r["val"], r["dur_ms"], r["lat"], r["lon"])) for r in d["rows"]])
    print("recorded %d flashes" % d["n"])


def neliota_cli():
    quiet_warnings()
    mode = sys.argv[1] if len(sys.argv) > 1 else "analyse"
    {"fetch": neliota_fetch, "analyse": neliota_analyse, "record": neliota_record}[mode]()


# --------------------------------------------------------------------------------------------------------------
# flares together and alike, and stars steadier than their photons

YANG_FLARES = '"J/A+A/669/A15/table1"'
YANG_STARS = '"J/A+A/669/A15/table2"'


flaresync_DOC = r"""Flares firing together across the sky: the TESS flares of Yang et al. (2023) (VizieR J/A+A/669/A15), pairs on stars more than 10 deg apart whose peak times agree to 1 minute, against the pair rate at separations of 10-60 minutes scaled to the 2-minute window.

    python -I seti.py flashes flaresync [record] [--dt-min 1] [--sep-deg 10] [--control-min 10] [--control-max 60]"""


def flaresync_main(record=False, dt_min=1.0, sep_deg=10.0, control_min=10.0, control_max=60.0):
    fl = core.tap("vizier", "SELECT TIC, TimeP, E FROM %s" % YANG_FLARES, timeout=900)
    st = {str(r["TIC"]): (num(r["RAJ2000"]), num(r["DEJ2000"])) for r in core.tap("vizier", "SELECT TIC, RAJ2000, DEJ2000 FROM %s" % YANG_STARS, timeout=900)}
    fl = sorted([r for r in fl if num(r["TimeP"]) is not None and str(r["TIC"]) in st], key=lambda r: num(r["TimeP"]))
    t = np.array([num(r["TimeP"]) for r in fl])
    near, wide, zero, hits = 0, 0, 0, []
    m1, m10, m60 = dt_min / 1440.0, control_min / 1440.0, control_max / 1440.0
    for i in range(len(fl)):
        j = i + 1
        while j < len(fl) and t[j] - t[i] < m60:
            a, b = fl[i], fl[j]
            if str(a["TIC"]) != str(b["TIC"]) and core.sep_arcsec(*st[str(a["TIC"])], *st[str(b["TIC"])]) > sep_deg * 3600:
                dt = t[j] - t[i]
                if dt < m1:
                    near += 1
                    zero += dt * 86400 < 1
                    hits.append((a["TIC"], b["TIC"], dt * 86400))
                elif dt >= m10:
                    wide += 1
            j += 1
    expect = wide * dt_min / (control_max - control_min)
    print("%d flares on %d stars; pairs more than %g deg apart within %g min: %d (%d of them at one timestamp, the 2-minute cadence all stars "
          "share) against %.1f from the rate at %g-%g min" % (len(fl), len(st), sep_deg, dt_min, near, zero, expect, control_min, control_max))
    if record:
        core.record_run(
            "tess_flare_synchrony", "Flares firing together across the sky", "Yang et al. (2023) TESS flare catalogue (VizieR J/A+A/669/A15)",
            "flares with peak times on catalogued stars: %d" % len(fl), os.path.abspath(__file__), "flaresync", len(fl), "flares",
            dict(dt_min=dt_min, sep_deg=sep_deg, control_min=[control_min, control_max]), "pairs at %g-%g min scaled to the window: %.1f" % (
                control_min, control_max, expect), "flares on distant stars within %g min of each other" % dt_min, "%d pairs against %.1f" % (near, expect), [])
        print("recorded")


flarecopies_DOC = r"""Standardised flares: stars of the Yang et al. (2023) TESS flare catalogue with ten or more flares whose flare energies scatter by less than 5 per cent, where a star's flares spread over decades of energy.

    python -I seti.py flashes flarecopies [record] [--n-min 10] [--spread 0.05]"""


def flarecopies_main(record=False, n_min=10, spread=0.05):
    fl = core.tap("vizier", "SELECT TIC, E FROM %s" % YANG_FLARES, timeout=900)
    by = {}
    for r in fl:
        if num(r["E"]):
            by.setdefault(str(r["TIC"]), []).append(num(r["E"]))
    many = {k: np.array(v) for k, v in by.items() if len(v) >= n_min}
    # energies may be listed as logarithms; the spread is taken on the linear energy either way
    spreads = {k: float(np.std(10 ** v if np.median(v) < 100 else v) / np.mean(10 ** v if np.median(v) < 100 else v)) for k, v in many.items()}
    hits = sorted([(k, s) for k, s in spreads.items() if s < spread], key=lambda x: x[1])
    vals = np.array(list(spreads.values()))
    print("%d stars with %d or more flares; spread of energy median %.2f, lowest %.3f; below %g per cent: %d %s" % (
        len(many), n_min, float(np.median(vals)), float(vals.min()), 100 * spread, len(hits), hits[:10]))
    if record:
        core.record_run(
            "tess_standard_flares", "Standardised flares", "Yang et al. (2023) TESS flare catalogue (VizieR J/A+A/669/A15)",
            "stars with %d or more flares: %d" % (n_min, len(many)), os.path.abspath(__file__), "flarecopies", len(many), "stars", dict(spread=spread, n_min=n_min),
            "the stars' own energy spreads: median %.2f" % float(np.median(vals)), "a star whose flares all carry one energy to %g per cent" % (100 * spread),
            "%d below %g per cent; the lowest spread %.3f" % (len(hits), 100 * spread, float(vals.min())),
            [dict(target="tic:%s" % k, stat=s, unit="fractional spread of flare energy") for k, s in hits])
        print("recorded")


steady_DOC = r"""Stars steadier than photon noise allows: the Kepler DR25 stellar table's 6-hour combined differential photometric precision against the shot noise of the star's Kepler magnitude, 1.74e5 electrons per second at Kp 12 over 6 hours at a 92 per cent duty cycle (17 ppm at Kp 12), flagged below 0.8 of it.

    python -I seti.py flashes steady [record] [--fraction 0.8] [--hours 6] [--electrons-kp12 1.74e5] [--duty 0.92]

The hours name the CDPP column (3, 6 or 12 in the DR25 table)."""


def steady_main(record=False, fraction=0.8, hours=6.0, electrons_kp12=1.74e5, duty=0.92):
    cols = list(core.exoplanets("SELECT TOP 1 * FROM q1_q17_dr25_ks")[0])
    tag = "%02dp%d" % (int(hours), round((hours % 1) * 10))
    cd = next((c for c in cols if "cdpp" in c.lower() and tag in c), None)
    rows = core.exoplanets("SELECT kepid, kepmag, %s FROM q1_q17_dr25_ks WHERE %s IS NOT NULL AND kepmag IS NOT NULL" % (cd, cd))
    out = []
    for r in rows:
        kp, cdpp = float(r["kepmag"]), float(r[cd])
        shot = 1e6 / math.sqrt(electrons_kp12 * 10 ** (-0.4 * (kp - 12)) * hours * 3600 * duty)
        out.append((r["kepid"], kp, cdpp, cdpp / shot))
    hits = sorted([o for o in out if o[3] < fraction], key=lambda o: o[3])
    ratio = np.array([o[3] for o in out])
    print("%d Kepler stars with %g-hour CDPP (%s); noise over shot noise median %.2f; below %g: %d %s" % (
        len(out), hours, cd, float(np.median(ratio)), fraction, len(hits), [(o[0], o[1], o[2], round(o[3], 2)) for o in hits[:10]]))
    if record:
        core.record_run(
            "kepler_steadier_than_photons", "Stars steadier than photon noise allows", "NASA Exoplanet Archive Q1-Q17 DR25 Kepler stellar table",
            "stars with a %g-hour CDPP: %d" % (hours, len(out)), os.path.abspath(__file__), "steady", len(out), "stars",
            dict(fraction=fraction, hours=hours, electrons_kp12=electrons_kp12, duty=duty),
            "each star's shot noise from its Kepler magnitude", "a star whose measured noise falls below the photons' own",
            "%d below %g of the shot noise; median ratio %.2f" % (len(hits), fraction, float(np.median(ratio))),
            [dict(target="kic:%s" % o[0], stat=o[3], unit="CDPP over shot noise", vet="artefact",
                  note="Kp %.2f, %s ppm, below the photons' own noise: the pipeline's figure" % (o[1], o[2])) for o in hits[:500]])
        print("recorded")


sndeficit_DOC = r"""Galaxies that suppress supernovae: HECATE galaxies within 20 Mpc with a homogenised star-formation rate, each given the core-collapse supernovae its rate predicts over 1980-2015 at one per 100 solar masses formed, against the supernovae the Asiago catalogue (B/sn) lists in it, hosts matched within 1 arcmin; a galaxy whose supernovae fall short of its star formation beyond chance is either hiding them in dust or stopping them. A galaxy whose infrared star-formation rate exceeds its ultraviolet one by a factor of three, or whose infrared luminosity passes 2e10 solar, is dusty, and an active nucleus lends its infrared to the rate.

    python -I seti.py flashes sndeficit [record] [--d-max-mpc 20] [--sn-per-msun 0.01] [--year-min 1980] [--year-max 2015] [--host-arcmin 1] [--dust-ratio 3] [--ltir-max 10.3] [--p-max 0.01]"""


def sndeficit_main(record=False, d_max_mpc=20.0, sn_per_msun=0.01, year_min=1980, year_max=2015, host_arcmin=1.0, dust_ratio=3.0, ltir_max=10.3, p_max=0.01):
    from scipy.stats import poisson
    SN_YEARS = (year_min, year_max)
    gal = core.tap("vizier", 'SELECT PGC, NED, RAJ2000, DEJ2000, D, logSFRh, logSFRTIR, logSFRGSW, logLTIR, logMtot, Nuclear, AGNs17, AGNh FROM "J/MNRAS/506/1896/hecate" '
                             'WHERE D < %g AND logSFRh IS NOT NULL' % d_max_mpc)
    sne = core.tap("vizier", 'SELECT SN, Galaxy, Type, RAG, DEG FROM "B/sn/sncat" WHERE HRV < 3000 AND RAG IS NOT NULL')
    sne = [s for s in sne if SN_YEARS[0] <= int(s["SN"][:4]) <= SN_YEARS[1]]
    hosts = {}
    for m in core.xmatch([dict(source_id="s%d" % i, ra=float(s["RAG"]), dec=float(s["DEG"])) for i, s in enumerate(sne)], "J/MNRAS/506/1896/hecate", host_arcmin * 60.0):
        i = int(m["source_id"][1:])
        if i not in hosts or float(m["angDist"]) < hosts[i][1]:
            hosts[i] = (m["PGC"], float(m["angDist"]))
    count = {}
    for i, (pgc, _) in hosts.items():
        count[pgc] = count.get(pgc, 0) + 1
    years = SN_YEARS[1] - SN_YEARS[0] + 1
    out = []
    for g in gal:
        sfr = 10 ** float(g["logSFRh"])
        exp = sfr * sn_per_msun * years
        n = count.get(g["PGC"], 0)
        p = float(poisson.cdf(n, exp))
        tir, uv, ltir = core.num(g["logSFRTIR"]), core.num(g["logSFRGSW"]), core.num(g["logLTIR"])
        # a luminous infrared galaxy hides its supernovae in dust; an active nucleus lends its infrared to the rate
        dusty = (tir is not None and uv is not None and tir - uv > math.log10(dust_ratio)) or (ltir is not None and ltir > ltir_max)
        agn = bool((g["AGNs17"] or "").strip() or (g["AGNh"] or "").strip() or (g["Nuclear"] or "").strip() not in ("", "0"))
        out.append(dict(pgc=g["PGC"], name=(g["NED"] or "").strip(), d=float(g["D"]), sfr=round(sfr, 2), expected=round(exp, 1), seen=n, p=p, dusty=dusty, agn=agn,
                        vet="dust" if dusty else "background-galaxy" if agn else None))
    short = sorted([o for o in out if o["p"] < p_max], key=lambda o: o["p"])
    print("%d galaxies within %g Mpc with star-formation rates, %d supernovae of %d-%d matched to them; %d galaxies short of their supernovae at p below %g, %d of them dusty "
          "and %d more with active nuclei; the rest %s" % (len(gal), d_max_mpc, len(hosts), SN_YEARS[0], SN_YEARS[1], len(short), p_max, sum(1 for o in short if o["dusty"]),
                                                            sum(1 for o in short if o["agn"] and not o["dusty"]),
                                                            [(o["name"], o["d"], o["sfr"], o["expected"], o["seen"], round(o["p"], 4)) for o in short if o["vet"] is None]))
    for o in short:
        print("  %s (PGC %s, %.1f Mpc): %.1f Msun/yr, %.1f expected, %d seen, p %.4f%s%s" % (o["name"], o["pgc"], o["d"], o["sfr"], o["expected"], o["seen"], o["p"],
                                                                                             ", dusty" if o["dusty"] else "", ", active nucleus" if o["agn"] else ""))
    if record:
        core.record_run("supernova_deficits", "Galaxies that suppress supernovae", "HECATE (J/MNRAS/506/1896); Asiago supernova catalogue (B/sn) by CDS XMatch",
                        "galaxies within %g Mpc with star-formation rates: %d" % (d_max_mpc, len(gal)), os.path.abspath(__file__), "sndeficit", len(gal), "galaxies",
                        dict(sn_per_msun=sn_per_msun, years=SN_YEARS, host_arcmin=host_arcmin, dust_ratio=dust_ratio, ltir_max=ltir_max, d_max_mpc=d_max_mpc, p_max=p_max),
                        "Poisson counts at each galaxy's own rate",
                        "a galaxy with fewer supernovae than its star formation gives at p below %g" % p_max, "%d short, %d of them dusty and %d with active nuclei; %d unexplained" % (
                            len(short), sum(1 for o in short if o["dusty"]), sum(1 for o in short if o["agn"] and not o["dusty"]), sum(1 for o in short if o["vet"] is None)),
                        [dict(target="pgc:%s" % o["pgc"], stat=o["seen"], p=o["p"], unit="supernovae seen against %.1f expected" % o["expected"], vet=o["vet"],
                              note="%s, %.1f Mpc, %.1f Msun/yr" % (o["name"], o["d"], o["sfr"])) for o in short])
        print("recorded")


symflares_DOC = r"""Symmetric flares: the TESS flares of Yang et al. (2023) (VizieR J/A+A/669/A15) lasting 40 minutes or more, so that the two-minute cadence resolves their shape, whose rise from start to peak takes as long as the decay from peak to end within 10 per cent of the duration, or longer; a stellar flare rises impulsively and decays slowly, and a shape that runs backwards or stands symmetric is not a flare's. Each read for its amplitude and energy, and the star's flare count, since two flares in succession give one symmetric event and a catalogue's boundaries follow its own noise threshold.

    python -I seti.py flashes symflares [record] [--dur-min-d 0.028] [--sym-frac 0.1]"""


def symflares_main(record=False, dur_min_d=0.028, sym_frac=0.1):
    fl = core.tap("vizier", "SELECT TIC, TimeS, TimeP, TimeE, Dur, Amp, E FROM %s" % YANG_FLARES, timeout=900)
    per_star = {}
    for r in fl:
        per_star[str(r["TIC"])] = per_star.get(str(r["TIC"]), 0) + 1
    long_, sym, back = [], [], []
    for r in fl:
        ts, tp, te, d = num(r["TimeS"]), num(r["TimeP"]), num(r["TimeE"]), num(r["Dur"])
        if None in (ts, tp, te, d) or d < dur_min_d:
            continue
        rise, decay = tp - ts, te - tp
        o = dict(tic=str(r["TIC"]), start=ts, dur=d, rise=rise, decay=decay, ratio=rise / d, amp=num(r["Amp"]), e=num(r["E"]), nflares=per_star[str(r["TIC"])])
        long_.append(o)
        if rise > decay:
            back.append(o)
        elif abs(rise - decay) < sym_frac * d:
            sym.append(o)
    ratios = np.array([o["ratio"] for o in long_])
    print("%d flares, %d lasting %g d or more; rise over duration median %.2f, 90th percentile %.2f; symmetric within %g of the duration: %d; rising longer than "
          "decaying: %d" % (len(fl), len(long_), dur_min_d, np.median(ratios) if len(ratios) else 0, np.percentile(ratios, 90) if len(ratios) else 0, sym_frac, len(sym),
                            len(back)))
    for o in sorted(back + sym, key=lambda o: -o["ratio"])[:25]:
        print("  TIC %s at %.3f: %.3f d, rise %.3f, decay %.3f (rise %.2f of the duration), amplitude %s, energy %s, %d flares on the star" % (
            o["tic"], o["start"], o["dur"], o["rise"], o["decay"], o["ratio"], o["amp"], o["e"], o["nflares"]))
    json.dump(dict(n=len(fl), long=len(long_), symmetric=sym, backwards=back), open(core.results("symmetric_flares.json"), "w"), indent=1)
    if record:
        hits = back + sym
        core.record_run(
            "tess_symmetric_flares", "Symmetric flares", "Yang et al. (2023) TESS flare catalogue (VizieR J/A+A/669/A15)",
            "flares lasting %g d or more: %d of %d" % (dur_min_d, len(long_), len(fl)), os.path.abspath(__file__), "symflares", len(long_), "flares",
            dict(dur_min_d=dur_min_d, sym_frac=sym_frac), "the flares' own rise fractions: median %.2f, 90th percentile %.2f" % (
                np.median(ratios) if len(ratios) else 0, np.percentile(ratios, 90) if len(ratios) else 0),
            "a flare whose rise takes as long as its decay to %g of its duration" % sym_frac,
            "%d symmetric and %d rising longer than decaying, of %d, at every duration and on quiet and busy stars alike: the catalogue's event boundaries, not flare "
            "shapes" % (len(sym), len(back), len(long_)),
            [dict(target="tic:%s" % o["tic"], stat=o["ratio"], unit="rise as a fraction of the duration",
                  vet="event-boundary",
                  note="start %.3f, duration %.3f d, amplitude %s, energy %s, %d flares on the star" % (o["start"], o["dur"], o["amp"], o["e"], o["nflares"])) for o in hits])
        print("recorded")


flicker_DOC = r"""Nearby stars flickering by the minute: Gaia DR3 short-timescale variables (gaiadr3.vari_short_timescale) within 100 pc whose characteristic frequency exceeds 48 cycles a day, light changing within half an hour at a star next door, where a photosphere's own pulsations and spots are slower by far and only compact stars and close binaries keep such time; each read for its amplitude, its white-dwarf locus in colour and absolute magnitude, its RUWE and its SIMBAD type; a red dwarf (BP-RP above 2) the pipeline picks up flares, and its variogram timescale is the flares' rise rather than a period.

    python -I seti.py flashes flicker [record] [--plx-min 10] [--freq-min 48] [--red-colour 2.0]"""


def flicker_main(record=False, plx_min=10.0, freq_min=48.0, red_colour=2.0):
    rows = core.gaia_by_range("SELECT v.source_id, v.frequency, v.amplitude_estimate, v.number_of_fov_transits, g.parallax, g.phot_g_mean_mag, g.bp_rp, g.ruwe, "
                              "g.ipd_frac_multi_peak, g.ra, g.dec FROM gaiadr3.vari_short_timescale AS v JOIN gaiadr3.gaia_source AS g ON v.source_id = g.source_id "
                              "WHERE v.source_id BETWEEN {lo} AND {hi} AND g.parallax > %g AND g.parallax_over_error > 10" % plx_min, n=8, maxrec=100000)
    st = []
    for r in rows:
        f, plx, g, c = num(r["frequency"]), num(r["parallax"]), num(r["phot_g_mean_mag"]), num(r["bp_rp"])
        if f is None or plx is None or g is None:
            continue
        st.append(dict(id=str(r["source_id"]), f=f, period_min=1440.0 / f, amp=num(r["amplitude_estimate"]), n=num(r["number_of_fov_transits"]), plx=plx, g=g, c=c,
                       mg=g + 5 * math.log10(plx / 100.0), ruwe=num(r["ruwe"]), multi=num(r["ipd_frac_multi_peak"])))
    fast = [s for s in st if s["f"] > freq_min]
    sim = {}
    for k in range(0, len(fast), 200):
        for r in core.simbad("SELECT i.id, b.main_id, b.otype, b.sp_type FROM ident AS i JOIN basic AS b ON i.oidref = b.oid WHERE i.id IN (%s)" % ",".join(
                "'Gaia DR3 %s'" % s["id"] for s in fast[k:k + 200])):
            sim[str(r["id"]).split()[-1]] = (" ".join(str(r["main_id"]).split()), str(r["otype"]).strip(), (r["sp_type"] or "").strip())
    for s in fast:
        s["simbad"] = sim.get(s["id"])
        typ, sp = (s["simbad"] or (None, "", ""))[1], (s["simbad"] or (None, "", ""))[2]
        # the white-dwarf locus: fainter than the main sequence by five magnitudes at the colour
        wd = s["c"] is not None and s["mg"] > 5.0 * s["c"] + 7.0
        # a red dwarf the short-timescale pipeline picks up flares, and its variogram timescale is the flares' rise, not a period
        s["vet"] = ("white-dwarf" if wd or typ in ("WD*", "WD?", "CV*", "No*") or "D" in sp[:2] else
                    "natural-type" if typ not in ("", "*", "PM*", "HV*", "low-mass*") else
                    "blend" if (s["ruwe"] or 0) > 1.4 or (s["multi"] or 0) > 2 else
                    "flare" if (s["c"] or 0) > red_colour else None)
    left = [s for s in fast if s["vet"] is None]
    count = {v: sum(s["vet"] == v for s in fast) for v in ("white-dwarf", "natural-type", "blend", "flare")}
    print("%d short-timescale variables within %.0f pc, the fastest at %.1f cycles a day; faster than %g cycles a day: %d, %d on the white-dwarf locus or of compact type, "
          "%d of other known types, %d blended, %d red dwarfs whose timescale is their flares'; unexplained %d" % (
              len(st), 1000 / plx_min, max(s["f"] for s in st) if st else 0, freq_min, len(fast), count["white-dwarf"], count["natural-type"], count["blend"], count["flare"],
              len(left)))
    for s in sorted(fast, key=lambda s: -s["f"]):
        print("  Gaia DR3 %s: %.1f cycles a day (%.1f min), amplitude %s mag over %s transits, G %.2f, BP-RP %s, M_G %.2f, RUWE %s; SIMBAD %s; %s" % (
            s["id"], s["f"], s["period_min"], s["amp"], s["n"], s["g"], s["c"], s["mg"], s["ruwe"], s["simbad"], s["vet"] or "unexplained"))
    json.dump(dict(n=len(st), fast=fast), open(core.results("nearby_flicker.json"), "w"), indent=1)
    if record:
        core.record_run(
            "gaia_short_timescale_nearby", "Nearby stars flickering by the minute", "Gaia DR3 vari_short_timescale and gaia_source; SIMBAD",
            "short-timescale variables within %.0f pc: %d" % (1000 / plx_min, len(st)), os.path.abspath(__file__), "flicker", len(st), "stars",
            dict(plx_min=plx_min, freq_min=freq_min, red_colour=red_colour), "the white-dwarf locus, SIMBAD's types, the image flags and the flare stars' colour",
            "a characteristic frequency above %g cycles a day at a star within %.0f pc" % (freq_min, 1000 / plx_min),
            "%d: %d white dwarfs or compact types, %d other known types, %d blended, %d flaring red dwarfs; %d unexplained" % (
                len(fast), count["white-dwarf"], count["natural-type"], count["blend"], count["flare"], len(left)),
            [dict(target="gaia_dr3:%s" % s["id"], stat=s["f"], unit="cycles a day", vet=s["vet"],
                  note="period %.1f min, amplitude %s mag, G %.2f, BP-RP %s, M_G %.2f, RUWE %s; SIMBAD %s" % (s["period_min"], s["amp"], s["g"], s["c"], s["mg"], s["ruwe"],
                                                                                                           s["simbad"])) for s in fast])
        print("recorded")


unclassified_DOC = r"""Variables Gaia could not classify next door: Gaia DR3 variables (vari_summary) within 50 pc whose G range exceeds 0.3 mag and to which the DR3 classifier (vari_classifier_result) gave no class, light changing by a third at a star next door in a way none of the two dozen known kinds of variable fits; each read in SIMBAD for its type, for its RUWE and image flags and for the white-dwarf locus, since an eclipsing pair, a flare star or a young star often escapes the classifier on few transits, and a red dwarf's range is one flaring transit against the rest.

    python -I seti.py flashes unclassified [record] [--plx-min 20] [--range-min 0.3] [--red-colour 2.0]"""


def unclassified_main(record=False, plx_min=20.0, range_min=0.3, red_colour=2.0):
    rows = core.gaia_by_range("SELECT g.source_id, g.parallax, g.phot_g_mean_mag, g.bp_rp, g.ruwe, g.ipd_frac_multi_peak, v.num_selected_g_fov, v.range_mag_g_fov, "
                              "v.in_vari_classification_result, c.best_class_name, c.best_class_score FROM gaiadr3.gaia_source AS g JOIN gaiadr3.vari_summary AS v "
                              "ON g.source_id = v.source_id LEFT OUTER JOIN gaiadr3.vari_classifier_result AS c ON g.source_id = c.source_id WHERE g.source_id BETWEEN {lo} "
                              "AND {hi} AND g.parallax > %g AND g.parallax_over_error > 10 AND v.range_mag_g_fov > %g" % (plx_min, range_min), n=8, maxrec=100000)
    st = [dict(id=str(r["source_id"]), plx=num(r["parallax"]), g=num(r["phot_g_mean_mag"]), c=num(r["bp_rp"]), ruwe=num(r["ruwe"]), multi=num(r["ipd_frac_multi_peak"]),
               n=num(r["num_selected_g_fov"]), rng=num(r["range_mag_g_fov"]), cls=(r["best_class_name"] or "").strip() or None, score=num(r["best_class_score"])) for r in rows]
    un = [s for s in st if not s["cls"]]
    sim = {}
    for k in range(0, len(un), 200):
        for r in core.simbad("SELECT i.id, b.main_id, b.otype, b.sp_type FROM ident AS i JOIN basic AS b ON i.oidref = b.oid WHERE i.id IN (%s)" % ",".join(
                "'Gaia DR3 %s'" % s["id"] for s in un[k:k + 200])):
            sim[str(r["id"]).split()[-1]] = (" ".join(str(r["main_id"]).split()), str(r["otype"]).strip(), (r["sp_type"] or "").strip())
    for s in un:
        s["simbad"] = sim.get(s["id"])
        typ, sp = (s["simbad"] or (None, "", ""))[1], (s["simbad"] or (None, "", ""))[2]
        s["mg"] = s["g"] + 5 * math.log10(s["plx"] / 100.0) if s["g"] is not None and s["plx"] else None
        wd = s["c"] is not None and s["mg"] is not None and s["mg"] > 5.0 * s["c"] + 7.0
        # a red dwarf's range is one flaring transit against the rest
        s["vet"] = ("white-dwarf" if wd or typ in ("WD*", "WD?", "CV*", "No*") or "D" in sp[:2] else
                    "natural-type" if typ not in ("", "*", "PM*", "HV*", "low-mass*") else
                    "blend" if (s["ruwe"] or 0) > 1.4 or (s["multi"] or 0) > 2 else
                    "flare" if (s["c"] or 0) > red_colour else None)
    left = [s for s in un if s["vet"] is None]
    count = {v: sum(s["vet"] == v for s in un) for v in ("white-dwarf", "natural-type", "blend", "flare")}
    print("%d Gaia variables within %.0f pc with a G range above %g mag; without a class: %d, %d white dwarfs or compact, %d of other known types, %d blended, %d flaring red "
          "dwarfs; unexplained %d" % (len(st), 1000 / plx_min, range_min, len(un), count["white-dwarf"], count["natural-type"], count["blend"], count["flare"], len(left)))
    for s in sorted(un, key=lambda s: -(s["rng"] or 0)):
        print("  Gaia DR3 %s: range %.2f mag over %s transits, G %.2f, BP-RP %s, M_G %s, RUWE %s; SIMBAD %s; %s" % (
            s["id"], s["rng"], s["n"], s["g"], s["c"], "%.2f" % s["mg"] if s["mg"] is not None else "-", s["ruwe"], s["simbad"], s["vet"] or "unexplained"))
    json.dump(dict(n=len(st), unclassified=un), open(core.results("gaia_unclassified_nearby.json"), "w"), indent=1)
    if record:
        core.record_run(
            "gaia_unclassified_variables_nearby", "Variables Gaia could not classify next door", "Gaia DR3 vari_summary and vari_classifier_result with gaia_source; SIMBAD",
            "Gaia variables within %.0f pc with a G range above %g mag: %d" % (1000 / plx_min, range_min, len(st)), os.path.abspath(__file__), "unclassified", len(st), "stars",
            dict(plx_min=plx_min, range_min=range_min, red_colour=red_colour), "the %d classified variables of the same sample" % (len(st) - len(un)),
            "a variable of a third of a magnitude that no class of the DR3 classifier fits",
            "%d unclassified: %d white dwarfs or compact, %d of other known types, %d blended, %d flaring red dwarfs; %d unexplained" % (
                len(un), count["white-dwarf"], count["natural-type"], count["blend"], count["flare"], len(left)),
            [dict(target="gaia_dr3:%s" % s["id"], stat=s["rng"], unit="G range, mag", vet=s["vet"],
                  note="%s transits, G %.2f, BP-RP %s, RUWE %s; SIMBAD %s" % (s["n"], s["g"], s["c"], s["ruwe"], s["simbad"])) for s in un])
        print("recorded")


wdvary_DOC = r"""White dwarfs varying outside the pulsation strips: Gaia DR3 variables (vari_summary) within 100 pc on the white-dwarf locus (fainter than the main sequence by five magnitudes at their colour) with a G range above 0.1 mag, matched to Gentile Fusillo et al. (2021) for temperature, gravity and white-dwarf probability, those outside the three pulsation strips with their margins (DA 10,000-13,500 K, DB 21,000-31,000 K, GW Vir above 70,000 K); a white dwarf that varies without pulsating is eclipsed, spotted, magnetic or crossed by debris, and debris in orbit around a dead star is the one natural kind of structure the archive would show a made one against. Each read in SIMBAD for its type, including the magnetic types that vary with their rotation, for a companion in its BP/RP flux excess and for its RUWE.

    python -I seti.py flashes wdvary [record] [--plx-min 10] [--range-min 0.1] [--pwd-min 0.75]"""

WD_STRIPS = ((10000.0, 13500.0), (21000.0, 31000.0), (70000.0, 1e6))


def wdvary_main(record=False, plx_min=10.0, range_min=0.1, pwd_min=0.75):
    rows = core.gaia_by_range("SELECT g.source_id, g.parallax, g.phot_g_mean_mag, g.bp_rp, g.ruwe, g.phot_bp_rp_excess_factor, v.num_selected_g_fov, v.range_mag_g_fov "
                              "FROM gaiadr3.gaia_source AS g JOIN gaiadr3.vari_summary AS v ON g.source_id = v.source_id WHERE g.source_id BETWEEN {lo} AND {hi} "
                              "AND g.parallax > %g AND g.parallax_over_error > 10 AND v.range_mag_g_fov > %g AND g.bp_rp IS NOT NULL "
                              "AND g.phot_g_mean_mag + 5 * LOG10(g.parallax / 100) > 5 * g.bp_rp + 7" % (plx_min, range_min), n=8, maxrec=100000)
    ids = [str(r["source_id"]) for r in rows]
    wd = {}
    for k in range(0, len(ids), 300):
        for r in core.tap("vizier", 'SELECT GaiaEDR3, TeffH, loggH, TeffHe, Pwd, RUWE FROM "J/MNRAS/508/3877/maincat" WHERE GaiaEDR3 IN (%s)' % ",".join(ids[k:k + 300])):
            wd[str(r["GaiaEDR3"])] = r
    sim = {}
    for k in range(0, len(ids), 200):
        for s in core.simbad("SELECT i.id, b.main_id, b.otype, b.sp_type FROM ident AS i JOIN basic AS b ON i.oidref = b.oid WHERE i.id IN (%s)" % ",".join(
                "'Gaia DR3 %s'" % x for x in ids[k:k + 200])):
            sim[str(s["id"]).split()[-1]] = (" ".join(str(s["main_id"]).split()), str(s["otype"]).strip(), (s["sp_type"] or "").strip())
    out = []
    for r in rows:
        w = wd.get(str(r["source_id"]))
        if not w or (num(w["Pwd"]) or 0) < pwd_min:
            continue
        teff = num(w["TeffH"])
        c = num(r["bp_rp"])
        ex = (num(r["phot_bp_rp_excess_factor"]) or 0) - (1.154360 + 0.033772 * c + 0.032277 * c * c if c < 0.5 else 1.162004 + 0.011464 * c + 0.049255 * c * c - 0.005879 * c ** 3)
        o = dict(id=str(r["source_id"]), plx=num(r["parallax"]), g=num(r["phot_g_mean_mag"]), c=c, ruwe=num(r["ruwe"]), excess=ex, n=num(r["num_selected_g_fov"]),
                 rng=num(r["range_mag_g_fov"]), teff=teff, logg=num(w["loggH"]), pwd=num(w["Pwd"]), simbad=sim.get(str(r["source_id"])))
        o["strip"] = teff is not None and any(lo <= teff <= hi for lo, hi in WD_STRIPS)
        typ, sp = (o["simbad"] or (None, "", ""))[1], (o["simbad"] or (None, "", ""))[2]
        # a magnetic white dwarf (P or H in its type) varies with its rotation as its spots and field cross the disc
        o["vet"] = ("natural-type" if o["strip"] or typ in ("CV*", "No*", "EB*", "**", "SB*", "Pu*", "V*", "WD?") or "ZZ" in sp or "+" in sp or "P" in sp or "H" in sp[1:] else
                    "companion" if abs(ex) > 0.2 or (o["ruwe"] or 0) > 1.4 else None)
        out.append(o)
    left = [o for o in out if o["vet"] is None]
    print("%d Gaia variables within %.0f pc on the white-dwarf locus with a G range above %g mag and a catalogued white dwarf; in a pulsation strip or of a known kind %d, "
          "with a companion's flux excess or RUWE %d; unexplained %d" % (len(out), 1000 / plx_min, range_min, sum(o["vet"] == "natural-type" for o in out),
                                                                        sum(o["vet"] == "companion" for o in out), len(left)))
    for o in sorted(out, key=lambda o: -(o["rng"] or 0)):
        print("  Gaia DR3 %s: range %.2f mag over %s transits, G %.2f, BP-RP %.2f, Teff %s K, log g %s, P_wd %s, excess %+.3f, RUWE %s; SIMBAD %s; %s" % (
            o["id"], o["rng"], o["n"], o["g"], o["c"], o["teff"], o["logg"], o["pwd"], o["excess"], o["ruwe"], o["simbad"], o["vet"] or "unexplained"))
    json.dump(out, open(core.results("white_dwarfs_varying.json"), "w"), indent=1)
    if record:
        core.record_run(
            "gaia_white_dwarfs_varying", "White dwarfs varying outside the pulsation strips", "Gaia DR3 vari_summary and gaia_source; Gentile Fusillo et al. (2021) white dwarfs "
            "(VizieR J/MNRAS/508/3877); SIMBAD", "catalogued white dwarfs within %.0f pc that Gaia flags variable with a G range above %g mag: %d" % (1000 / plx_min, range_min, len(out)),
            os.path.abspath(__file__), "wdvary", len(out), "white dwarfs", dict(plx_min=plx_min, range_min=range_min, pwd_min=pwd_min, strips_k=WD_STRIPS[:2]),
            "the pulsation strips, SIMBAD's types, the BP/RP flux excess and RUWE", "a white dwarf varying by a tenth of a magnitude outside every pulsation strip",
            "%d: %d in a strip or of a known kind, %d with a companion's signature; %d unexplained" % (
                len(out), sum(o["vet"] == "natural-type" for o in out), sum(o["vet"] == "companion" for o in out), len(left)),
            [dict(target="gaia_dr3:%s" % o["id"], stat=o["rng"], unit="G range, mag", vet=o["vet"],
                  note="%s transits, Teff %s K, log g %s, BP-RP %.2f, excess %+.3f, RUWE %s; SIMBAD %s" % (o["n"], o["teff"], o["logg"], o["c"], o["excess"], o["ruwe"], o["simbad"]))
             for o in out])
        print("recorded")


copyebs_DOC = r"""Copy eclipsing binaries: the Kepler eclipsing binary catalogue (Prsa et al. 2011, VizieR J/AJ/141/83), pairs of systems whose periods agree within the catalogue's own quantum of 0.00001 d, two clocks set alike, and among them the pairs whose temperature ratios and summed radii agree to 10 per cent as copies would, against the count the periods' own density gives in a window ten times wider and the pairs within 2 arcmin, where one binary's light leaks into another's aperture and the catalogue lists one signal twice.

    python -I seti.py flashes copyebs [record] [--tol-d 0.00001] [--like 0.1] [--blend-arcmin 2]"""


def copyebs_main(record=False, tol_d=0.00001, like=0.1, blend_arcmin=2.0):
    from scipy.stats import poisson
    rows = core.tap("vizier", 'SELECT KIC, Per, "T2/T1", "r1+r2", "_RA", "_DE" FROM "J/AJ/141/83/table1" WHERE Per > 0')
    st = sorted([dict(kic=str(r["KIC"]), P=num(r["Per"]), ra=num(r["_RA"]), dec=num(r["_DE"]), t21=num(r["T2/T1"]), rr=num(r["r1+r2"])) for r in rows if num(r["Per"])],
                key=lambda s: s["P"])
    P = np.array([s["P"] for s in st])

    def pairs_within(w):
        out = []
        for i in range(len(st)):
            j = i + 1
            while j < len(st) and P[j] - P[i] <= w:
                out.append((st[i], st[j]))
                j += 1
        return out
    pairs = pairs_within(tol_d)
    # the periods' own density: the pairs within a window ten times wider, scaled down
    exp = len(pairs_within(10 * tol_d)) / 10.0

    def alike(x, y):
        return x is not None and y is not None and abs(x - y) <= like * max(abs(x), abs(y), 1e-9)
    out = []
    for a, b in pairs:
        sep = core.sep_arcsec(a["ra"], a["dec"], b["ra"], b["dec"]) / 60.0
        same = alike(a["t21"], b["t21"]) and alike(a["rr"], b["rr"])
        out.append(dict(kic1=a["kic"], kic2=b["kic"], P1=a["P"], P2=b["P"], dP=abs(a["P"] - b["P"]), sep_arcmin=sep, t21=(a["t21"], b["t21"]), rr=(a["rr"], b["rr"]), copy=same,
                        vet="blend" if sep < blend_arcmin else "chance-rate" if not same else None))
    far = [o for o in out if o["vet"] != "blend"]
    copies = [o for o in far if o["copy"]]
    p = float(poisson.sf(len(far) - 1, exp)) if far else 1.0
    print("%d eclipsing binaries; pairs with periods within %g d: %d, %d within %g arcmin of each other (one signal in two apertures), %d apart on the sky against %.1f expected "
          "from the periods' own density (Poisson p %.3g); of those, alike in temperature ratio and summed radii to %g: %d" % (
              len(st), tol_d, len(out), sum(o["vet"] == "blend" for o in out), blend_arcmin, len(far), exp, p, like, len(copies)))
    for o in sorted(out, key=lambda o: o["dP"]):
        print("  KIC %s and KIC %s: P %.6f and %.6f d (%.1e d apart), %.1f arcmin, T2/T1 %s, r1+r2 %s; %s" % (
            o["kic1"], o["kic2"], o["P1"], o["P2"], o["dP"], o["sep_arcmin"], o["t21"], o["rr"], o["vet"] or "unexplained"))
    json.dump(dict(n=len(st), pairs=out), open(core.results("copy_eclipsing_binaries.json"), "w"), indent=1)
    if record:
        core.record_run(
            "kepler_eb_copies", "Copy eclipsing binaries", "Kepler eclipsing binary catalogue (Prsa et al. 2011, VizieR J/AJ/141/83)", "eclipsing binaries with periods: %d" % len(st),
            os.path.abspath(__file__), "copyebs", len(st), "binaries", dict(tol_d=tol_d, like=like, blend_arcmin=blend_arcmin),
            "the periods' own density gives %.1f pairs within the window; pairs within %g arcmin are one signal in two apertures" % (exp, blend_arcmin),
            "two binaries far apart on the sky with periods within %g d and temperature ratios and summed radii alike to %g" % (tol_d, like),
            "%d pairs within the period window, %d of them blends, %d apart against %.1f expected (Poisson p %.3g); alike in temperature ratio and radii: %d" % (
                len(out), sum(o["vet"] == "blend" for o in out), len(far), exp, p, len(copies)),
            [dict(target="kic:%s" % o["kic1"], stat=o["dP"], unit="period difference from KIC %s, d" % o["kic2"], vet=o["vet"],
                  note="P %.6f and %.6f d, %.1f arcmin apart, T2/T1 %s, r1+r2 %s" % (o["P1"], o["P2"], o["sep_arcmin"], o["t21"], o["rr"])) for o in out])
        print("recorded")


ebwholedays_DOC = r"""Eclipsing binaries on whole days: the Kepler eclipsing binary catalogue (Prsa et al. 2011, VizieR J/AJ/141/83), periods under 50 days measured from space, where no day's cycle aliases a period, those within 0.0002 d of a whole number of days, a binary set to Earth's day, against the same windows at half-integers and the count the periods' own density gives; the catalogue's periods are quoted to 0.00001 d, so none lands on the grid by rounding.

    python -I seti.py flashes ebwholedays [record] [--tol-d 0.0002] [--p-max-d 50]"""


def ebwholedays_main(record=False, tol_d=0.0002, p_max_d=50.0):
    from scipy.stats import poisson
    rows = core.tap("vizier", 'SELECT KIC, Per, "T2/T1", "r1+r2" FROM "J/AJ/141/83/table1" WHERE Per > 0')
    ps = [dict(kic=str(r["KIC"]).strip(), P=float(r["Per"]), ratio=num(r["T2/T1"]), radii=num(r["r1+r2"])) for r in rows]
    ps = [p for p in ps if 0.5 < p["P"] < p_max_d]
    whole = [p for p in ps if abs(p["P"] - round(p["P"])) < tol_d and round(p["P"]) >= 1]
    half = [p for p in ps if abs(p["P"] - (math.floor(p["P"]) + 0.5)) < tol_d]
    exp = 2 * tol_d * len(ps)
    p = float(poisson.sf(len(whole) - 1, exp)) if whole else 1.0
    for w in whole:
        w["vet"] = "chance-rate" if p > 0.01 else None
    print("%d eclipsing binaries with periods under %g d; within %g d of a whole day: %d (%.2f expected from the periods' density, Poisson p %.3g); at half-days: %d" % (
        len(ps), p_max_d, tol_d, len(whole), exp, p, len(half)))
    for w in sorted(whole, key=lambda w: abs(w["P"] - round(w["P"]))):
        print("  KIC %s: P %.5f d (off %d by %.5f), T2/T1 %s, r1+r2 %s; %s" % (w["kic"], w["P"], round(w["P"]), w["P"] - round(w["P"]), w["ratio"], w["radii"], w["vet"] or "unexplained"))
    json.dump(dict(n=len(ps), whole=whole, half=len(half)), open(core.results("kepler_eb_whole_days.json"), "w"), indent=1)
    if record:
        core.record_run(
            "kepler_eb_whole_days", "Eclipsing binaries on whole days", "Kepler eclipsing binary catalogue (Prsa et al. 2011, VizieR J/AJ/141/83)",
            "eclipsing binaries with periods between 0.5 and %g d: %d" % (p_max_d, len(ps)), os.path.abspath(__file__), "ebwholedays", len(ps), "binaries", dict(tol_d=tol_d, p_max_d=p_max_d),
            "the periods' density gives %.2f in the windows; the half-day windows hold %d" % (exp, len(half)), "a period within %g d of a whole number of days" % tol_d,
            "%d against %.2f (Poisson p %.3g), %d at half-days" % (len(whole), exp, p, len(half)),
            [dict(target="kic:%s" % w["kic"], stat=round(w["P"] - round(w["P"]), 5), unit="d from the whole day", vet=w["vet"], note="P %.5f d, T2/T1 %s, r1+r2 %s" % (w["P"], w["ratio"], w["radii"]))
             for w in whole])
        print("recorded")


tesswholedays_DOC = r"""Rotation periods on whole days in TESS: the Fetherolf et al. (2023) catalogue of TESS rotation periods (VizieR J/ApJS/266/22), one period per star from space with no day's cycle to alias it, periods between 0.5 and 13 days measured to 0.005 d, those within 0.005 d of a whole number of days, a star turning on Earth's day, against the same windows at half-integers and the count the periods' own density gives.

    python -I seti.py flashes tesswholedays [record] [--tol-d 0.005] [--p-max-d 13] [--err-max-d 0.005]"""


def tesswholedays_main(record=False, tol_d=0.005, p_max_d=13.0, err_max_d=0.005):
    from scipy.stats import poisson
    from setitools.highenergy import dynamo_vizier
    t = dynamo_vizier("fetherolf")[0]
    best = {}
    for r in t:
        P, e = num(r["Pvar"]), num(r["e_Pvar"])
        tic = str(r["TIC"]).strip()
        if P is None or e is None or e > err_max_d or not 0.5 < P < p_max_d:
            continue
        if tic not in best or e < best[tic]["e"]:
            best[tic] = dict(tic=tic, P=P, e=e, sector=str(r["Sec"]).strip(), tmag=num(r["Tmag"]), teff=num(r["Teff"]))
    ps = list(best.values())
    whole = [p for p in ps if abs(p["P"] - round(p["P"])) < tol_d and round(p["P"]) >= 1]
    half = [p for p in ps if abs(p["P"] - (math.floor(p["P"]) + 0.5)) < tol_d]
    exp = 2 * tol_d * len(ps)
    p = float(poisson.sf(len(whole) - 1, exp)) if whole else 1.0
    for w in whole:
        w["vet"] = "chance-rate" if p > 0.01 else None
    print("%d stars with a TESS rotation period between 0.5 and %g d to %g d; within %g d of a whole day: %d (%.2f expected from the periods' density, Poisson p %.3g); at half-days: %d" % (
        len(ps), p_max_d, err_max_d, tol_d, len(whole), exp, p, len(half)))
    for w in sorted(whole, key=lambda w: abs(w["P"] - round(w["P"]))):
        print("  TIC %s: P %.4f +- %.4f d (off %d by %.4f), sector %s, Tmag %s, %s K; %s" % (w["tic"], w["P"], w["e"], round(w["P"]), w["P"] - round(w["P"]), w["sector"], w["tmag"], w["teff"],
                                                                                        w["vet"] or "unexplained"))
    json.dump(dict(n=len(ps), whole=whole, half=len(half)), open(core.results("tess_rotation_whole_days.json"), "w"), indent=1)
    if record:
        core.record_run(
            "tess_rotation_whole_days", "Rotation periods on whole days in TESS", "Fetherolf et al. (2023) TESS rotation periods (VizieR J/ApJS/266/22)",
            "stars with a rotation period between 0.5 and %g d to %g d: %d" % (p_max_d, err_max_d, len(ps)), os.path.abspath(__file__), "tesswholedays", len(ps), "stars",
            dict(tol_d=tol_d, p_max_d=p_max_d, err_max_d=err_max_d), "the periods' density gives %.2f in the windows; the half-day windows hold %d" % (exp, len(half)),
            "a rotation period within %g d of a whole number of days" % tol_d, "%d against %.2f (Poisson p %.3g), %d at half-days" % (len(whole), exp, p, len(half)),
            [dict(target="tic:%s" % w["tic"], stat=round(w["P"] - round(w["P"]), 4), unit="d from the whole day", vet=w["vet"], note="P %.4f +- %.4f d, sector %s, %s K" % (w["P"], w["e"], w["sector"], w["teff"]))
             for w in whole])
        print("recorded")


ebtwins_DOC = r"""Eclipsing twins to a part in a thousand: the Kepler eclipsing binary catalogue (Prsa et al. 2011, VizieR J/AJ/141/83), systems whose fitted temperature ratio T2/T1 lies within 0.001 of one, two stars alike to a tenth of a per cent, a pair made to one measure, against the count the ratios' own density within 0.05 of one gives; a fit that holds the ratio at exactly 1.000 has pegged a parameter at its boundary, so each is read for the exact value and for its summed radius.

    python -I seti.py flashes ebtwins [record] [--tol 0.001] [--band 0.05]"""


def ebtwins_main(record=False, tol=0.001, band=0.05):
    from scipy.stats import poisson
    rows = core.tap("vizier", 'SELECT KIC, Per, "T2/T1", "r1+r2" FROM "J/AJ/141/83/table1" WHERE "T2/T1" > 0')
    st = [dict(kic=str(r["KIC"]).strip(), P=num(r["Per"]), ratio=num(r["T2/T1"]), radii=num(r["r1+r2"])) for r in rows]
    st = [s for s in st if s["ratio"] is not None]
    near = [s for s in st if abs(s["ratio"] - 1) < band]
    twins = [s for s in st if abs(s["ratio"] - 1) < tol]
    pegged = [s for s in twins if s["ratio"] == 1.0]
    exp = len(near) * (2 * tol) / (2 * band)
    free = [s for s in twins if s["ratio"] != 1.0]
    p = float(poisson.sf(len(free) - 1, exp)) if free else 1.0
    for s in twins:
        s["vet"] = "fit-boundary" if s["ratio"] == 1.0 else ("chance-rate" if p > 0.01 else None)
    left = [s for s in twins if s["vet"] is None]
    print("%d eclipsing binaries with a temperature ratio, %d within %g of one; within %g: %d, %d of them at exactly 1.000 (the fit's boundary) and %d free against %.1f from the density "
          "(Poisson p %.3g); unexplained %d" % (len(st), len(near), band, tol, len(twins), len(pegged), len(free), exp, p, len(left)))
    for s in sorted(free, key=lambda s: abs(s["ratio"] - 1)):
        print("  KIC %s: T2/T1 %.4f, r1+r2 %s, P %s d; %s" % (s["kic"], s["ratio"], s["radii"], s["P"], s["vet"] or "unexplained"))
    json.dump(dict(n=len(st), near=len(near), twins=twins), open(core.results("kepler_eb_twins.json"), "w"), indent=1)
    if record:
        core.record_run(
            "kepler_eb_twins", "Eclipsing twins to a part in a thousand", "Kepler eclipsing binary catalogue (Prsa et al. 2011, VizieR J/AJ/141/83)",
            "eclipsing binaries with a fitted temperature ratio: %d, %d within %g of one" % (len(st), len(near), band), os.path.abspath(__file__), "ebtwins", len(st), "binaries",
            dict(tol=tol, band=band), "the ratios' density within %g of one: %.1f expected within %g; fits pegged at exactly 1.000: %d" % (band, exp, tol, len(pegged)),
            "a temperature ratio within %g of one not pegged by the fit" % tol, "%d within %g, %d pegged at the boundary, %d free against %.1f (Poisson p %.3g); %d unexplained" % (
                len(twins), tol, len(pegged), len(free), exp, p, len(left)),
            [dict(target="kic:%s" % s["kic"], stat=round(s["ratio"], 4), unit="T2/T1", vet=s["vet"], note="r1+r2 %s, P %s d" % (s["radii"], s["P"])) for s in twins])
        print("recorded")


vascobelt_DOC = r"""VASCO transients and the geosynchronous belt: the Villarroel et al. (2020) lists of sources on POSS-I plates with no counterpart on later images (VizieR J/AJ/159/8), their declinations against the belt of geosynchronous satellites, which from Palomar stands at -5.44 deg, the band from -6.1 to -4.6 deg; a glint from something in that orbit on a 1950s plate would be a reflector there before the first launch, and the count in the band is taken against the band's share of the plates' sky, which spans the declinations the lists reach.

    python -I seti.py flashes vascobelt [record] [--belt-lo-deg -6.1] [--belt-hi-deg -4.6]"""


def vascobelt_main(record=False, belt_lo_deg=-6.1, belt_hi_deg=-4.6):
    from scipy.stats import binomtest
    from astropy.coordinates import SkyCoord
    import astropy.units as u
    dec, n_tab = [], {}
    for t in core.vizier("J/AJ/159/8"):
        name = t.meta.get("name", "")
        if "DEJ2000" not in t.colnames:
            continue
        c = SkyCoord([str(x) for x in t["RAJ2000"]], [str(x) for x in t["DEJ2000"]], unit=(u.hourangle, u.deg), frame="icrs")
        dec += list(c.dec.deg)
        n_tab[name] = len(t)
    dec = np.array(dec)
    lo, hi = float(dec.min()), float(dec.max())
    share = (math.sin(math.radians(belt_hi_deg)) - math.sin(math.radians(belt_lo_deg))) / (math.sin(math.radians(hi)) - math.sin(math.radians(lo)))
    n = int(np.sum((dec >= belt_lo_deg) & (dec <= belt_hi_deg)))
    p = binomtest(n, len(dec), share, alternative="greater").pvalue if len(dec) else 1.0
    print("%d transient positions from %s spanning declinations %.1f to %.1f; in the belt's band %g to %g deg: %d against %.2f from the band's share of that sky (binomial p %.3g)" % (
        len(dec), n_tab, lo, hi, belt_lo_deg, belt_hi_deg, n, share * len(dec), p))
    if record:
        core.record_run(
            "vasco_geosynchronous_belt", "VASCO transients and the geosynchronous belt", "Villarroel et al. (2020) POSS-I transient candidates (VizieR J/AJ/159/8)",
            "transient positions: %d, spanning declinations %.1f to %.1f" % (len(dec), lo, hi), os.path.abspath(__file__), "vascobelt", len(dec), "positions",
            dict(belt_lo_deg=belt_lo_deg, belt_hi_deg=belt_hi_deg), "the band's share of the sky between the lists' extreme declinations: %.4f" % share,
            "transients crowding the declination of the geosynchronous belt seen from Palomar", "%d in the band against %.2f (binomial p %.3g)" % (n, share * len(dec), p), [])
        print("recorded")


twomessengers_DOC = r"""Two messengers at once: the IceCube alert tracks of IceCat-1 (Abbasi et al. 2023, VizieR J/ApJS/269/25) that arrived while TESS was observing for the Yang et al. (2023) flare catalogue (sectors 1-30, VizieR J/A+A/669/A15), the catalogue's stars inside each track's 90 per cent region, and their flares peaking within an hour of the neutrino, against the coincidences the same regions give with the alert times shifted by up to 100 days; a transmitter that fires a neutrino beam and lights its star, or a flare that makes neutrinos, would pair the two, and the stars within 100 pc are counted on their own.

    python -I seti.py flashes twomessengers [record] [--window-h 1] [--shifts 2000] [--near-pc 100]"""


def twomessengers_main(record=False, window_h=1.0, shifts=2000, near_pc=100.0):
    al = [t for t in core.vizier("J/ApJS/269/25") if t.meta.get("name", "").endswith("table2")][0]
    st = core.tap("vizier", "SELECT TIC, RAJ2000, DEJ2000, DistG FROM %s" % YANG_STARS, timeout=900)
    fl = core.tap("vizier", "SELECT TIC, TimeP FROM %s" % YANG_FLARES, timeout=900)
    times = {}
    for r in fl:
        if num(r["TimeP"]) is not None:
            times.setdefault(str(r["TIC"]), []).append(num(r["TimeP"]) + 56999.5)
    lo, hi = min(min(v) for v in times.values()), max(max(v) for v in times.values())
    stars = [(str(r["TIC"]), num(r["RAJ2000"]), num(r["DEJ2000"]), num(r["DistG"])) for r in st if str(r["TIC"]) in times and num(r["RAJ2000"]) is not None]
    ra_s = np.array([s[1] for s in stars])
    de_s = np.array([s[2] for s in stars])
    alerts = []
    for a in al:
        t = float(a["MJD"])
        if not lo <= t <= hi:
            continue
        ra, de = float(a["RAJ2000"]), float(a["DEJ2000"])
        dra = (ra_s - ra + 180) % 360 - 180
        dde = de_s - de
        inside = np.nonzero((dra <= float(a["E_RAJ2000"])) & (dra >= -float(a["e_RAJ2000"])) & (dde <= float(a["E_DEJ2000"])) & (dde >= -float(a["e_DEJ2000"])))[0]
        ft = [(stars[k][0], stars[k][3], tt) for k in inside for tt in times[stars[k][0]]]
        alerts.append(dict(alert=str(a["Alert"]).strip(), mjd=t, ra=ra, de=de, energy=num(a["Energy"]), n_stars=len(inside), flares=ft))
    w = window_h / 24.0

    def hits(shift, near=False):
        return sum(1 for a in alerts for tic, d, tt in a["flares"] if abs(tt - (a["mjd"] + shift)) < w and (not near or (d is not None and d < near_pc)))
    real, real_near = hits(0.0), hits(0.0, True)
    rng = np.random.default_rng(9)
    off = rng.uniform(-100, 100, shifts)
    off = off[np.abs(off) > 2 * w]
    null = np.array([hits(s) for s in off])
    null_near = np.array([hits(s, True) for s in off])
    p, p_near = float((null >= real).mean()) if real else 1.0, float((null_near >= real_near).mean()) if real_near else 1.0
    found = [(a["alert"], tic, d, round((tt - a["mjd"]) * 24, 2)) for a in alerts for tic, d, tt in a["flares"] if abs(tt - a["mjd"]) < w]
    print("%d IceCat-1 alerts between MJD %.0f and %.0f; %d flaring stars inside their regions (%d flares); flares within %g h: %d against %.3f from %d shifts (p %.3g); "
          "within %g pc: %d against %.3f (p %.3g); %s" % (len(alerts), lo, hi, sum(a["n_stars"] for a in alerts), sum(len(a["flares"]) for a in alerts), window_h, real,
                                                         null.mean(), len(off), p, near_pc, real_near, null_near.mean(), p_near, found))
    if record:
        core.record_run(
            "icecat_tess_flare_coincidence", "Two messengers at once", "IceCat-1 (VizieR J/ApJS/269/25); Yang et al. (2023) TESS flares (VizieR J/A+A/669/A15)",
            "IceCube alert tracks during TESS sectors 1-30: %d, with %d flaring stars inside their 90 per cent regions" % (len(alerts), sum(a["n_stars"] for a in alerts)),
            os.path.abspath(__file__), "twomessengers", len(alerts), "alerts", dict(window_h=window_h, shifts=shifts, near_pc=near_pc),
            "%d shifts of the alert times by up to 100 days: %.3f coincidences, %.3f within %g pc" % (len(off), null.mean(), null_near.mean(), near_pc),
            "a flare on a star inside a neutrino track's region within %g h of the neutrino" % window_h,
            "%d coincidences against %.3f (p %.3g), %d within %g pc against %.3f" % (real, null.mean(), p, real_near, near_pc, null_near.mean()),
            [dict(target="tic:%s" % tic, stat=dt_h, unit="hours from the neutrino", vet=None, note="%s, %s pc" % (alert, d)) for alert, tic, d, dt_h in found])
        print("recorded")


flareecho_DOC = r"""Flare echoes: every flare of a flare star in Kepler's one-minute photometry (GJ 1243 by default), or with --tess true of the nearest flare stars in TESS's twenty-second and two-minute photometry (Proxima Centauri, AU Mic, YZ CMi, EV Lac, AD Leo, GJ 1243, Wolf 359 and UV Ceti), or with --nearby true of every star within 30 pc brighter than Tmag 15 whose twenty-second light curves hold 30 flares, the light curve made relative to an hour's running median and then to nine minutes', each flare's five-minute impulse correlated with the light curve after it at every delay from 3 to 180 minutes and the flares averaged with weights of their impulse's signal-to-noise, every other peak at 5 widths masked, the secondary bursts of complex flares among them; a mirror, a ring or a swarm in orbit returns each flare as a faint copy after the light-travel delay of its own geometry, 17 minutes from 1 au straight back, where a disc or a shell spreads its echo over a range of delays and sympathetic flares come at any delay; the same correlation before each flare, where no echo can arrive, is the null, a delay that loses half its echo when one flare is left out rests on that flare, and VSX names the eclipsing binaries whose eclipse edges pass for flares.

    python -I seti.py flashes flareecho [record] [--kic 9726699] [--tess false] [--nearby false] [--min-flares 30] [--snr-min 10] [--tau-max 180] [--z-min 5] [--mask-snr 5]"""

FLAREECHO_TESS = {"Proxima Cen": 388857263, "AU Mic": 441420236, "YZ CMi": 266744225, "EV Lac": 154101678, "AD Leo": 95431305, "GJ 1243": 273589987,
                  "Wolf 359": 365006789, "UV Cet": 632499595}


# the high-pass spans nine minutes and the impulse five, wide enough to hold a flare's rise and peak whole
FLAREECHO_HP_MIN, FLAREECHO_IMPULSE_MIN = 9.0, 5.0


def flareecho_odd(minutes, cad):
    """An odd number of cadences spanning about the given minutes, three at least."""
    return max(3, int(round(minutes / cad)) | 1)


def flareecho_prepare(t, f, cad):
    """A light curve in minutes as (minutes, relative flux, its high-pass, noise, cadence): flux over an hour's running median, less the relative flux's running median over FLAREECHO_HP_MIN."""
    from scipy.ndimage import median_filter
    r = f / median_filter(f, size=flareecho_odd(60, cad), mode="nearest") - 1.0
    hp = r - median_filter(r, size=flareecho_odd(FLAREECHO_HP_MIN, cad), mode="nearest")
    return t, r, hp, 1.4826 * float(np.median(np.abs(r - np.median(r)))), cad


def flareecho_kepler(kic):
    """A Kepler star's short-cadence light curves, PDCSAP flux of quality 0, prepared per file."""
    from astropy.io import fits
    out = []
    for key in core.kepler_keys(kic, short=True):
        local = core.data("flareecho", os.path.basename(key))
        if not os.path.exists(local):
            os.makedirs(os.path.dirname(local), exist_ok=True)
            body = core.get(core.S3 + key, timeout=600)
            open(local, "wb").write(body)
        with fits.open(local) as h:
            d = h[1].data
            t, f, q = np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["SAP_QUALITY"])
        ok = np.isfinite(t) & np.isfinite(f) & (q == 0)
        out.append(flareecho_prepare(t[ok] * 1440.0, f[ok], 0.98084))
    return out


def flareecho_tess(tic, sectors=range(1, 100), threads=8):
    """A TESS star's SPOC light curves, twenty-second where a sector has them and two-minute otherwise, PDCSAP flux of quality 0, prepared per sector."""
    from astropy.io import fits

    def one(sector):
        for fast in (True, False):
            local = core.data("flareecho", "tic%d_s%04d_%s.fits" % (tic, sector, "fast" if fast else "2min"))
            if not os.path.exists(local):
                keys = core.spoc_keys(tic, sector, fast)
                if not keys:
                    continue
                body = core.get(core.S3 + keys[0], timeout=600, retries=3, soft=True, pause=2)
                if not body or not body.startswith(b"SIMPLE"):
                    continue
                os.makedirs(os.path.dirname(local), exist_ok=True)
                open(local, "wb").write(body)
            with fits.open(local) as h:
                d = h[1].data
                t, f, q = np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["QUALITY"])
            ok = np.isfinite(t) & np.isfinite(f) & (q == 0)
            if ok.sum() < 1000:
                return None
            t = t[ok] * 1440.0
            return flareecho_prepare(t, f[ok], float(np.median(np.diff(t))))
        return None
    with ThreadPoolExecutor(threads) as ex:
        return [c for c in ex.map(one, sectors) if c is not None]


def flareecho_stack(curves, snr_min, tau_max, mask_snr=5.0):
    """The echo depth of each flare's impulse at every delay of whole minutes from -tau_max to tau_max, weighted over the flares, with their count; every peak at mask_snr widths masked, the secondary bursts of complex flares among them."""
    taus = np.arange(-tau_max, tau_max + 1)
    num_, den, n = np.zeros(len(taus)), np.zeros(len(taus)), 0
    for t, r, hp, sig, cad in curves:
        h = flareecho_odd(FLAREECHO_IMPULSE_MIN, cad) // 2
        # a flare: a local peak at snr_min noise widths with a neighbour above three
        peaks = [i for i in range(h, len(r) - h) if r[i] >= snr_min * sig and r[i] == r[i - h:i + h + 1].max() and max(r[i - 1], r[i + 1]) >= 3 * sig]
        busy = np.zeros(len(r), bool)
        for i in [i for i in range(h, len(r) - h) if r[i] >= mask_snr * sig and r[i] == r[i - h:i + h + 1].max()]:
            busy[max(i - int(3 / cad) - h, 0):i + int(6 / cad) + h + 1] = True
        # cadences on one grid, so that a delay counts in steps even across gaps
        step = np.round((t - t[0]) / cad).astype(int)
        where = np.full(step.max() + 1, -1)
        where[step] = np.arange(len(step))
        shift = np.round(taus / cad).astype(int)
        for i in peaks:
            g = hp[i - h:i + h + 1]
            gg = float(g @ g)
            if gg <= 0:
                continue
            n += 1
            w = gg / sig ** 2
            for k, tau in enumerate(taus):
                j = step[i] + shift[k]
                if abs(tau) < 3 or j - h < 0 or j + h + 1 > len(where):
                    continue
                idx = where[j - h:j + h + 1]
                if (idx < 0).any() or busy[idx].any():
                    continue
                num_[k] += w * float(g @ hp[idx]) / gg
                den[k] += w
    return taus, np.where(den > 0, num_ / np.maximum(den, 1e-30), np.nan), n


def flareecho_score(taus, eps, z_min):
    """The echo's strongest delay after the flares in widths of its spread before them, the largest before them, and the delays at z_min or more."""
    pre, post = (taus <= -3) & np.isfinite(eps), (taus >= 3) & np.isfinite(eps)
    med = float(np.median(eps[pre]))
    width = 1.4826 * float(np.median(np.abs(eps[pre] - med)))
    z = (eps - med) / max(width, 1e-12)
    k = int(np.nanargmax(np.where(post, z, -np.inf)))
    hits = [(int(taus[j]), float(eps[j]), float(z[j])) for j in np.flatnonzero(post & (z >= z_min))]
    return dict(tau=int(taus[k]), eps=float(eps[k]), z=float(z[k]), pre=float(np.nanmax(np.abs(z[pre]))), width=width, hits=hits)


def flareecho_nearby(min_flares, procs=8, tmag_max=15.0):
    """Every star of the twenty-second light curves within 30 pc (the discrete-level search's set) brighter than tmag_max with min_flares flares at ten noise widths, its sectors prepared, as (name, target, curves)."""
    import glob
    from concurrent.futures import ProcessPoolExecutor
    from astropy.io import fits
    files = [f for f in sorted(glob.glob(core.data("tess20", "*.fits"))) if (fits.getheader(f).get("TESSMAG") or 99) <= tmag_max]
    with ProcessPoolExecutor(procs) as ex:
        curves = list(ex.map(flareecho_file, files, chunksize=8))
    by = {}
    for path, c in zip(files, curves):
        if c is not None:
            by.setdefault(int(fits.getheader(path).get("TICID")), []).append(c)
    out = []
    for tic, cs in sorted(by.items()):
        n = 0
        for t, r, hp, sig, cad in cs:
            h = flareecho_odd(FLAREECHO_IMPULSE_MIN, cad) // 2
            n += sum(1 for i in range(h, len(r) - h) if r[i] >= 10 * sig and r[i] == r[i - h:i + h + 1].max() and max(r[i - 1], r[i + 1]) >= 3 * sig)
        if n >= min_flares:
            out.append(("TIC %d" % tic, "tic:%d" % tic, cs))
    return out


def flareecho_file(path):
    """One twenty-second light curve file prepared, or None."""
    from astropy.io import fits
    with fits.open(path) as h:
        d = h[1].data
        t, f, q = np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["QUALITY"])
    ok = np.isfinite(t) & np.isfinite(f) & (q == 0)
    if ok.sum() < 1000:
        return None
    t = t[ok] * 1440.0
    return flareecho_prepare(t, f[ok], float(np.median(np.diff(t))))


def flareecho_main(record=False, kic=9726699, tess=False, nearby=False, min_flares=30, snr_min=10.0, tau_max=180, z_min=5.0, mask_snr=5.0):
    if nearby:
        stars = flareecho_nearby(min_flares)
    elif tess:
        stars = [(name, "tic:%d" % tic, flareecho_tess(tic)) for name, tic in FLAREECHO_TESS.items()]
    else:
        stars = [("KIC %d" % kic, "kic:%d" % kic, flareecho_kepler(kic))]
    out = []
    for name, target, curves in stars:
        if not curves:
            print("  %s: no light curves" % name)
            continue
        taus, eps, n = flareecho_stack(curves, snr_min, tau_max, mask_snr)
        if n == 0 or not np.isfinite(eps).any():
            print("  %s: no flare at %g widths in %d curves" % (name, snr_min, len(curves)))
            continue
        sc = flareecho_score(taus, eps, z_min)
        sc.update(name=name, target=target, n=n, curves=len(curves), cadences=sorted({round(c[4], 2) for c in curves}), vet=None, loo=None)
        if sc["hits"] and sc["z"] > sc["pre"]:
            # an echo returns after every flare: one that falls by half with a single flare left out rests on that flare
            k = int(np.flatnonzero(taus == sc["tau"])[0])
            low = sc["eps"]
            for ci, (t, r, hp, sig, cad) in enumerate(curves):
                h = flareecho_odd(FLAREECHO_IMPULSE_MIN, cad) // 2
                for i in [i for i in range(h, len(r) - h) if r[i] >= snr_min * sig and r[i] == r[i - h:i + h + 1].max() and max(r[i - 1], r[i + 1]) >= 3 * sig]:
                    r2 = r.copy()
                    r2[i - h:i + h + 1] = np.minimum(r2[i - h:i + h + 1], 0.9 * snr_min * sig)
                    low = min(low, float(flareecho_stack(curves[:ci] + [(t, r2, hp, sig, cad)] + curves[ci + 1:], snr_min, tau_max, mask_snr)[1][k]))
            sc["loo"] = low
            if low < 0.5 * sc["eps"]:
                sc["vet"] = "single-epoch"
        elif sc["hits"]:
            sc["vet"] = "chance-rate"
        out.append(sc)
        if not nearby or (sc["hits"] and sc["z"] > sc["pre"]):
            print("%s: %d flares at %g noise widths or more in %d light curves (cadences %s min); echo spread per delay before the flares %.2g; strongest after them %.4f at %d min (%.1f widths), largest before them %.1f widths; %d delays at %g widths or more" % (
                name, n, snr_min, len(curves), ", ".join("%g" % c for c in sc["cadences"]), sc["width"], sc["eps"], sc["tau"], sc["z"], sc["pre"], len(sc["hits"]), z_min))
            for tau, e, zz in sc["hits"]:
                print("  delay %4d min: echo %.4f of the flare (%.1f widths)" % (tau, e, zz))
    # an eclipsing binary's eclipse edges pass the flare finder once its hour's median is taken out
    cand = [o for o in out if o["hits"] and o["z"] > o["pre"] and o["vet"] is None and o["target"].startswith("tic:")]
    if cand:
        pos = {}
        for o in cand:
            r_ = core.tap("simbad", "SELECT b.ra, b.dec FROM ident AS i JOIN basic AS b ON b.oid = i.oidref WHERE i.id = 'TIC %s'" % o["target"][4:])
            if r_:
                pos[o["target"]] = (float(r_[0]["ra"]), float(r_[0]["dec"]))
        vsx = {m["source_id"]: (m.get("Type") or "").strip() for m in core.xmatch([dict(source_id=k, ra=v[0], dec=v[1]) for k, v in pos.items()], "B/vsx/vsx", 5.0)} if pos else {}
        for o in cand:
            if vsx.get(o["target"], "").startswith("E"):
                o["vet"] = "eclipsing-binary"
    for o in out:
        if o["hits"] and o["z"] > o["pre"]:
            print("  %s: echo %.4f at %d min (%.1f widths, above the %.1f before the flares); with one flare left out as low as %s; %s" % (
                o["name"], o["eps"], o["tau"], o["z"], o["pre"], "%.4f" % o["loo"] if o["loo"] is not None else "-", o["vet"] or "LEFT"))
    if nearby:
        print("%d stars within 30 pc brighter than Tmag 15 with %d or more flares at %g widths; %d with a delay after their flares at %g widths above the largest before them, %d of them left; median spread per delay %.2g of the flare" % (
            len(out), min_flares, snr_min, sum(1 for o in out if o["hits"] and o["z"] > o["pre"]), z_min, sum(1 for o in out if o["hits"] and o["z"] > o["pre"] and o["vet"] is None),
            float(np.median([o["width"] for o in out])) if out else float("nan")))
    json.dump(out, open(core.results("flare_echoes_%s.json" % ("nearby" if nearby else "tess" if tess else kic)), "w"), indent=1)
    if record and out:
        core.record_run(
            "tess20_nearby_flare_echoes" if nearby else "tess_flare_echoes" if tess else "kepler_flare_echoes_%d" % kic, "Flare echoes",
            ("TESS SPOC twenty-second light curves of the stars within 30 pc" if nearby else "TESS SPOC twenty-second and two-minute" if tess else "Kepler short-cadence") + " PDCSAP light curves from the STScI AWS mirror",
            "; ".join("%s: %d flares" % (o["name"], o["n"]) for o in out), os.path.abspath(__file__), "flareecho", sum(o["n"] for o in out), "flares",
            dict(kic=kic, tess=tess, nearby=nearby, min_flares=min_flares, snr_min=snr_min, tau_max=tau_max, z_min=z_min, mask_snr=mask_snr),
            "the same correlation at delays before each star's flares, spread per delay %s of the flare" % ", ".join("%.2g" % o["width"] for o in out),
            "a copy of each flare's impulse at one delay of 3-%d minutes; %g widths is an echo of %s of the flare" % (tau_max, z_min, ", ".join("%.2g" % (z_min * o["width"]) for o in out)),
            "%d of %d stars with a delay at %g widths or more after their flares, %d of them above the largest before their flares, %d of those left once single flares and eclipses are read" % (
                sum(1 for o in out if o["hits"]), len(out), z_min, sum(1 for o in out if o["hits"] and o["z"] > o["pre"]),
                sum(1 for o in out if o["hits"] and o["z"] > o["pre"] and o["vet"] is None)),
            [dict(target=o["target"], stat=round(o["z"], 2), unit="widths of the echo spread before the flares, at the strongest delay",
                  vet=o["vet"] if o["vet"] else (None if o["z"] >= max(z_min, o["pre"]) else "chance-rate"),
                  note="%s: echo %.4f of the flare at %d min; spread %.2g; largest before the flares %.1f widths; %d flares%s" % (
                      o["name"], o["eps"], o["tau"], o["width"], o["pre"], o["n"], "; with one flare left out %.4f" % o["loo"] if o["loo"] is not None else ""))
             for o in out])
        print("recorded")

tones_DOC = r"""Minute tones at the nearest stars: every TESS twenty-second light curve the discrete-level search read (the stars within 30 pc), its hour-long variations and its flares removed, its Fourier amplitude read at every period from 1 to 60 minutes against its own running median; a beacon modulating a star or a swarm of regular structure sounds a pure tone that holds its frequency, where cool dwarfs pulsate too faintly to see at these periods, so a tone belongs to a pulsating white dwarf, delta Scuti or roAp star in the aperture, a hot star of its own, or the spacecraft, and the tone's amplitude in the flux-weighted centroid places the source on or off the target; a frequency other stars share within 0.05 cycles a day is the spacecraft's, a whole multiple of the light curve's strongest slow signal, up to the twentieth, is that signal's harmonic, and VSX and SIMBAD name eclipsing binaries and white dwarfs.

    python -I seti.py flashes tones [record] [--snr-min 6] [--p-min-min 1] [--p-max-min 60] [--offset-max 0.3]"""


def tones_one(path, f_lo, f_hi):
    """A twenty-second light curve's strongest tone between f_lo and f_hi cycles a day: frequency, amplitude, signal-to-noise against the running median of the amplitude spectrum, and the centroid's amplitude there in pixels per unit flux."""
    from astropy.io import fits
    from scipy.ndimage import median_filter
    with fits.open(path) as h:
        H, d = h[0].header, h[1].data
        t, f, q = np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["QUALITY"])
        cad = np.asarray(d["CADENCENO"], int)
        cx, cy = np.asarray(d["MOM_CENTR1"], float), np.asarray(d["MOM_CENTR2"], float)
    ok = np.isfinite(t) & np.isfinite(f) & np.isfinite(cx) & np.isfinite(cy) & (q == 0)
    if ok.sum() < 20000:
        return None
    t, f, cad, cx, cy = t[ok], f[ok], cad[ok], cx[ok], cy[ok]
    r = f / np.median(f) - 1.0
    r = r - median_filter(r, size=181, mode="nearest")
    sig = 1.4826 * float(np.median(np.abs(r)))
    # a flare and the half-hour after it are taken out
    bad = np.zeros(len(r), bool)
    for i in np.flatnonzero(r > 5 * sig):
        bad[max(i - 3, 0):i + 90] = True
    cx = cx - median_filter(cx, size=181, mode="nearest")
    cy = cy - median_filter(cy, size=181, mode="nearest")
    n = int(cad.max() - cad.min() + 1)
    grid = np.zeros((3, n))
    k = cad - cad.min()
    for row, x in enumerate((r, cx, cy)):
        grid[row, k] = np.where(bad, 0.0, x)
    dt = float(np.median(np.diff(t)))
    A = np.abs(np.fft.rfft(grid, axis=1)) * 2 / max((~bad).sum(), 1)
    freq = np.fft.rfftfreq(n, d=dt)
    # the light curve's strongest slow signal, before the hour-long variations are taken out, of which a tone may be a harmonic
    raw = np.zeros(n)
    raw[k] = f / np.median(f) - 1.0
    R = np.abs(np.fft.rfft(raw - raw[k].mean() * (raw != 0)))
    slow = freq > 0.3
    f0 = float(freq[slow][np.argmax(R[slow])])
    band = (freq >= f_lo) & (freq <= f_hi)
    noise = median_filter(A[0], size=401, mode="nearest")
    snr = np.where(band, A[0] / np.maximum(noise, 1e-30), 0)
    j = int(np.argmax(snr))
    return dict(tic=int(H.get("TICID")), sector=int(H.get("SECTOR")), tmag=float(H.get("TESSMAG") or np.nan), radius=H.get("RADIUS"), ra=float(H.get("RA_OBJ")), dec=float(H.get("DEC_OBJ")),
                freq=float(freq[j]), amp=float(A[0][j]), snr=float(snr[j]), offset=float(np.hypot(A[1][j], A[2][j]) / max(A[0][j], 1e-12)), f0=f0)


def tones_main(record=False, snr_min=6.0, p_min_min=1.0, p_max_min=60.0, offset_max=0.3, procs=8):
    import glob
    from concurrent.futures import ProcessPoolExecutor
    files = sorted(glob.glob(core.data("tess20", "*.fits")))
    f_lo, f_hi = 1440.0 / p_max_min, 1440.0 / p_min_min
    with ProcessPoolExecutor(procs) as ex:
        res = [x for x in ex.map(tones_one, files, [f_lo] * len(files), [f_hi] * len(files), chunksize=8) if x]
    tones = [x for x in res if x["snr"] >= snr_min]
    for x in tones:
        x["shared"] = sorted({y["tic"] for y in tones if y["tic"] != x["tic"] and abs(y["freq"] - x["freq"]) < 0.05})
        x["sectors_same"] = sorted({y["sector"] for y in tones if y["tic"] == x["tic"] and abs(y["freq"] - x["freq"]) < 0.05})
    stars = len({x["tic"] for x in res})
    # a tone from the target barely moves the centroid; a neighbour's moves it by its separation in pixels per unit of the target's flux
    for x in tones:
        x["vet"] = "detector-wide" if x["shared"] else ("blend" if x["offset"] > offset_max else None)
        # a harmonic of order up to 20 within two frequency bins of a sector (0.08 cycles a day)
        h = int(round(x["freq"] / x["f0"]))
        x["harmonic"] = h if 2 <= h <= 20 and abs(x["freq"] - h * x["f0"]) <= 0.08 else 0
    if tones:
        sb = {}
        for m in core.xmatch([dict(source_id=str(x["tic"]), ra=x["ra"], dec=x["dec"]) for x in tones], "I/355/gaiadr3", 5.0):
            if m["source_id"] not in sb or float(m["angDist"]) < sb[m["source_id"]][0]:
                sb[m["source_id"]] = (float(m["angDist"]), num(m.get("BP-RP")), num(m.get("Gmag")))
        for x in tones:
            g = sb.get(str(x["tic"]))
            x["bp_rp"], x["g"] = (g[1], g[2]) if g else (None, None)
            # A and F stars pulsate at these periods themselves, delta Scuti and roAp alike
            if x["vet"] is None and x["bp_rp"] is not None and x["bp_rp"] < 0.7:
                x["vet"] = "variable-star"
        # the rest by what the star is: a harmonic of its own slow light curve, an eclipsing binary in VSX, a white dwarf in SIMBAD, which pulsates at minutes
        rest = [x for x in tones if x["vet"] is None]
        vsx = {}
        if rest:
            for m in core.xmatch([dict(source_id=str(x["tic"]), ra=x["ra"], dec=x["dec"]) for x in rest], "B/vsx/vsx", 3.0):
                vsx[m["source_id"]] = (m.get("Type") or "").strip()
            otype = {r_["id"]: r_["otype"] for r_ in core.tap("simbad", "SELECT i.id, b.otype FROM ident AS i JOIN basic AS b ON b.oid = i.oidref WHERE i.id IN (%s)" % ",".join(
                "'TIC %d'" % x["tic"] for x in rest))}
            for x in rest:
                x["vsx"], x["otype"] = vsx.get(str(x["tic"]), ""), otype.get("TIC %d" % x["tic"], "")
                if x["otype"].startswith("WD") or x["vsx"].startswith("ZZ"):
                    x["vet"] = "variable-star"
                elif x["vsx"].startswith("E"):
                    x["vet"] = "eclipsing-binary"
                elif x["harmonic"]:
                    x["vet"] = "rotation"
    left = [x for x in tones if x["vet"] is None]
    print("%d twenty-second light curves of %d stars within 30 pc; tones at %g times the local amplitude between %g and %g minutes: %d, of which %d at frequencies other stars share, %d moving the centroid as a neighbour would, %d on stars that pulsate at minutes (hot stars and white dwarfs), %d harmonics of a slower signal or an eclipsing binary's, %d left" % (
        len(res), stars, snr_min, p_min_min, p_max_min, len(tones), sum(x["vet"] == "detector-wide" for x in tones), sum(x["vet"] == "blend" for x in tones),
        sum(x["vet"] == "variable-star" for x in tones), sum(x["vet"] in ("rotation", "eclipsing-binary") for x in tones), len(left)))
    for x in sorted(tones, key=lambda x: -x["snr"])[:25]:
        print("  TIC %-10d s%02d  %8.3f c/d (%6.2f min)  %7.1f ppm  S/N %5.1f  centroid %.3f px  BP-RP %s  harmonic %d  %s %s  %s" % (
            x["tic"], x["sector"], x["freq"], 1440 / x["freq"], 1e6 * x["amp"], x["snr"], x["offset"], "%.2f" % x["bp_rp"] if x.get("bp_rp") is not None else "-",
            x["harmonic"], x.get("vsx") or "", x.get("otype") or "", x["vet"] or "LEFT"))
    json.dump(dict(n=len(res), tones=tones), open(core.results("tess20_minute_tones.json"), "w"), indent=1)
    if record:
        core.record_run(
            "tess20_minute_tones", "Minute tones at the nearest stars", "TESS SPOC twenty-second PDCSAP light curves of the stars within 30 pc (the discrete-level search's set)",
            "%d light curves of %d stars" % (len(res), stars), os.path.abspath(__file__), "tones", len(res), "light curves",
            dict(snr_min=snr_min, p_min_min=p_min_min, p_max_min=p_max_min, offset_max=offset_max),
            "each amplitude spectrum's running median over 401 bins; a Rayleigh spectrum passes %g times its median once in %.0e bins" % (snr_min, math.exp(-math.log(2) * snr_min ** 2)),
            "a tone of 1-60 minutes at %g times the local amplitude" % snr_min,
            "%d tones: %d shared, %d off the target, %d on hot stars and white dwarfs, %d harmonics of a slower signal or an eclipsing binary's, %d left" % (
                len(tones), sum(x["vet"] == "detector-wide" for x in tones), sum(x["vet"] == "blend" for x in tones), sum(x["vet"] == "variable-star" for x in tones),
                sum(x["vet"] in ("rotation", "eclipsing-binary") for x in tones), len(left)),
            [dict(target="tic:%d" % x["tic"], stat=round(x["snr"], 1), unit="times the local amplitude", vet=x["vet"],
                  note="sector %d, %.3f c/d (%.2f min), %.1f ppm, centroid %.3f px per unit flux, BP-RP %s; harmonic %d of %.4f c/d; VSX %s, SIMBAD %s; shared with %s" % (
                      x["sector"], x["freq"], 1440 / x["freq"], 1e6 * x["amp"], x["offset"], "%.2f" % x["bp_rp"] if x.get("bp_rp") is not None else "-", x["harmonic"], x["f0"],
                      x.get("vsx") or "-", x.get("otype") or "-", ", ".join("TIC %d" % t for t in x["shared"][:5]) or "none")) for x in tones])
        print("recorded")


chirps_DOC = r"""Chirps at the nearest stars: every TESS twenty-second light curve the discrete-level search read (the stars within 30 pc, brighter than Tmag 15), its hour-long variations and its flares removed, cut into back-to-back windows of six hours for drifts up to 16 cycles a day per day and of an hour and a half beyond, each window's power at periods of 1-60 minutes, each frequency levelled by its median over the windows and each window by its median over the frequencies, and the windows summed along every straight frequency track whose drift lies between 1 and 60 cycles a day per day either way, as radio searches de-Doppler a drifting carrier; a beacon that sweeps its modulation, or a modulation whose frequency slides with a source's acceleration, gathers its power along one track, where stars hold their pulsation frequencies and a stationary tone belongs to the tone search; a sum's chance is its Gamma tail, a track that the same light curve's windows in shuffled order match belongs to bursts of power rather than to a sweep, a track that other stars of the same sector carry at p 1e-4 when read along it in absolute time, that the flux-weighted centroid carries as strongly as the flux, or that lies within 15 per cent in frequency and a factor of two in drift of such a track in any sector is the spacecraft's, one that vanishes once the band starts 8 cycles a day higher rides the star's own slower variability, and chirps laid into light curves measure the reach.

    python -I seti.py flashes chirps [record] [--rate-max 60] [--rate-min 1] [--p-max 1e-12] [--tmag-max 15] [--inject 0.0005] [--shuffles 100]"""


def chirps_prepare(path):
    """A twenty-second light curve as (TIC, sector, Tmag, days, relative flux): hour-long variations and flares with the half-hour after them removed, flares set to zero; None when too short."""
    from astropy.io import fits
    from scipy.ndimage import median_filter
    with fits.open(path) as h:
        H, d = h[0].header, h[1].data
        t, f, q = np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["QUALITY"])
        cad = np.asarray(d["CADENCENO"], int)
    ok = np.isfinite(t) & np.isfinite(f) & (q == 0)
    if ok.sum() < 20000:
        return None
    t, f, cad = t[ok], f[ok], cad[ok]
    r = f / np.median(f) - 1.0
    r = r - median_filter(r, size=181, mode="nearest")
    sig = 1.4826 * float(np.median(np.abs(r)))
    bad = np.zeros(len(r), bool)
    for i in np.flatnonzero(r > 5 * sig):
        bad[max(i - 3, 0):i + 90] = True
    n = int(cad.max() - cad.min() + 1)
    x, have = np.zeros(n), np.zeros(n, bool)
    x[cad - cad.min()] = np.where(bad, 0.0, r)
    have[cad - cad.min()] = ~bad
    dt = float(np.median(np.diff(t)))
    return int(H.get("TICID")), int(H.get("SECTOR")), float(H.get("TESSMAG") or 99), float(t[0]), dt, x, have


def chirps_spectrogram(x, have, dt, f_lo, f_hi, window_days):
    """Power of back-to-back windows at frequencies f_lo-f_hi (cycles a day), each frequency over its median in time and then each window over its median, with the windows' centres (days) and the frequency step; windows under 70 per cent filled left out."""
    w = int(round(window_days / dt))
    step = w
    freq = np.fft.rfftfreq(w, d=dt)
    band = (freq >= f_lo) & (freq <= f_hi)
    rows, centres = [], []
    hann = np.hanning(w)
    for a in range(0, len(x) - w + 1, step):
        if have[a:a + w].mean() < 0.7:
            continue
        p = np.abs(np.fft.rfft(x[a:a + w] * hann)) ** 2
        rows.append(p[band])
        centres.append((a + w / 2) * dt)
    P = np.array(rows)
    if len(P):
        # each frequency over its own median in time, as radio searches level each channel, so that steady lines and red noise drop out and a sweep, which passes each frequency briefly, stays
        P = P / np.maximum(np.median(P, axis=0, keepdims=True), 1e-30)
        P = P / np.maximum(np.median(P, axis=1, keepdims=True), 1e-30) * math.log(2)
    return P, np.array(centres), freq[band], 1.0 / window_days


def chirps_logp(acc, cnt):
    """The natural log of the chance that cnt unit exponentials sum to acc or more: the Gamma tail, by its leading term where the tail underflows."""
    from scipy.special import gammaln
    from scipy.stats import gamma
    lp = gamma.logsf(acc, np.maximum(cnt, 1))
    lead = -acc + (np.maximum(cnt, 1) - 1) * np.log(np.maximum(acc, 1e-300)) - gammaln(np.maximum(cnt, 1))
    return np.where(np.isfinite(lp), lp, lead)


def chirps_search(P, centres, df, rates):
    """Each straight track's summed power: for every drift rate the windows' power shifted to the track's frequency at the window's time, summed; the track of least chance as (log p, rate, bin)."""
    K, B = P.shape
    tc = centres - centres.mean()
    best = (np.inf, None, None)
    out = []
    for rate in rates:
        shift = np.round(rate * tc / df).astype(int)
        acc = np.zeros(B)
        cnt = np.zeros(B)
        for k in range(K):
            lo, hi = max(0, -shift[k]), min(B, B - shift[k])
            if hi > lo:
                acc[lo:hi] += P[k, lo + shift[k]:hi + shift[k]]
                cnt[lo:hi] += 1
        good = cnt >= max(3, K // 2)
        if not good.any():
            continue
        lp = np.where(good, chirps_logp(acc, cnt), 0.0)
        j = int(np.argmin(lp))
        out.append((float(lp[j]), float(rate), j, int(cnt[j])))
        if lp[j] < best[0]:
            best = (float(lp[j]), float(rate), j)
    return best, out


def chirps_one(args):
    """One light curve's strongest drifting track and strongest stationary one, as a dict, optionally with a chirp laid in first and with the strongest drifting track of each of SHUFFLES orderings of the windows."""
    path, f_lo, f_hi, rates, inject = args[:5]
    shuffles = args[5] if len(args) > 5 else 0
    c = chirps_prepare(path)
    if c is None:
        return None
    tic, sector, tmag, t0, dt, x, have = c
    if inject:
        amp, f0, rate = inject
        tt = np.arange(len(x)) * dt
        tm = tt.mean()
        x = x + np.where(have, amp * np.sin(2 * np.pi * (f0 * (tt - tm) + 0.5 * rate * (tt - tm) ** 2)), 0.0)
    best = dict(tic=tic, sector=sector, tmag=tmag, logp=0.0, rate=0.0, f=None, windows=0, logp_still=0.0, f_still=None, logp_shuffled=[], t_ref=None, window_days=None)
    rng = np.random.default_rng(tic % 100000)
    shuffled = np.zeros(shuffles)
    # each pass's windows are short enough that its fastest drift crosses no more than one frequency bin within a window
    for window_days, rs in rates:
        P, centres, freq, df = chirps_spectrogram(x, have, dt, f_lo, f_hi, window_days)
        if len(P) < 6:
            continue
        (lp, rate, j), _ = chirps_search(P, centres, df, rs)
        if j is not None and lp < best["logp"]:
            best.update(logp=lp, rate=rate, f=float(freq[j]), windows=len(P), t_ref=float(t0 + centres.mean()), window_days=window_days)
        # the windows' order shuffled keeps every burst of power and breaks every straight track
        for k in range(shuffles):
            shuffled[k] = min(shuffled[k], chirps_search(P[rng.permutation(len(P))], centres, df, rs)[0][0])
        if window_days == rates[0][0]:
            (lp0, _, j0), _ = chirps_search(P, centres, df, [0.0])
            best.update(logp_still=lp0, f_still=float(freq[j0]) if j0 is not None else None)
    best["logp_shuffled"] = shuffled.tolist()
    return best if best["windows"] else None


def chirps_track(args):
    """The chance of one light curve's power along a given track in absolute time (frequency f at time t_ref, drifting by rate), with the window length the track was found in."""
    path, f_lo, f_hi, f, rate, t_ref, window_days = args
    c = chirps_prepare(path)
    if c is None:
        return None
    tic, sector, tmag, t0, dt, x, have = c
    P, centres, freq, df = chirps_spectrogram(x, have, dt, f_lo, f_hi, window_days)
    if len(P) < 6:
        return None
    b = np.round((f + rate * (t0 + centres - t_ref) - freq[0]) / df).astype(int)
    ok = (b >= 0) & (b < P.shape[1])
    if ok.sum() < 6:
        return None
    acc = float(P[np.flatnonzero(ok), b[ok]].sum())
    return tic, float(chirps_logp(np.array([acc]), np.array([ok.sum()]))[0])


def chirps_centroid(args):
    """The chance of a light curve's flux and of each axis of its flux-weighted centroid along one track in absolute time, the flux's first."""
    from astropy.io import fits
    from scipy.ndimage import median_filter
    path, f_lo, f_hi, f, rate, t_ref, window_days = args
    with fits.open(path) as h:
        d = h[1].data
        t, fl, q = np.asarray(d["TIME"], float), np.asarray(d["PDCSAP_FLUX"], float), np.asarray(d["QUALITY"])
        cad = np.asarray(d["CADENCENO"], int)
        cx, cy = np.asarray(d["MOM_CENTR1"], float), np.asarray(d["MOM_CENTR2"], float)
    ok = np.isfinite(t) & np.isfinite(fl) & np.isfinite(cx) & np.isfinite(cy) & (q == 0)
    t, fl, cad, cx, cy = t[ok], fl[ok], cad[ok], cx[ok], cy[ok]
    dt = float(np.median(np.diff(t)))
    out = []
    for v in (fl / np.median(fl) - 1, cx - np.median(cx), cy - np.median(cy)):
        v = v - median_filter(v, size=181, mode="nearest")
        n = int(cad.max() - cad.min() + 1)
        x, have = np.zeros(n), np.zeros(n, bool)
        x[cad - cad.min()] = v
        have[cad - cad.min()] = True
        P, centres, freq, df = chirps_spectrogram(x, have, dt, f_lo, f_hi, window_days)
        b = np.round((f + rate * (t[0] + centres - t_ref) - freq[0]) / df).astype(int)
        good = (b >= 0) & (b < P.shape[1])
        out.append(float(chirps_logp(np.array([float(P[np.flatnonzero(good), b[good]].sum())]), np.array([good.sum()]))[0]))
    return out


def chirps_prepare_key(path):
    """A twenty-second light curve file's (TIC, sector) from its header."""
    from astropy.io import fits
    H = fits.getheader(path)
    return int(H.get("TICID")), int(H.get("SECTOR"))


def chirps_main(record=False, rate_max=60.0, rate_min=1.0, p_max=1e-12, tmag_max=15.0, inject=0.0005, shuffles=100, procs=8):
    import glob
    from concurrent.futures import ProcessPoolExecutor
    from astropy.io import fits
    files = [f for f in sorted(glob.glob(core.data("tess20", "*.fits"))) if (fits.getheader(f).get("TESSMAG") or 99) <= tmag_max]
    f_lo, f_hi = 24.0, 1440.0
    # six-hour windows for drifts up to 16 cycles a day per day, an hour and a half beyond; the step keeps the track within half a bin over 25 days
    span = 25.0
    slow = [r for r in np.arange(-16.0, 16.0 + 1e-9, 4.0 / span / 2) if rate_min <= abs(r) <= min(16.0, rate_max)]
    fast = [r for r in np.arange(-64.0, 64.0 + 1e-9, 16.0 / span / 2) if 16.0 < abs(r) <= rate_max]
    rates = [(0.25, slow), (0.0625, fast)] if fast else [(0.25, slow)]
    with ProcessPoolExecutor(procs) as ex:
        res = [x for x in ex.map(chirps_one, [(f, f_lo, f_hi, rates, None) for f in files], chunksize=4) if x]
    lim = math.log(p_max)
    hits = [x for x in res if x["logp"] <= lim]
    if hits:
        # each light curve below the threshold read again with its windows shuffled
        byname = {chirps_prepare_key(f): f for f in files}
        with ProcessPoolExecutor(procs) as ex:
            again = list(ex.map(chirps_one, [(byname[(x["tic"], x["sector"])], f_lo, f_hi, rates, None, shuffles) for x in hits], chunksize=1))
        for x, y in zip(hits, again):
            x["logp_shuffled"] = y["logp_shuffled"] if y else []
    keys = {chirps_prepare_key(f): f for f in files} if hits else {}
    for x in hits:
        x["vet"] = "chance-rate" if x["logp_shuffled"] and min(x["logp_shuffled"]) <= x["logp"] else None
        x["shared"] = []
    # a track that other stars of the same sector carry at p 1e-4 belongs to the spacecraft: every other light curve of the sector read along it
    for x in [x for x in hits if x["vet"] is None]:
        others = [f for (tic, sec), f in keys.items() if sec == x["sector"] and tic != x["tic"]]
        with ProcessPoolExecutor(procs) as ex:
            got = [g for g in ex.map(chirps_track, [(f, f_lo, f_hi, x["f"], x["rate"], x["t_ref"], x["window_days"]) for f in others], chunksize=4) if g]
        x["shared"] = [tic for tic, lp in got if lp <= math.log(1e-4)]
        x["sector_read"] = len(got)
        if x["shared"]:
            x["vet"] = "detector-wide"
    rest = [x for x in hits if x["vet"] is None]
    for x in rest:
        path = keys[(x["tic"], x["sector"])]
        # pointing jitter moves the image: a track at least as strong in the centroid as in the flux is the spacecraft's
        fl, col, row = chirps_centroid((path, f_lo, f_hi, x["f"], x["rate"], x["t_ref"], x["window_days"]))
        x["centroid"] = (col, row)
        if min(col, row) <= fl:
            x["vet"] = "detector-wide"
            continue
        # a track that vanishes once the band starts 8 cycles a day higher rode the leakage of the star's own slower variability
        y = chirps_one((path, f_lo + 8.0, f_hi, rates, None, 20))
        x["above_floor"] = (y["logp"], min(y["logp_shuffled"])) if y else None
        if y and y["logp"] >= min(y["logp_shuffled"]):
            x["vet"] = "variable-star"
    # a track like a spacecraft track of any sector, within 15 per cent in frequency and a factor of two in drift of the same sign, joins its family
    craft = [y for y in hits if y["vet"] == "detector-wide"]
    for x in [x for x in hits if x["vet"] is None]:
        x["family"] = [(y["tic"], y["sector"]) for y in craft if abs(x["f"] / y["f"] - 1) < 0.15 and x["rate"] * y["rate"] > 0 and 0.5 < x["rate"] / y["rate"] < 2]
        if x["family"]:
            x["vet"] = "detector-wide"
    # the reach: chirps of 0.05 per cent at 300 cycles a day sliding 20 cycles a day per day laid into twenty light curves
    rng = np.random.default_rng(3)
    pick = [files[i] for i in rng.choice(len(files), size=min(20, len(files)), replace=False)]
    with ProcessPoolExecutor(procs) as ex:
        inj = [x for x in ex.map(chirps_one, [(f, f_lo, f_hi, rates, (inject, 300.0, 20.0)) for f in pick], chunksize=1) if x]
    found = sum(1 for x in inj if x["logp"] <= lim and abs(x["rate"] - 20.0) <= 1.0 and abs(x["f"] - 300.0) <= 16.0)
    lps = np.array([x["logp"] for x in res])
    print("%d twenty-second light curves of stars within 30 pc at Tmag %g or brighter; drifting tracks of %g-%g cycles a day per day between 1 and 60 minutes: median best log p %.1f, %d below p %g: %d matched by their own windows shuffled, %d the spacecraft's (shared in the sector, in the centroid or of a spacecraft family), %d riding a star's slower variability, %d left; a chirp of %.2f per cent at 300 c/d drifting 20 c/d per day is found in %d of %d light curves" % (
        len(res), tmag_max, rate_min, rate_max, float(np.median(lps)), len(hits), p_max, sum(x["vet"] == "chance-rate" for x in hits), sum(x["vet"] == "detector-wide" for x in hits),
        sum(x["vet"] == "variable-star" for x in hits), sum(x["vet"] is None for x in hits), 100 * inject, found, len(inj)))
    for x in sorted(res, key=lambda x: x["logp"])[:15]:
        print("  TIC %-10d s%02d Tmag %5.2f  log p %7.1f at %+6.1f c/d per day from %7.2f c/d; shuffled best %s; stationary log p %7.1f; %s" % (
            x["tic"], x["sector"], x["tmag"], x["logp"], x["rate"], x["f"] or 0, "%.1f" % min(x["logp_shuffled"]) if x.get("logp_shuffled") else "-", x["logp_still"],
            x.get("vet") or ("" if x["logp"] > lim else "LEFT")))
    json.dump(dict(n=len(res), hits=hits, injected=inj), open(core.results("tess20_chirps.json"), "w"), indent=1)
    if record:
        core.record_run(
            "tess20_chirps", "Chirps at the nearest stars", "TESS SPOC twenty-second PDCSAP light curves of the stars within 30 pc (the discrete-level search's set)",
            "%d light curves at Tmag %g or brighter" % (len(res), tmag_max), os.path.abspath(__file__), "chirps", len(res), "light curves",
            dict(rate_max=rate_max, rate_min=rate_min, p_max=p_max, tmag_max=tmag_max, inject=inject, shuffles=shuffles),
            "the Gamma tail of each track's summed power over its windows' medians; the median light curve's best track has log p %.1f" % float(np.median(lps)),
            "a track drifting %g-%g cycles a day per day at p %g; a chirp of %.2f per cent at 300 c/d drifting 20 c/d per day is found in %d of %d light curves" % (rate_min, rate_max, p_max, 100 * inject, found, len(inj)),
            "%d tracks below p %g: %d matched by their own windows shuffled, %d the spacecraft's, %d riding a star's slower variability, %d left" % (
                len(hits), p_max, sum(x["vet"] == "chance-rate" for x in hits), sum(x["vet"] == "detector-wide" for x in hits), sum(x["vet"] == "variable-star" for x in hits),
                sum(x["vet"] is None for x in hits)),
            [dict(target="tic:%d" % x["tic"], stat=round(x["logp"], 1), unit="natural log of the track's chance", vet=x["vet"],
                  note="sector %d, Tmag %.2f, %+.1f c/d per day from %.2f c/d over %d windows; shuffled best log p %s; centroid log p %s; band 8 c/d higher %s; shared with %s; family %s" % (
                      x["sector"], x["tmag"], x["rate"], x["f"] or 0, x["windows"], "%.1f" % min(x["logp_shuffled"]) if x.get("logp_shuffled") else "-",
                      "%.1f/%.1f" % x["centroid"] if x.get("centroid") else "-", "%.1f against %.1f shuffled" % x["above_floor"] if x.get("above_floor") else "-",
                      ", ".join("TIC %d" % t for t in x["shared"]) or "none", ", ".join("TIC %d s%d" % fs for fs in x.get("family", [])) or "none"))
             for x in hits])
        print("recorded")


APPROACHES = {
    "chirps": (lambda: core.run_cli(chirps_main), chirps_DOC),
    "tones": (lambda: core.run_cli(tones_main), tones_DOC),
    "flareecho": (lambda: core.run_cli(flareecho_main), flareecho_DOC),
    "twomessengers": (lambda: core.run_cli(twomessengers_main), twomessengers_DOC),
    "vascobelt": (lambda: core.run_cli(vascobelt_main), vascobelt_DOC),
    "ebtwins": (lambda: core.run_cli(ebtwins_main), ebtwins_DOC),
    "tesswholedays": (lambda: core.run_cli(tesswholedays_main), tesswholedays_DOC),
    "ebwholedays": (lambda: core.run_cli(ebwholedays_main), ebwholedays_DOC),
    "copyebs": (lambda: core.run_cli(copyebs_main), copyebs_DOC),
    "wdvary": (lambda: core.run_cli(wdvary_main), wdvary_DOC),
    "unclassified": (lambda: core.run_cli(unclassified_main), unclassified_DOC),
    "flicker": (lambda: core.run_cli(flicker_main), flicker_DOC),
    "symflares": (lambda: core.run_cli(symflares_main), symflares_DOC),
    "sndeficit": (lambda: core.run_cli(sndeficit_main), sndeficit_DOC),
    "flaresync": (lambda: core.run_cli(flaresync_main), flaresync_DOC),
    "flarecopies": (lambda: core.run_cli(flarecopies_main), flarecopies_DOC),
    "steady": (lambda: core.run_cli(steady_main), steady_DOC),
    "neliota": (neliota_cli, neliota_DOC),
    "superflare": (superflare_cli, superflare_DOC),
    "clocks": (lambda: core.run_cli(clocks_main), clocks_DOC),
    "nova": (nova_cli, nova_DOC),
    "etz": (etz_cli, etz_DOC),
    "levels": (levels_cli, levels_DOC),
    "keplevels": (keplevels_cli, keplevels_DOC),
    "determinism": (determinism_cli, determinism_DOC),
    "psfflash": (psfflash_cli, psfflash_DOC),
    "clockwork": (clockwork_cli, clockwork_DOC),
    "tophat": (tophat_cli, tophat_DOC),
    "tess20": (tess20_cli, tess20_DOC),
    "toi": (toi_cli, toi_DOC),
    "kepler": (kepler_cli, kepler_DOC),
    "asteroids": (asteroids_cli, asteroids_DOC),
}