scripts/survey_meta.py

"""Field-coverage survey over all meta/ sidecars (report input, not pipeline)."""
import json
from collections import Counter
from concurrent.futures import ThreadPoolExecutor

import fsspec

BUCKET = "davanstrien/prelinger-films"
fs = fsspec.filesystem("hf")
paths = fs.glob(f"buckets/{BUCKET}/meta/*.json")
print("meta sidecars:", len(paths))


def one(p):
    try:
        with fs.open(p, "rb") as f:
            return json.loads(f.read())
    except Exception:
        return None


with ThreadPoolExecutor(32) as ex:
    metas = [m for m in ex.map(one, paths) if m]
print("read:", len(metas))

present = Counter()
for m in metas:
    for k, v in m.items():
        if v not in (None, ""):
            present[k] += 1
print("\nfield presence (of %d):" % len(metas))
for k, n in present.most_common():
    print(f"  {k:12s} {n:5d}  ({100*n/len(metas):.1f}%)")

print("\nlicenseurl values:")
for v, n in Counter(m.get("licenseurl") for m in metas).most_common():
    print(f"  {n:5d}  {v}")

years = Counter()
for m in metas:
    d = m.get("date")
    years[d[:4] if d else "MISSING"] += 1
print("\ndate: missing=%d, present=%d" % (years["MISSING"],
                                          len(metas) - years["MISSING"]))
print("year range:", min((y for y in years if y != "MISSING"), default=None),
      "-", max((y for y in years if y != "MISSING"), default=None))
print("no title:", sum(1 for m in metas if not m.get("title")))
print("no runtime:", sum(1 for m in metas if not m.get("runtime")))