release_tools/sync_public_model_cards.py

"""Render the complete HF card from the canonical GitHub README.

Preserve remote metadata, the featured film and frozen neural weights. Default
mode only prepares artifacts on D; --publish explicitly updates the HF card,
matching existing inference wrapper/docs and this small reproduction utility.
No forecasts are rerun, no media are regenerated and no neural modules change.
--docs-only limits uploads to an explicit documentation/figure allowlist.
"""
from __future__ import annotations

import argparse
import hashlib
import html
import json
import re
import subprocess
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
REPO = 'euler314/typhoon-predict'
MEDIA_REVISION = '87a6e366b42bb4cc0edc95d2c50c55fca21a2c93'
CACHE = '/Volumes/D/typhoon_predict/.cache/huggingface-intensity'
HF = 'https://huggingface.co/' + REPO
FILMS = ('mangkhut',)
REPORT_FIGURES = tuple('docs/report_figures/' + name for name in (
    'day_1_to_5.png',
    'fung_wong_pressure_comparison.png', 'fung_wong_track_pressure.png', 'fung_wong_observed_r34.png',
    'mangkhut_pressure_comparison.png', 'mangkhut_track_pressure.png', 'mangkhut_observed_r34.png',
    'jebi_pressure_comparison.png', 'jebi_track_pressure.png', 'jebi_observed_r34.png',
))
CURRENT_FIGURES = frozenset((
    'evaluation/released_daily/model_1_2_benchmark.png',
    'evaluation/released_daily/model_1_2_after_2024_benchmark.png',
    'evaluation/released_daily/model_1_2_before_2024_benchmark.png',
    'docs/trackformer_1_2_architecture.svg',
    *REPORT_FIGURES,
))
LEGACY_SYNC_FILES = (
    'models/trackformer_1_2_field/README.md',
    'models/trackformer_1_2_field/WIND_ESTIMATION.md',
    'models/trackformer_1_2_field/predict.py',
    'models/trackformer_1_2_field/plot_pressure.py',
    'models/trackformer_1_2_field/wind_estimation.py',
    'release_tools/sync_public_model_cards.py',
    'release_tools/test_public_model_cards.py',
    'release_tools/test_pressure_field_export.py',
    'release_tools/verify_pressure_field_export.py',
    'RELEASE_NOTES_TRACKFORMER_1_2.md',
    'release_tools/build_release_benchmark.py',
    'release_tools/plot_release_pressure_benchmark.py',
    'release_tools/plot_daily_storm_final.py',
    'release_tools/plot_model_announcement.py',
    'release_tools/build_deepmind_period_comparison.py',
    'release_tools/test_deepmind_period_comparison.py',
    'release_tools/import_deepmind_release_results.py',
    'release_tools/test_deepmind_release_results.py',
    'release_tools/build_mangkhut_gif.py',
    'release_tools/deepmind_daily_benchmark.py',
    'release_tools/test_deepmind_daily_benchmark.py',
    'docs/daily_storm_benchmark.md',
    'docs/intensity_benchmark.md',
    'docs/trackformer_1_2_evaluation.md',
    'docs/trackformer_1_2_training_data.md',
    'evaluation/training_data/trackformer_1_2_provenance.json',
    'release_tools/audit_training_data.py',
    'docs/deepmind_daily_benchmark.md',
    'docs/showcase_archive.md',
    'evaluation/README.md',
    'docs/trackformer_1_2_architecture.svg',
    'docs/trackformer_1_2_mangkhut.gif',
    'docs/trackformer_1_2_mangkhut.gif.json',
    'evaluation/daily_storm_final.json',
    'evaluation/released_daily/released_daily_benchmark.json',
    'evaluation/released_daily/released_daily_verification.json',
    'evaluation/intensity/intensity_final.json',
    'evaluation/intensity/verification.json',
    'evaluation/trackformer_1_2_vs_1_1_270_storms_bars.png',
    'evaluation/released_daily/pressure_comparison.png',
    'evaluation/released_daily/model_1_2_benchmark.png',
    'evaluation/released_daily/model_1_2_benchmark.json',
    'evaluation/released_daily/model_1_2_after_2024_benchmark.png',
    'evaluation/released_daily/model_1_2_after_2024_benchmark.json',
    'evaluation/released_daily/model_1_2_before_2024_benchmark.png',
    'evaluation/released_daily/model_1_2_before_2024_benchmark.json',
    'evaluation/deepmind_daily/benchmark.json',
    'evaluation/deepmind_daily/verification.json',
    'evaluation/deepmind_daily/publication_audit.json',
    'evaluation/deepmind_daily/protocol.json',
    'evaluation/deepmind_daily/case-manifest.json',
    'evaluation/deepmind_daily/canary.json',
    'evaluation/deepmind_daily/period_comparison.json',
    'docs/research_report_zh.md',
    'docs/report_figures/day_1_to_5.json',
    'docs/report_figures/provenance.json',
    *REPORT_FIGURES,
)
LEGACY_DOC_SYNC_FILES = (
    'models/trackformer_1_2_field/README.md',
    'models/trackformer_1_2_field/WIND_ESTIMATION.md',
    'RELEASE_NOTES_TRACKFORMER_1_2.md',
    'docs/daily_storm_benchmark.md', 'docs/intensity_benchmark.md',
    'docs/trackformer_1_2_evaluation.md', 'docs/trackformer_1_2_training_data.md',
    'docs/deepmind_daily_benchmark.md', 'docs/showcase_archive.md',
    'evaluation/README.md', 'docs/research_report_zh.md',
    'docs/report_figures/day_1_to_5.json', 'docs/report_figures/provenance.json',
    'docs/trackformer_1_2_architecture.svg', 'docs/trackformer_1_2_mangkhut.gif',
    *REPORT_FIGURES,
)


_DOC_INDEX = json.loads((ROOT / 'docs/archive/documentation-manifest.json').read_text())
_RETIRED_DOCS = frozenset(_DOC_INDEX['removed_paths'])
_COMPACT_DOCS = (
    'docs/REFERENCE.md',
    'docs/TrackFormer_1.2_Training_and_Reproducibility.pdf',
    'docs/archive/documentation-before-consolidation-20261030.zip',
    'docs/archive/documentation-manifest.json',
)
SYNC_FILES = tuple(dict.fromkeys(name for name in (*LEGACY_SYNC_FILES, *_COMPACT_DOCS)
                                if name not in _RETIRED_DOCS))
DOC_SYNC_FILES = tuple(dict.fromkeys(name for name in (*LEGACY_DOC_SYNC_FILES, *_COMPACT_DOCS)
                                    if name not in _RETIRED_DOCS))


def sha(path):
    with Path(path).open('rb') as stream:
        return hashlib.file_digest(stream, 'sha256').hexdigest()


def media_url(stem, poster=False):
    path = ('evaluation/trackformer_1_2_' + stem + '_video_poster.png'
            if poster else 'docs/trackformer_1_2_' + stem + '.mp4')
    return HF + '/resolve/' + MEDIA_REVISION + '/' + path


def player(stem, title):
    return ('<video controls preload="metadata" width="100%" '
            'aria-label="' + html.escape(title, quote=True) + '" '
            'poster="' + media_url(stem, True) + '" '
            'src="' + media_url(stem) + '"></video>')


def render_card(original, github, figure_revision='main'):
    metadata = re.match(r'\A---\n.*?\n---\n', original, re.S)
    if not metadata:
        raise ValueError('Preserve existing HF YAML metadata; missing header')
    if github.count('## See the forecast: Mangkhut (2018)\n') != 1:
        raise ValueError('Missing or ambiguous featured-film section')
    # The canonical GitHub README also carries valid YAML now. Retain exactly
    # the existing remote header rather than nesting a second YAML block.
    card = re.sub(r'\A---\n.*?\n---\n', '', github, count=1, flags=re.S).lstrip()
    for stem, alt in (('mangkhut', 'Trackformer 1.2 Mangkhut pressure and route forecast'),):
        old = '[![' + alt + '](docs/trackformer_1_2_' + stem + '.gif)](' + media_url(stem) + ')'
        if card.count(old) != 1:
            raise ValueError('Missing or ambiguous primary preview: ' + stem)
        card = card.replace(old, player(stem, alt), 1)
    # Relative GitHub links need HF-specific URLs. Keep evolving documentation
    # on main; pin unchanged scientific image bytes to the verified asset commit.
    def link(match):
        image, label, url = match.groups()
        if url.startswith(('https:', 'http:', '#', 'mailto:')):
            return match.group(0)
        path, _, fragment = url.partition('#')
        resolved = (ROOT / path).resolve()
        if not resolved.is_relative_to(ROOT) or not resolved.exists():
            raise ValueError('Invalid or missing relative public link: ' + path)
        revision = figure_revision if path in CURRENT_FIGURES else MEDIA_REVISION
        kind = ('resolve/' + revision if image or path in CURRENT_FIGURES
                else ('tree/main' if resolved.is_dir() else 'blob/main'))
        target = HF + '/' + kind + '/' + path + ('#' + fragment if fragment else '')
        return image + '[' + label + '](' + target + ')'
    card = re.sub(r'(!?)\[([^\[\]\n]*)\]\(([^)\s]+)\)', link, card)
    card = metadata.group(0) + '\n' + card
    if card.count('<video ') != 1:
        raise ValueError('Expected exactly one featured native player')
    for stem in FILMS:
        if card.count('src="' + media_url(stem) + '"') != 1:
            raise ValueError('Missing, repeated or stale player: ' + stem)
    if re.search(r'/resolve/main/docs/trackformer_1_2_[^"\s)]+\.mp4', card):
        raise ValueError('Unversioned movie source')
    return card


def frozen_identity(api, revision):
    manifest = json.loads((ROOT / 'models/trackformer_1_2_field/manifest.json').read_text())
    names = ['models/trackformer_1_2_field/weights.pt'] + [
        'models/trackformer_1_2_field/' + name for name in manifest['source_module_sha256']]
    entries = {i.path: i for i in api.get_paths_info(REPO, names, revision=revision)}
    weight = entries.get(names[0])
    actual = getattr(getattr(weight, 'lfs', None), 'sha256', None)
    if actual != manifest['inference_weights_sha256']:
        raise ValueError('Released weights are absent or differ from frozen identity')
    for name, expected in manifest['source_module_sha256'].items():
        path = ROOT / 'models/trackformer_1_2_field' / name
        data = path.read_bytes()
        entry = entries.get('models/trackformer_1_2_field/' + name)
        if sha(path) != expected or entry is None or entry.blob_id != hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest():
            raise ValueError('Frozen neural source differs: ' + name)
    return actual


def git_hf_token():
    """Use the existing credential helper for this exact HF host, never log it."""
    import os
    result = subprocess.run(['git', 'credential', 'fill'], cwd=ROOT,
        input='protocol=https\nhost=huggingface.co\n\n',
        text=True, capture_output=True, env={**os.environ, 'GIT_TERMINAL_PROMPT': '0'})
    fields = dict(line.split('=', 1) for line in result.stdout.splitlines() if '=' in line)
    if result.returncode or not fields.get('password'):
        raise ValueError('No existing Hugging Face git credential; authenticate before publishing')
    return fields['password']


def remote_matches(entry, path):
    if entry is None:
        return False
    lfs_sha = getattr(getattr(entry, 'lfs', None), 'sha256', None)
    if lfs_sha:
        return lfs_sha == sha(path)
    data = path.read_bytes()
    return entry.blob_id == hashlib.sha1(b'blob ' + str(len(data)).encode() + b'\0' + data).hexdigest()


def run(output, publish=False, docs_only=False, use_git_credential=False):
    # Lazy imports keep portable CI document tests independent of HF packages.
    from huggingface_hub import CommitOperationAdd, HfApi, hf_hub_download
    output = output.resolve()
    if not output.is_relative_to(Path('/Volumes/D')):
        raise ValueError('All generated documentation/cache artifacts must stay on D')
    output.mkdir(parents=True, exist_ok=True)
    api = HfApi(token=git_hf_token() if use_git_credential else None)
    parent = api.model_info(REPO).sha
    weights_before = frozen_identity(api, parent)
    original = Path(hf_hub_download(REPO, 'README.md', revision=parent, cache_dir=CACHE, token=api.token))
    github = (ROOT / 'README.md').read_text()
    card_path = output / 'README.md'
    card_path.write_text(render_card(original.read_text(), github))
    selected = DOC_SYNC_FILES if docs_only else SYNC_FILES
    files = {'README.md': card_path, **{name: ROOT / name for name in selected}}
    source_commit = subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip()
    receipt = {
        'status': 'prepared', 'repo_id': REPO, 'parent_commit': parent,
        'sync_scope': 'documentation_only' if docs_only else 'full_existing_release_sync',
        'github_source_commit': source_commit,
        'github_readme_sha256': sha(ROOT / 'README.md'),
        'original_hf_readme_sha256': sha(original),
        'files': {name: sha(path) for name, path in files.items()},
        'media_revision': MEDIA_REVISION, 'native_video_players': 1,
        'inference_weights_sha256': weights_before,
        'weights_changed': False, 'neural_modules_changed': False,
        'forecast_arrays_changed': False, 'videos_changed': False,
    }
    if publish:
        dirty = subprocess.check_output(['git', 'status', '--porcelain', '--', 'README.md', *selected], cwd=ROOT, text=True)
        if dirty:
            raise ValueError('Commit canonical README and matching source files before publishing')
        # Publish scientific assets first, then pin the new figures to that
        # immutable commit in the card. Never pin new bytes to the old movie
        # revision or let a missing future image render as a stale figure.
        existing = {entry.path: entry for entry in api.get_paths_info(REPO, list(selected), revision=parent)}
        changed = [name for name in selected if not remote_matches(existing.get(name), ROOT/name)]
        figure_revision = parent
        if changed:
            assets = api.create_commit(repo_id=REPO, repo_type='model', parent_commit=parent,
                commit_message='Align 1.2 research documentation and report figures',
                operations=[CommitOperationAdd(path_in_repo=name, path_or_fileobj=str(ROOT/name)) for name in changed])
            figure_revision = assets.oid
        card_path.write_text(render_card(original.read_text(), github, figure_revision))
        receipt['files']['README.md'] = sha(card_path)
        receipt['figure_revision'] = figure_revision
        receipt['uploaded_asset_paths'] = changed
        committed = api.create_commit(repo_id=REPO, repo_type='model', parent_commit=figure_revision,
            commit_message='Match released 1.2 model card to the Chinese research report',
            operations=[CommitOperationAdd(path_in_repo='README.md', path_or_fileobj=str(card_path))])
        for name, expected in receipt['files'].items():
            saved = hf_hub_download(REPO, name, revision=committed.oid, cache_dir=CACHE, token=api.token)
            if sha(saved) != expected:
                raise ValueError('Published documentation/source differs: ' + name)
        if frozen_identity(api, committed.oid) != weights_before:
            raise ValueError('Frozen identity changed during documentation publication')
        receipt.update(status='published_verified', commit=committed.oid,
            commit_url=str(committed.commit_url), all_remote_files_sha256_verified=True)
    (output / 'publication.json').write_text(json.dumps(receipt, indent=2) + '\n')
    print(json.dumps({key: value for key, value in receipt.items() if key != 'files'}, indent=2))


if __name__ == '__main__':
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument('--output', type=Path, required=True)
    parser.add_argument('--publish', action='store_true')
    parser.add_argument('--docs-only', action='store_true', help='Upload only whitelisted Markdown and report assets')
    parser.add_argument('--use-git-credential', action='store_true', help='Use the existing HF credential helper without saving a token')
    args = parser.parse_args()
    run(args.output, args.publish, args.docs_only, args.use_git_credential)