diff --git a/.gitattributes b/.gitattributes index 3e80b98ac7c4be3db6c162728d175249368ebfe6..1a368652b8c0e3ced700a01e14d6328fddd28eb9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ *.zip filter=lfs diff=lfs merge=lfs -text *.pdf filter=lfs diff=lfs merge=lfs -text *.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/README.md b/README.md index e16b7e6eb172b5f8070c9f027947fb9ef41dfa12..6d2f2d5ad0f6604cc6a563502f6a7554d253b1d0 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,10 @@ and writes the poster as standalone HTML; the Space renders it to PNG. to finish a poster. The demo asks for a 10-minute GPU step first and falls back to shorter steps. - API: call `/analyze` once (PDF + settings), then `/step` repeatedly until the status starts with `Done`, `Paused` or `Stopped`; each step is its own request so the ZeroGPU token stays valid. -- The examples on the page were generated by the same checkpoint with the default instruction and settings. +- The example cards are five cases of the RAP-150 benchmark. Picking one loads the paper, the exact instruction and + layout specification used in the paper's evaluation and the canvas size of the reference poster; **Generate poster** + then runs the model on the very prompt we evaluated, so the result should match the shown poster. Editing any field + switches back to the regular pipeline (your PDF is parsed and a layout specification is derived automatically). - Research demo: scanned PDFs are not supported, uploads are temporary, and every statement, number and caption should be checked before a poster is used. diff --git a/demo_ui.py b/demo_ui.py index d792f8fd7ea315ebe9a430873c257fe452a73526..3b2b5b8cf4bca2b2de2ff2a7b003cddfe0ac8ab3 100644 --- a/demo_ui.py +++ b/demo_ui.py @@ -35,22 +35,65 @@ footer {display: none !important;} ''' def load_examples(): - """examples//meta.json + paper.pdf + poster.png (+ poster.html, poster_bundle.zip).""" + """examples//: meta.json, paper.pdf, poster.png, poster.html, prompt.json (the exact benchmark prompt), + thumbnails/ (what the model sees) and assets/ (full-resolution crops for rendering).""" + import hashlib out = [] for d in sorted((ROOT / 'examples').glob('*/')): meta = d / 'meta.json' if not meta.exists() or not (d / 'poster.png').exists() or not (d / 'paper.pdf').exists(): continue m = json.loads(meta.read_text(encoding='utf-8')) - out.append({'dir': d, 'title': m.get('title', d.name), 'short': m.get('short', m.get('title', d.name))[:40], 'venue': m.get('venue', ''), 'instruction': m.get('instruction', DEFAULT_INSTRUCTION), - 'canvas': m.get('canvas', 'Landscape · 3:2'), 'columns': m.get('columns', 3), 'accent': m.get('accent', '#584287')}) + out.append({'dir': d, 'title': m.get('title', d.name), 'short': m.get('short', m.get('title', d.name))[:40], 'venue': m.get('venue', ''), + 'paper_id': m.get('paper_id', d.name), 'instruction': m.get('instruction', DEFAULT_INSTRUCTION), 'spec': m.get('spec', ''), + 'canvas': m.get('canvas', 'Landscape · 3:2'), 'columns': m.get('columns', 3), 'accent': m.get('accent', '#584287'), + 'prompt': d / 'prompt.json' if (d / 'prompt.json').exists() else None, + 'pdf_sha': hashlib.sha256((d / 'paper.pdf').read_bytes()).hexdigest()}) return out def fileout(path): return gr.update(value=str(path), visible=True) if path else gr.update(value=None, visible=False) def build(model, processor): + def match_example(pdf, instruction, custom_spec): + """The exact benchmark prompt is used when the PDF is one of the examples and instruction + spec are unchanged.""" + import hashlib + try: + sha = hashlib.sha256(Path(pdf).read_bytes()).hexdigest() + except OSError: + return None + for ex in examples: + if ex['prompt'] and ex['pdf_sha'] == sha and instruction.strip() == ex['instruction'].strip() and custom_spec.strip() == ex['spec'].strip(): + return ex + return None + + def analyze_example(ex): + """Reproduce a benchmark case: the stored prompt (paper text, asset list, thumbnails, instruction, layout spec).""" + d = ex['dir']; pr = json.loads((d / 'prompt.json').read_text(encoding='utf-8')) + system, user = pr['messages'][0]['content'], pr['messages'][1]['content'] + parts = user.split(''); images = [str(d / im) for im in pr['images']] + content = [] + for k, text in enumerate(parts): + if text: content.append({'type': 'text', 'text': text}) + if k < len(images): content.append({'type': 'image', 'image': images[k]}) + messages = [{'role': 'system', 'content': [{'type': 'text', 'text': system}]}, {'role': 'user', 'content': content}] + assets = [] + for im in pr['images']: + aid = Path(im).stem; f = d / 'assets' / f'{aid}.png' + kind = aid.rsplit('_', 1)[0] + assets.append({'id': aid, 'kind': kind, 'file': str(f if f.exists() else d / im), 'thumb': str(d / im), 'page': None}) + w, h = (int(v) for v in re.match(r'(\d+)x(\d+)', ex['canvas']).groups()) if re.match(r'\d+x\d+', ex['canvas']) else CANVASES.get(ex['canvas'], (2400, 1600)) + job = Path(tempfile.mkdtemp(prefix='example_', dir=ARTIFACTS)) + return {'title': ex['title'], 'authors': '', 'assets': assets, 'instruction': ex['instruction'], 'spec': ex['spec'], 'canvas': [w, h], + 'messages_pre': messages, 'job': str(job), 'example': ex['paper_id'], 'sections': [], 'body': ''} + def analyze(pdf, instruction, canvas, accent, columns, custom_spec): + ex = match_example(pdf, instruction, custom_spec) + if ex is not None: + p = analyze_example(ex) + inputs = processor.apply_chat_template(chat(p), tokenize=True, add_generation_prompt=True, enable_thinking=False, return_dict=True, return_tensors='pt') + p['input_tokens'] = int(inputs['input_ids'].shape[1]); p['continuation_ids'] = []; p['raw'] = ''; p['done'] = False + return p job = Path(tempfile.mkdtemp(prefix='paper_', dir=ARTIFACTS)) p = prepare(pdf, job, instruction, canvas, accent, columns, custom_spec) budget = 22000 @@ -103,7 +146,7 @@ def build(model, processor): HIDE = gr.update(value=None, visible=False) def gallery_of(p): - return [(a['thumb'], f"{a['id']} · page {a['page']}") for a in p['assets']] + return [(a['thumb'], f"{a['id']} · page {a['page']}" if a.get('page') else a['id']) for a in p['assets']] def analyze_evt(pdf, instruction, canvas, accent, columns, custom_spec, progress=gr.Progress()): """Request 1 of a run: parse the PDF on CPU. GPU steps run in their own requests (see step).""" @@ -115,7 +158,9 @@ def build(model, processor): except (ValueError, RuntimeError) as e: raise gr.Error(str(e)) from None p = dict(p, step=0, paused=False) - return (p, f"Parsed **{p['title'][:90]}** · {len(p['assets'])} figures/tables found · generating (step 1)…", + head = (f"Benchmark case **{p['title'][:80]}** · exact test prompt, {len(p['assets'])} figures/tables" if p.get('example') + else f"Parsed **{p['title'][:90]}** · {len(p['assets'])} figures/tables found") + return (p, f"{head} · generating (step 1)…", None, HIDE, HIDE, gallery_of(p), p['spec'], '', gr.update(visible=False)) def step(p, progress=gr.Progress()): @@ -166,8 +211,8 @@ def build(model, processor): def pick_example(evt: gr.SelectData): ex = examples[evt.index]; d = ex['dir'] html = d / 'poster.html'; archive = d / 'poster_bundle.zip' - return (str(d / 'paper.pdf'), ex['instruction'], ex['canvas'], ex['accent'], ex['columns'], - f"Example · {ex['title']}" + (f" ({ex['venue']})" if ex['venue'] else ''), + return (str(d / 'paper.pdf'), ex['instruction'], ex['canvas'], ex['accent'], ex['columns'], ex['spec'], + f"Benchmark case · {ex['title']}" + (f" ({ex['venue']})" if ex['venue'] else '') + ' · click **Generate poster** to reproduce it', str(d / 'poster.png'), fileout(html if html.exists() else None), fileout(archive if archive.exists() else None)) with gr.Blocks(title='RAP · Paper to Poster', css=CSS, delete_cache=(3600, 86400), @@ -182,11 +227,11 @@ def build(model, processor): pdf = gr.File(label='Paper PDF', file_types=['.pdf'], type='filepath', height=110) instruction = gr.Textbox(label='Instruction', value=DEFAULT_INSTRUCTION, lines=3) with gr.Row(): - canvas = gr.Dropdown(list(CANVASES), value='Landscape · 3:2', label='Canvas', scale=2) + canvas = gr.Dropdown(list(CANVASES), value='Landscape · 3:2', label='Canvas (preset or WxH)', allow_custom_value=True, scale=2) columns = gr.Slider(1, 4, value=3, step=1, label='Columns', scale=2) accent = gr.ColorPicker(value='#584287', label='Accent', scale=1) with gr.Accordion('Custom layout specification (optional)', open=False): - custom_spec = gr.Textbox(show_label=False, placeholder='Leave empty to let the demo plan the layout.', lines=4) + custom_spec = gr.Textbox(show_label=False, placeholder='Leave empty to let the demo plan the layout. Benchmark cases fill in their exact layout specification here.', lines=4) run_btn = gr.Button('Generate poster', variant='primary', size='lg') cont_btn = gr.Button('Continue generation', variant='secondary', visible=False) gr.Markdown('Generation runs in ZeroGPU steps of up to 120 s; a full poster usually takes 3–6 steps. ' @@ -221,5 +266,5 @@ def build(model, processor): for _ in range(MAX_STEPS): ev = ev.then(step, [state], outs, api_name=False, concurrency_limit=1) if examples: - ex_gallery.select(pick_example, None, [pdf, instruction, canvas, accent, columns, status, poster, html_download, bundle], api_name=False) + ex_gallery.select(pick_example, None, [pdf, instruction, canvas, accent, columns, custom_spec, status, poster, html_download, bundle], api_name=False) return demo, ARTIFACTS diff --git a/examples/01_iclr_2023_11250/assets/figure_1.png b/examples/01_iclr_2023_11250/assets/figure_1.png new file mode 100755 index 0000000000000000000000000000000000000000..11dc6ebce0fb00231754ae4b54c1fdba5f796546 --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/figure_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d58dc38c413e659544aeb80476b01cbd2326a5b889e6aa5fa05edb9f1e2b094 +size 113781 diff --git a/examples/01_iclr_2023_11250/assets/figure_2.png b/examples/01_iclr_2023_11250/assets/figure_2.png new file mode 100755 index 0000000000000000000000000000000000000000..cfc9c7033e32fa34048f02f0b8eb8ed70d4f2b65 --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/figure_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:310d49680c5233218c2dc40b6f3f58158703f07a5ee5b4962016c49d32bd1016 +size 95675 diff --git a/examples/01_iclr_2023_11250/assets/figure_3.png b/examples/01_iclr_2023_11250/assets/figure_3.png new file mode 100755 index 0000000000000000000000000000000000000000..d84ccc171a61ff80c1b4b18ba2211852dde38095 --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/figure_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e017b757e8244e72ff2a8827bfe85129a00de6f0767e084f146205081ea3ac1 +size 32239 diff --git a/examples/01_iclr_2023_11250/assets/figure_4.png b/examples/01_iclr_2023_11250/assets/figure_4.png new file mode 100755 index 0000000000000000000000000000000000000000..7a99c7934e2f194664bcc938194bcefeab208338 --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/figure_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0de7463d71aa2f1d24927856b013bb346693f50534792b63e8ed0837781af70 +size 82101 diff --git a/examples/01_iclr_2023_11250/assets/figure_5.png b/examples/01_iclr_2023_11250/assets/figure_5.png new file mode 100755 index 0000000000000000000000000000000000000000..968fba80cd30dac6c687395a7e4ed7d9ebf9e6a4 --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/figure_5.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a395a9c68dfad4582a36652ce5a2e8083e19b955099b9d9ca66db53394fdc58 +size 45820 diff --git a/examples/01_iclr_2023_11250/assets/figure_6.png b/examples/01_iclr_2023_11250/assets/figure_6.png new file mode 100755 index 0000000000000000000000000000000000000000..379bff81fb1fe4ac3b14158d21577b576e72f06b --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/figure_6.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9583e866061306c36a394b6668657b2876ce1beeb5602d1916f1f558f8d33fa3 +size 117342 diff --git a/examples/01_iclr_2023_11250/assets/figure_7.png b/examples/01_iclr_2023_11250/assets/figure_7.png new file mode 100755 index 0000000000000000000000000000000000000000..5dd04ecc563badb71aa77db1809d8eac2d04131c --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/figure_7.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d77b39fb8f0a7be564a27e7864dcac4725bd7c513f9ab55f0bc11aa019ee118a +size 54370 diff --git a/examples/01_iclr_2023_11250/assets/formula_1.png b/examples/01_iclr_2023_11250/assets/formula_1.png new file mode 100755 index 0000000000000000000000000000000000000000..d47b493d1e124a2dc20e78153465f637b04440d0 --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/formula_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86b74a79513dd3334645a95991baefbdf9779102df192cd4cb4d3dc7c819220b +size 9977 diff --git a/examples/01_iclr_2023_11250/assets/formula_2.png b/examples/01_iclr_2023_11250/assets/formula_2.png new file mode 100755 index 0000000000000000000000000000000000000000..937cf79f97b082092bb273ef7a714e2a68194844 --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/formula_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6c9f8b33a42de0a81a3c9ce60bccfdc65872653e9d768b6fa58a4bb15935571a +size 12886 diff --git a/examples/01_iclr_2023_11250/assets/table_1.png b/examples/01_iclr_2023_11250/assets/table_1.png new file mode 100755 index 0000000000000000000000000000000000000000..37de36a095ee3abe20904f6587826351ad31f0a7 --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/table_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fbba8ec43038b6d54abc642c15eca138b91a38efccb186e081f22ad728a285f7 +size 106639 diff --git a/examples/01_iclr_2023_11250/assets/table_2.png b/examples/01_iclr_2023_11250/assets/table_2.png new file mode 100755 index 0000000000000000000000000000000000000000..c94a5eb23f091c016172ee35a7dff54cfd35ef03 --- /dev/null +++ b/examples/01_iclr_2023_11250/assets/table_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:915c1342a5f01bda64a68816ba6d17df850860d68755175ddd62daf8c1be5150 +size 22094 diff --git a/examples/01_iclr_2023_11250/meta.json b/examples/01_iclr_2023_11250/meta.json index 9230d46a76bf6f71683259d44a0efe4d52354e2a..e546f3cae70b2bf7d96f9fa6fadc657ae2010e1a 100644 --- a/examples/01_iclr_2023_11250/meta.json +++ b/examples/01_iclr_2023_11250/meta.json @@ -1,10 +1,11 @@ { - "title": "MAESTRO: Open-Ended Environment Design for Multi-Agent Reinforcement Learning", + "title": "MAESTRO: OPEN-ENDED ENVIRONMENT DESIGN FOR MULTI-AGENT REINFORCEMENT LEARNING", "short": "MAESTRO · ICLR 2023", "venue": "ICLR 2023", "paper_id": "ICLR_2023_11250", - "instruction": "Create a clear academic poster with a strong title, a short motivation, the main method, key quantitative results, and takeaways. Prioritize the most informative figures. Use concise text, readable fonts, and generous spacing. Do not invent results.", - "canvas": "Landscape · 3:2", + "instruction": "Design spec: canvas 2500x1666px; visuals should dominate and accent colour #FFD700.", + "spec": "Canvas 2500x1666 landscape, 3 columns (21% | 54% | 21%), balanced (31 blocks). Header band (30% tall): title 84px #FFFFFF, authors line, 2 logo slots. Column 1: [top 26%; 1 text; h 4%] -> Goal - Provide a sequence of levels and co-players to (top 31%; 3 text; h 24%) -> Why this matters? (top 55%; 2 text; h 40%) -> MAESTRO (top 63%; 4 text; h 27%). Column 2: [top 28%; figure_1 (w 84%); h 31%] -> Emerging Complexity in LaserTag and Multi-Car Racing (top 61%; figure_2 (full column) -> 1 figure slot, 1 logo slot; boxed; h 38%). Column 3: Evaluation Against Specialist Agents (top 33%; figure_5 (full column); h 9%) -> Emergent Curriculum Analysis (top 45%; figure_6 (full column) -> 2 text -> figure_7 (w 89%) -> 1 text; h 42%). Figures used: figure_3, figure_1, figure_5, figure_6, figure_2, figure_7. Style: #1A1A2E background, Helvetica Neue, headers #FFFFFF 24px, body 19px; 1 panels: rgba(255,255,255,0.3) x1 (rule).", + "canvas": "2500x1666", "columns": 3, "accent": "#584287" } \ No newline at end of file diff --git a/examples/01_iclr_2023_11250/prompt.json b/examples/01_iclr_2023_11250/prompt.json new file mode 100644 index 0000000000000000000000000000000000000000..6f9ae2e76829469c4b65f51a1ce248b77c53b73e --- /dev/null +++ b/examples/01_iclr_2023_11250/prompt.json @@ -0,0 +1 @@ +{"messages": [{"role": "system", "content": "You are an expert academic poster designer. Given a paper and a design instruction, produce the poster as a standalone HTML document: a - - -
-
-
-
-
-
NExT-GPT: Any-to-Any Multimodal LLM
-
Shengqiong Wu*, Hao Fei*, Leigang Qu, Wei Ji, Tat-Seng Chua National University of Singapore
-
National University of Singapore
-
Lightweight Multimodal Alignment Learning
-
Demonstrations
-
Overview
-
➤ Encoding-side LLM-centric Alignment: We design a type of learnable concept tokens to hierarchically aggregate the grid-level features into semantic concept tokens via a grouping mechanism, and then the conceptual representation is fed into LLM.
-
➤ Motivation: Current AI systems still struggle to match human abilities in handling diverse modalities like text, images, audio, and video.
-
➤ NExT-GPT can comprehend and reason contents across various modalities in any combination, as shown below.
-
-
Highlights
-
-
➤ We present an end-to-end general-purpose any-to-any MM-LLM, named NExT-GPT, capable of semantic understanding and reasoning and generation of free input and output combinations of text, image, video, and audio.
-
-
➤ Decoding-side Instruction-following Alignment: We design several types of special tokens as signal tokens, which implicitly carry rich and flexible instructions for the downstream diffusion model.
-
-
➤ We introduce lightweight alignment learning techniques, the LLM-centric alignment at the encoding side, and the instruction-following alignment at the decoding side, efficiently requiring only minimal parameter adjustments (only 1% params) while maintaining highly effective semantic alignment.
-
➤ Modality-switching Instruction Tuning: We propose a novel Modality-switching Instruction Tuning (MosIT) approach. MosIT not only supports complex cross-modal understanding and reasoning but also enables sophisticated multimodal content generation.
-
-
-
➤ We propose a modality-switching instruction tuning (MosIT) approach and manually curate a high-quality MosIT dataset comprising 5,000 samples to facilitate the development of any-to-any MM-LLMs.
-
➤ Extensive experiments on 15 benchmarks covering multimodal understanding and generation demonstrate the superiority of NExT-GPT, achieving SOTA performance in 11 benchmarks and competitive results in the remaining ones.
-
- - \ No newline at end of file diff --git a/examples/04_icml_2024_34200/poster.png b/examples/04_icml_2024_34200/poster.png deleted file mode 100644 index 582386209ec3c18eeb1341bce542367e96994b85..0000000000000000000000000000000000000000 --- a/examples/04_icml_2024_34200/poster.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e4355081b942818a70f5a0f1c1a09280833d45ee2b1a79fc28faf7de559e2845 -size 763496 diff --git a/examples/04_icml_2025_15371/assets/figure_1.png b/examples/04_icml_2025_15371/assets/figure_1.png new file mode 100755 index 0000000000000000000000000000000000000000..dc59459c28fb10a37b17dc7bf4252b9b0dc2729f --- /dev/null +++ b/examples/04_icml_2025_15371/assets/figure_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d0393d5f3966baa568c275ea05f6bfa9f5426d400affa3eb57d080623098202c +size 123245 diff --git a/examples/04_icml_2025_15371/assets/figure_2.png b/examples/04_icml_2025_15371/assets/figure_2.png new file mode 100755 index 0000000000000000000000000000000000000000..c919125ec1812ea89c9df02b3ca8f9041132ab58 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/figure_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99cd5696b602a030f50ccc1910932fdf266d9cc7e649664cd803701870f02e81 +size 38170 diff --git a/examples/04_icml_2025_15371/assets/figure_3.png b/examples/04_icml_2025_15371/assets/figure_3.png new file mode 100755 index 0000000000000000000000000000000000000000..8ac1f4c6cecb120a341a427c98b76bb67bc7a7c6 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/figure_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8534dcb38e4022856f0498d11be77d18127ca98eb5128a8e5e55f0a3bcad9a7f +size 125675 diff --git a/examples/04_icml_2025_15371/assets/figure_4.png b/examples/04_icml_2025_15371/assets/figure_4.png new file mode 100755 index 0000000000000000000000000000000000000000..cb8fa62db0914c406b459877e3bfa78f94b1e571 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/figure_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b160f49075f52985ed58f2f688df0d0d5262db9cb0152676c2cb1030e5c8d57 +size 31580 diff --git a/examples/04_icml_2025_15371/assets/figure_5.png b/examples/04_icml_2025_15371/assets/figure_5.png new file mode 100755 index 0000000000000000000000000000000000000000..02fb72a886ea2d2e90860e1a7ed2679fe1bb4b16 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/figure_5.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2334f98e9a5f61fa09cfabff2167130bbb52e85993a41d36fcdacd65095dc13 +size 195355 diff --git a/examples/04_icml_2025_15371/assets/figure_6.png b/examples/04_icml_2025_15371/assets/figure_6.png new file mode 100755 index 0000000000000000000000000000000000000000..59a343de9d6223d07fe22991392e05c8c4c055af --- /dev/null +++ b/examples/04_icml_2025_15371/assets/figure_6.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:be12fa1bc570c01b736de2f859c39c2866c714d8eea94e48c4f537c2873e0a81 +size 51642 diff --git a/examples/04_icml_2025_15371/assets/figure_7.png b/examples/04_icml_2025_15371/assets/figure_7.png new file mode 100755 index 0000000000000000000000000000000000000000..1c1043be0cccba1bce952976f693f0c1f4c83eae --- /dev/null +++ b/examples/04_icml_2025_15371/assets/figure_7.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:597f190829b0f4b764e5a33825eaddc17b430fc660eb6a6027d130a287da1f45 +size 185620 diff --git a/examples/04_icml_2025_15371/assets/figure_8.png b/examples/04_icml_2025_15371/assets/figure_8.png new file mode 100755 index 0000000000000000000000000000000000000000..282a3941d94afad0057d6d04e0ec1c6ccba68514 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/figure_8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0838e994809baced3328d7af3ee29ee3802f3c586d407d144f6c78eaa89ea479 +size 52551 diff --git a/examples/04_icml_2025_15371/assets/figure_9.png b/examples/04_icml_2025_15371/assets/figure_9.png new file mode 100755 index 0000000000000000000000000000000000000000..c96095613646e6b2c73ac060e9566de18eb7b227 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/figure_9.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e4eb4b623f515d7a9b3fddec26a9a1bd0c3e7638ab00eb78746b46a13cc6326d +size 102726 diff --git a/examples/04_icml_2025_15371/assets/formula_1.png b/examples/04_icml_2025_15371/assets/formula_1.png new file mode 100755 index 0000000000000000000000000000000000000000..da9ffc657d851bba469253f7b3662578e3694769 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/formula_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e2d18ee9a85db8778478f4830ac0ff9e9438bf8f723da53dd0e4af559264c510 +size 16917 diff --git a/examples/04_icml_2025_15371/assets/formula_2.png b/examples/04_icml_2025_15371/assets/formula_2.png new file mode 100755 index 0000000000000000000000000000000000000000..e7c3867a0aad9c061751b88cda3dda895efc817e --- /dev/null +++ b/examples/04_icml_2025_15371/assets/formula_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b310c865d1b21d54abc3f3354823f3649c1fbd6bde479fedcee1c9ee3bf10024 +size 3857 diff --git a/examples/04_icml_2025_15371/assets/formula_3.png b/examples/04_icml_2025_15371/assets/formula_3.png new file mode 100755 index 0000000000000000000000000000000000000000..7f00eb4e82f4f6adf929edd01cb6c79a7393b152 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/formula_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:48d3c88c8aa8ffabd10038fe731f54e9a35a01aad12abc4de9dc3474bd4aec1b +size 6458 diff --git a/examples/04_icml_2025_15371/assets/formula_4.png b/examples/04_icml_2025_15371/assets/formula_4.png new file mode 100755 index 0000000000000000000000000000000000000000..dec4fd434cd7787fc9ada20a11d717621270b281 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/formula_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37ff212a1bc38e934005cd0b4e722628b8403130304a1d56eb40e44843ff9642 +size 4385 diff --git a/examples/04_icml_2025_15371/assets/formula_5.png b/examples/04_icml_2025_15371/assets/formula_5.png new file mode 100755 index 0000000000000000000000000000000000000000..fdbb2ff259db957526fa66c0fce358ded7c296ac --- /dev/null +++ b/examples/04_icml_2025_15371/assets/formula_5.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29900b181edc05244848c240660861c18bbf250f1f67b0b1fe57eb8dd5cdc4d5 +size 6221 diff --git a/examples/04_icml_2025_15371/assets/table_1.png b/examples/04_icml_2025_15371/assets/table_1.png new file mode 100755 index 0000000000000000000000000000000000000000..1608211f4dd88d30c65cb2e732a9dc584170d0f7 --- /dev/null +++ b/examples/04_icml_2025_15371/assets/table_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:66f1aafb907058ab470573cf20e0e14955df372e38bd5650e8a01258bf911870 +size 126296 diff --git a/examples/04_icml_2025_15371/meta.json b/examples/04_icml_2025_15371/meta.json new file mode 100644 index 0000000000000000000000000000000000000000..15b456e8817392946caa6857975091e089bf4ede --- /dev/null +++ b/examples/04_icml_2025_15371/meta.json @@ -0,0 +1,11 @@ +{ + "title": "The Disparate Benefits of Deep Ensembles", + "short": "Deep Ensembles · ICML 2025", + "venue": "ICML 2025", + "paper_id": "ICML_2025_15371", + "instruction": "3439x2306px. figure-heavy, let the visuals lead, #FFF background, information-dense, it should feel polished, three-column layout.", + "spec": "Canvas 3439x2306 landscape, 3 columns (28% | 33% | 33%), balanced (34 blocks). Header band (19% tall): title 102px #fff on #1B3A5C band, authors line, 3 logo slots. Column 1: SUMMARY (top 22%; 4 text; h 34%) -> BACKGROUND (top 58%; 1 text -> formula_1 (w 73%) -> 2 text, 1 figure slot, 1 logo slot; boxed; h 41%). Column 2: [top 22%; figure_1 (full column) -> 1 text, table_1 (w 56%), 1 figure slot; boxed; h 34%] -> MITIGATING THE EFFECT (top 57%; 2 text -> figure_8 (full column); h 41%). Column 3: [top 22%; 1 text, figure_2 (w 66%) -> 1 text -> figure_3 (full column) -> 1 text -> figure_7 (full column); h 76%]. Figures used: figure_2, figure_1, table_1, figure_3, formula_1, figure_8, figure_7. Style: white background, Helvetica Neue, headers #fff 40px, body 36px; 10 panels: #fff x5 (bordered), #4A90D9 x5.", + "canvas": "3439x2306", + "columns": 3, + "accent": "#584287" +} \ No newline at end of file diff --git a/examples/04_icml_2025_15371/paper.pdf b/examples/04_icml_2025_15371/paper.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d9c6fe9b8e55c07d4824631b7df7c4aa3f9b7464 --- /dev/null +++ b/examples/04_icml_2025_15371/paper.pdf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fb1e43c249a4c79d380732915a3684aa60ef28e0fbf441a88375cf3da930775a +size 5151578 diff --git a/examples/04_icml_2025_15371/poster.html b/examples/04_icml_2025_15371/poster.html new file mode 100644 index 0000000000000000000000000000000000000000..2e805f05d9b79c52a988f1aaf8d3e826f3bf1a5e --- /dev/null +++ b/examples/04_icml_2025_15371/poster.html @@ -0,0 +1,79 @@ + + + + + +Poster + + + +
+
+
+
+
+
+
+
+
+
+
+
+
The Disparate Benefits of Deep Ensembles
+
Kajetan Schweighofer¹ Adrian Arnaiz-Rodriguez² Sepp Hochreiter¹˒³ Nuria Oliver²
+
¹Institute of Advanced Research in Artificial Intelligence (IARAI), Graz, Austria ²Computer Vision Center (CVC), Barcelona, Spain ³Institute of Theoretical Computer Science, Graz University of Technology, Graz, Austria
+
THE DISPARATE BENEFITS OF DEEP ENSEMBLES
+
SUMMARY
+
WHAT IS THE REASON FOR THE EFFECT?
+
● The disparate benefits effect stems from differences in predictive diversity across groups. ● Groups with higher diversity gain more from ensembling.
+
● Deep Ensembles improve predictive performance but can unintentionally harm algorithmic fairness, a phenomenon we call the disparate benefits effect.
+
+
+
● We show that this effect arises because ensemble members vary in predictive diversity across demographic groups, leading to unequal performance gains.
+
● We measure diversity using the log-likelihood gap between the ensemble and its average member.
+
● Adding ensemble members boosts performance but reduces fairness. ● The effect appears across datasets, metrics, and tasks. ● Some groups lose fairness while others gain.
+
● We propose a simple post-processing fix that restores fairness without sacrificing the performance benefits of ensembling.
+
+
+
● Our findings highlight the need to consider fairness when deploying Deep Ensembles in high-stakes applications like healthcare or facial analysis.
+
MITIGATING THE EFFECT
+
BACKGROUND
+
● Deep Ensembles are better calibrated but more sensitive to the prediction threshold. ● We apply Hardt Post-Processing to adjust thresholds per group.
+
● A controlled experiment confirms that higher diversity leads to stronger disparate benefits. ● Interpolating inputs adjusts diversity and modulates the effect.
+
● Deep Ensembles combine multiple neural networks to improve accuracy and estimate uncertainty.
+
+
+
+
● Algorithmic fairness evaluates whether a model treats demographic groups equally. We use three metrics: Statistical Parity, Equal Opportunity, and Average Odds.
+
● We create a synthetic dataset by interpolating between group-specific inputs. ● As diversity increases (higher α), the disparate benefits effect becomes stronger. ● This supports our hypothesis that predictive diversity drives the unequal impact of ensembling.
+
● This method reduces fairness violations while maintaining high accuracy. ● It is simple to implement and requires no retraining. ● Makes Deep Ensembles safer for real-world deployment.
+
● Our goal: Understand how Deep Enstances affect fairness across groups, why this happens, and how to fix it. We study this using facial analysis and medical imaging datasets with attributes like age, gender, and race.
+
+ + \ No newline at end of file diff --git a/examples/04_icml_2025_15371/poster.png b/examples/04_icml_2025_15371/poster.png new file mode 100644 index 0000000000000000000000000000000000000000..2b82e3595bbdec483b8e703341dccf4b9f557969 --- /dev/null +++ b/examples/04_icml_2025_15371/poster.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5d38ae03f0bf3e12321b1d8eb533b7bf649c97468833a39382b1dc88b1c953af +size 493819 diff --git a/examples/04_icml_2025_15371/prompt.json b/examples/04_icml_2025_15371/prompt.json new file mode 100644 index 0000000000000000000000000000000000000000..7b5ce5acdba504d225eef90985f007ae03b7ac7f --- /dev/null +++ b/examples/04_icml_2025_15371/prompt.json @@ -0,0 +1 @@ +{"messages": [{"role": "system", "content": "You are an expert academic poster designer. Given a paper and a design instruction, produce the poster as a standalone HTML document: a + + +
+
+
+
+
+
+
+
+
+
+
+
+
Computational Language Acquisition with Theory of Mind Andy Liu*, Hao Zhu*, Emmy Liu, Yonatan Bisk, Graham Neubig (*equal contribution)
+
Image Referential Games
+
Introducing Theory of Mind
+
Experimental Results
+
● We evaluate our models' performance and language quality on a held-out test set.
+
● We introduce an internal listener into the speaker, representing a Theory of Mind, to approximate the external listener.
+
● We simulate language acquisition in an image referential game. ● A speaker (language learner) and listener (fluent speaker) collaborate to identify a target image. ● The speaker is rewarded for generating utterances that lead the listener to select the target.
+
● Accuracy: How often the listener selects the correct referent. ● Fluency: Grammatical quality of output utterances.
+
+
● During training, the speaker generates multiple candidate utterances and uses the internal listener to rerank them. ● The speaker then selects the utterance that the internal listener believes is most likely to result in the desired listener behavior.
+
Background
+
● Human language acquisition is shaped by social-communicative goals in the grounded world. ● Developmental psychology attributes humans' ability to acquire language to Theory of Mind – the ability to ascribe mental states to other agents.
+
● ToM Accuracy: How often the candidate image that the internal listener believes has the highest probability of being chosen is the actual listener's choice.
+
+
+
Varying Game Difficulty
+
● We vary the difficulty of the referential game by sampling distractors with varying degrees of visual and/or semantic similarity to the target.
+
● Training with more influential ToM leads to significant performance and fluency gains. ● Training on harder distractors leads to the development of more complex and fluent languages. ● There remains a significant gap between the language acquired by our models and human captions, suggesting room for improvement in our simulation of the language acquisition process.
+
● We introduce a cross-entropy objective to train the internal listener to approximate the external listener. ● We experiment with varying the weight of the internal listener when reranking candidate utterances, from zero (no ToM) to normal (equally weighting speaker and listener) to high (listener dominates).
+
+
+ + \ No newline at end of file diff --git a/examples/05_iclr_2023_10772/poster.png b/examples/05_iclr_2023_10772/poster.png new file mode 100644 index 0000000000000000000000000000000000000000..fe3f1c49bd8a2042965e469aa89d4d4a92d20955 --- /dev/null +++ b/examples/05_iclr_2023_10772/poster.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a836572d4e1c11a0b840c92328b9edf6d1302d8742a4e9a5d6deded19c73dc6 +size 509958 diff --git a/examples/05_iclr_2023_10772/prompt.json b/examples/05_iclr_2023_10772/prompt.json new file mode 100644 index 0000000000000000000000000000000000000000..6be2fdfe73b8f2ae2f8fa222ce295b1e5c6ca668 --- /dev/null +++ b/examples/05_iclr_2023_10772/prompt.json @@ -0,0 +1 @@ +{"messages": [{"role": "system", "content": "You are an expert academic poster designer. Given a paper and a design instruction, produce the poster as a standalone HTML document: a - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
Unlocking hidden biomolecular conformational landscapes in diffusion models at inference time
-
Daniel D. Richman¹*, Jessica Karaguesian¹*, Carl-Mikael Suomivuori², Ron O. Dror¹ ¹Department of Structural Biology, Stanford University ²Department of Computer Science, Stanford University *Equal contribution
-
2. Identifying alternative conformations with ConforMixRMSD
-
1. Why and how to sample protein ensembles?
-
3. Outperforming other inference-time methods
-
We compare ConforMixRMSD against other inference-time methods that modify the multiple sequence alignment (MSA) input to increase sampling diversity. ConforMixRMSD outperforms these methods.
-
Proteins are dynamic molecules that sample an ensemble of 3D shapes, or conformations, which determines their function. Recent AI models can generate protein conformations, but they tend to focus on a single shape and miss important alternate states. We introduce ConforMix, a method that improves sampling of existing models at inference time to reveal these hidden states.
-
ConforMixRMSD identifies important alternate conformations that default sampling misses, including domain motions and transporter states.
-
-
We measure how well each method recovers known experimental structures. Higher is better.
-
Given:
-
Table 1: Coverage of experimentally determined reference conformations by method and dataset Domain motion Membrane transporters Cryptic pockets Fold switching (n=38) (n=15) (n=31) (n=15)
-
● A pretrained diffusion model for protein structure ● No extra training or modified inputs
-
Worst-matched reference conformation (harder task)
-
4. Sampling domain motion transitions with ConforMixRMSD
-
2. ConforMix: Enhanced sampling at inference time
-
5. Enabling fast free energy estimation
-
For proteins that switch between open and closed shapes, ConforMixRMSD captures the real motion, while other methods produce scattered, less meaningful variation.
-
ConforMix makes it practical to estimate free energy differences, a key quantity in biology. Where default sampling struggles to converge, ConforMix reaches reliable estimates quickly.
-
ConforMix guides the model toward under-sampled regions using simple biasing functions, then reweights the results. This reveals conformations that standard sampling misses.
-
-
-
-
Across many proteins, ConforMixRMSD consistently aligns with experimental motion patterns, making it the most reliable method for capturing functional dynamics.
-
-
6. Case studies: ConforMixRMSD can capture transporter cycling and RNA flexibility
-
In a multi-subunit sugar transporter, ConforMixRMSD recovers all three functional states and the transitions between them, which default sampling completely misses.
-
ConforMixRMSD also works on RNA. For the hammerhead ribozyme, it samples a range of conformations matching experimental observations, showing that the method generalizes beyond proteins to other biomolecules.
-
-
-
References: [1] Boltz-1, https://github.com/jwohlwend/boltz [2] AFCluster, https://github.com/njustnlp/AFCluster [3] CF-random, https://github.com/aqlaboratory/conformation-flo
-
- - \ No newline at end of file diff --git a/examples/05_neurips_2025_17956/poster.png b/examples/05_neurips_2025_17956/poster.png deleted file mode 100644 index fc9533cf4e684004c1fcca6fb97e3138b87e2dbe..0000000000000000000000000000000000000000 --- a/examples/05_neurips_2025_17956/poster.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7b0633156da4ca6dcecbb223a17cfc00090229f1f27a2390ab89ee592e22ba17 -size 616947 diff --git a/pdf_pipeline.py b/pdf_pipeline.py index dfc3b7861579c438de450c22a83796031f37747f..287f21f06cd95908323a0ddc6d87b12caa385c3d 100644 --- a/pdf_pipeline.py +++ b/pdf_pipeline.py @@ -59,7 +59,9 @@ def prepare(pdf, job, instruction, canvas, accent, columns, custom_spec=''): if len(instruction)>4000 or len(custom_spec)>3000: raise ValueError('Please shorten your design instruction / layout specification.') accent=normalize_accent(accent) - columns=max(1,min(4,int(columns))); width,height=CANVASES[canvas] + columns=max(1,min(4,int(columns))) + m=re.match(r'\s*(\d{3,5})\s*[xX×]\s*(\d{3,5})\s*$',str(canvas or '')) # exact size such as 2500x1666, or a preset name + width,height=(int(m.group(1)),int(m.group(2))) if m else CANVASES.get(canvas,(2400,1600)) extracted=job/'paper' extract(pdf, extracted) body,stats,sections=load_paper_body('paper',str(job),str(pdf),abstract_missing=True,total=22000) @@ -136,6 +138,8 @@ def prepare(pdf, job, instruction, canvas, accent, columns, custom_spec=''): def chat(prepared, body=None): p=prepared + if p.get('messages_pre'): # benchmark case: the stored prompt, verbatim + return p['messages_pre'] paper=f'title: {scrub(p["title"])}\n'+(f'authors: {scrub(p["authors"])}\n' if p['authors'] else '') paper+='body:\n'+scrub(p['body'] if body is None else body) content=[{'type':'text','text':f'\n{paper}\n\n\n\n{p["asset_lines"]}\n\n'}]