phd-report / assets /slides /framework.html
gspin's picture
Rebuild from 56aadd1
fec1624 verified
Raw History Blame Contribute Delete
22.5 kB
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>Describing Art at Several Levels</title>
<style>
:root {
--paper: #f5f1e8;
--paper-2: #ebe5d7;
--ink: #1f1c17;
--ink-2: #5d564b;
--ink-3: #948b7c;
--rule: #d6cdbb;
--lvl-form: #3e5a7a;
--lvl-generic: #5a7a4e;
--lvl-specific: #a0731c;
--lvl-abstract: #9e3b3b;
--lvl-catalogue: #6e5a48;
--lvl-none: #bdb4a3;
--serif: "Iowan Old Style", "Palatino Linotype", Palatino, "Book Antiqua", Georgia, serif;
--sans: "Inter", -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
}
@media (prefers-color-scheme: dark) {
:root:not([data-theme="light"]) {
--paper: #171613;
--paper-2: #22201b;
--ink: #ebe5d8;
--ink-2: #b4ac9c;
--ink-3: #7d7567;
--rule: #39352d;
--lvl-form: #93acc9;
--lvl-generic: #9dbb8e;
--lvl-specific: #d8ae5e;
--lvl-abstract: #d98b84;
--lvl-catalogue: #b8a48f;
--lvl-none: #4d483e;
}
}
:root[data-theme="dark"] {
--paper: #171613;
--paper-2: #22201b;
--ink: #ebe5d8;
--ink-2: #b4ac9c;
--ink-3: #7d7567;
--rule: #39352d;
--lvl-form: #93acc9;
--lvl-generic: #9dbb8e;
--lvl-specific: #d8ae5e;
--lvl-abstract: #d98b84;
--lvl-catalogue: #b8a48f;
--lvl-none: #4d483e;
}
* { box-sizing: border-box; }
html, body { margin: 0; height: 100%; }
body {
background: var(--paper);
color: var(--ink);
font-family: var(--sans);
overflow: hidden;
}
/* A fixed 1280x720 stage, scaled to fit the window. */
.deck {
position: absolute;
top: 50%;
left: 50%;
width: 1280px;
height: 720px;
transform-origin: center;
}
.slide {
position: absolute;
inset: 0;
padding: 64px 88px 72px;
display: none;
flex-direction: column;
}
.slide.active { display: flex; }
.kicker {
font-size: 13px;
letter-spacing: 0.14em;
text-transform: uppercase;
color: var(--ink-3);
margin: 0 0 14px;
}
h1, h2 {
font-family: var(--serif);
font-weight: 400;
margin: 0 0 28px;
line-height: 1.12;
}
h1 { font-size: 64px; letter-spacing: -0.01em; }
h2 { font-size: 42px; }
p, li { font-size: 22px; line-height: 1.5; margin: 0 0 14px; }
.lead { font-size: 26px; color: var(--ink-2); max-width: 900px; }
.small { font-size: 16px; color: var(--ink-2); }
.note { font-size: 14px; color: var(--ink-3); margin-top: auto; }
em.q { font-family: var(--serif); font-size: 1.08em; }
ul { padding-left: 1.1em; margin: 0; }
li { margin-bottom: 10px; }
.cols { display: grid; gap: 48px; grid-template-columns: 1fr 1fr; }
.cols > div > h3 {
font-family: var(--serif);
font-weight: 400;
font-size: 28px;
margin: 0 0 12px;
padding-bottom: 8px;
border-bottom: 1px solid var(--rule);
}
/* Phrases coloured by level. */
.ph {
border-bottom: 2px solid var(--c);
padding: 0 1px;
}
.ph sub {
color: var(--c);
font-family: var(--sans);
font-size: 12px;
font-weight: 600;
margin-left: 3px;
white-space: nowrap;
}
.form { --c: var(--lvl-form); }
.generic { --c: var(--lvl-generic); }
.specific { --c: var(--lvl-specific); }
.abstract { --c: var(--lvl-abstract); }
.catalogue { --c: var(--lvl-catalogue); }
.none { --c: var(--lvl-none); }
.query {
font-family: var(--serif);
font-size: 30px;
line-height: 2.2;
max-width: 1080px;
}
.legend { display: flex; gap: 22px; flex-wrap: wrap; font-size: 15px; color: var(--ink-2); }
.legend span::before {
content: "";
display: inline-block;
width: 12px;
height: 12px;
margin-right: 7px;
vertical-align: -1px;
background: var(--c);
}
/* Level rows: a framework's levels beside the example image. */
.levels { width: 100%; border-collapse: collapse; }
.levels td {
padding: 9px 14px;
border-bottom: 1px solid var(--rule);
font-size: 17px;
line-height: 1.4;
vertical-align: top;
}
.levels td:first-child {
width: 30%;
color: var(--c, var(--ink-3));
font-weight: 600;
border-left: 3px solid var(--c, var(--rule));
}
.levels tr.gap td { color: var(--ink-3); font-style: italic; }
/* Three-column variant (slides 5-8): level | what it covers | in the example. */
.levels.defs td { font-size: 15px; padding: 8px 12px; }
.levels.defs td:first-child { width: 22%; }
.levels.defs td.def { width: 36%; color: var(--ink-2); }
.levels.defs th { padding: 0 12px 8px; }
.levels th {
text-align: left;
font-size: 13px;
font-weight: 600;
letter-spacing: 0.08em;
text-transform: uppercase;
color: var(--ink-3);
padding: 0 14px 8px;
border-bottom: 1px solid var(--rule);
}
.with-image { display: grid; grid-template-columns: 1fr 210px; gap: 40px; align-items: start; }
.with-image img, .plate img {
width: 100%;
display: block;
border: 1px solid var(--rule);
}
.plate figcaption { font-size: 13px; color: var(--ink-2); margin-top: 10px; line-height: 1.45; }
.stat {
font-family: var(--serif);
font-size: 44px;
line-height: 1;
margin: 18px 0 6px;
}
/* Stacked bars (unused since the per-query grid). */
.bars { display: grid; grid-template-columns: 150px 1fr; gap: 16px 20px; align-items: center; margin: 8px 0 22px; }
.bars .name { font-family: var(--serif); font-size: 21px; text-align: right; }
.bar { display: flex; height: 34px; }
.bar span {
background: var(--c);
color: var(--paper);
font-size: 13px;
font-weight: 600;
display: flex;
align-items: center;
justify-content: center;
overflow: hidden;
white-space: nowrap;
}
.bar span.none { color: var(--ink-2); }
/* Share grid: queries asking for each kind (multi-label, so per cell). */
.share { width: 100%; border-collapse: collapse; margin: 8px 0 22px; table-layout: fixed; }
.share th { font-size: 13px; font-weight: 600; letter-spacing: 0.08em; text-transform: uppercase; color: var(--ink-3); text-align: left; padding: 0 10px 10px; }
.share th.name { font-family: var(--serif); font-size: 21px; letter-spacing: 0; text-transform: none; color: var(--ink); text-align: right; padding: 0 18px 0 0; width: 140px; font-weight: 400; }
.share td { height: 52px; padding: 0 10px; border-bottom: 1px solid var(--rule); }
.share td div { display: flex; align-items: center; gap: 8px; }
.share td span { flex: none; height: 26px; min-width: 2px; background: var(--ink-3); opacity: 0.6; }
.share td.none span { background: var(--ink-3); opacity: 0.35; }
.share td b { font-size: 16px; font-weight: 600; color: var(--ink); font-variant-numeric: tabular-nums; }
.share td.empty { color: var(--ink-3); padding-left: 18px; }
.title-slide { justify-content: center; }
.title-slide .rule { width: 120px; height: 2px; background: var(--ink); margin: 34px 0 24px; }
/* Chrome */
.progress { position: fixed; left: 0; bottom: 0; height: 3px; background: var(--ink-3); transition: width 0.2s; }
.count {
position: fixed;
right: 18px;
bottom: 12px;
font-size: 12px;
color: var(--ink-3);
font-variant-numeric: tabular-nums;
}
.nav {
position: fixed;
left: 14px;
bottom: 8px;
display: flex;
gap: 6px;
}
.nav button {
font: inherit;
font-size: 13px;
background: none;
color: var(--ink-2);
border: 1px solid var(--rule);
padding: 4px 10px;
cursor: pointer;
}
.nav button:hover { color: var(--ink); border-color: var(--ink-3); }
@media print {
@page { size: 1280px 720px; margin: 0; }
body { overflow: visible; }
.deck { position: static; transform: none !important; }
.slide { position: relative; display: flex; width: 1280px; height: 720px; page-break-after: always; }
.progress, .count, .nav { display: none; }
}
</style>
</head>
<body>
<main class="deck" id="deck">
<!-- 1 -->
<section class="slide title-slide">
<p class="kicker">PhD report · Part II · Theoretical framework</p>
<h1>Describing images<br />at several levels</h1>
<div class="rule"></div>
</section>
<!-- 2 -->
<section class="slide">
<p class="kicker">The framework</p>
<h2>Scope</h2>
<div class="cols">
<div>
<h3>1&nbsp; Requirements</h3>
<p>Which levels the retrieval system should handle.</p>
</div>
<div>
<h3>2&nbsp; Evaluation</h3>
<p>The system&rsquo;s ability, evaluated level by level.</p>
</div>
</div>
</section>
<!-- 3 -->
<section class="slide">
<p class="kicker">Running example</p>
<h2>Query example</h2>
<p class="query" style="font-size:26px;line-height:2;margin-bottom:6px">Find
<span class="ph form">photographs<sub>form</sub></span> and
<span class="ph form">details<sub>form</sub></span> of the
<span class="ph form">painting<sub>form</sub></span>
<span class="ph specific">Adoration of the Shepherds<sub>iconographic</sub></span> by
<span class="ph catalogue">Hugo van der Goes<sub>non-visual</sub></span>, focusing on the
<span class="ph generic">expressions<sub>depicted</sub></span> of the
<span class="ph generic">shepherds<sub>depicted</sub></span> and the
<span class="ph generic">foreground still life<sub>depicted</sub></span>.</p>
<div style="display:grid;grid-template-columns:700px 1fr;gap:40px;align-items:end">
<figure class="plate" style="margin:0">
<img src="../img/van-der-goes-adoration-f001.jpg" alt="Wide altarpiece: the Virgin kneels before the Child in a manger; shepherds rush in from the left; two half-length prophets draw back green curtains at either edge." />
<figcaption>One of the query&rsquo;s six targets. Hugo van der Goes, <em>Adoration of
the Shepherds</em>, ca. 1480. Oil on panel, 97 × 245 cm. Gemäldegalerie, Berlin.</figcaption>
</figure>
<p class="small">A system can find &ldquo;shepherds&rdquo; and miss &ldquo;Hugo van der
Goes&rdquo;. A single recall score says that it missed, not which kind of
information it missed.</p>
</div>
</section>
<!-- 4 -->
<section class="slide">
<p class="kicker">Existing frameworks</p>
<h2>Framework objectives</h2>
<table class="levels" style="max-width:1000px">
<tr><td>Panofsky</td><td>interpreting works of art</td></tr>
<tr><td>Jaimes &amp; Chang</td><td>indexing image collections</td></tr>
<tr><td>Hollink et al.</td><td>classifying how people describe images</td></tr>
<tr><td>Yu et al.</td><td>scoring critiques of artworks</td></tr>
</table>
<p class="lead" style="margin-top:32px">Applying them to queries is the adaptation. The same
levels on both sides give retrieval a shared vocabulary: each phrase of a query has a
level, and an image either supports a description at that level or does not.</p>
</section>
<!-- 5 -->
<section class="slide">
<p class="kicker">Framework 1 · Panofsky (1939)</p>
<h2>Three strata of meaning</h2>
<div class="with-image">
<table class="levels defs">
<tr><th>level</th><th>what it covers</th><th>in the example</th></tr>
<tr class="generic"><td>I&nbsp; Pre-iconographical</td><td class="def">what is factually and expressively depicted, recognised from everyday experience</td><td>A woman kneels before an infant in a manger; men rush in from the left; two bearded men draw back curtains. Haste and astonishment.</td></tr>
<tr class="specific"><td>II&nbsp; Iconographical</td><td class="def">conventional themes, stories and allegories, identified through texts</td><td>The Adoration of the Shepherds (Luke 2): the Virgin, Joseph, the Christ Child; Isaiah among the prophets.</td></tr>
<tr class="abstract"><td>III&nbsp; Iconological</td><td class="def">intrinsic meaning: what the work reveals about its culture</td><td>A costly, exuberant altarpiece for the urban elite of Flanders, not New Devotion asceticism.</td></tr>
<tr class="gap"><td>no level</td><td class="def">how the image looks; facts about its making</td><td>colour, composition, medium, painter, date, size</td></tr>
</table>
<div>
<p class="small">Of 300 queries</p>
<p class="stat">100%</p>
<p class="small">ask for something no stratum holds: almost always the medium (96%), often the artist (60%) or a date (56%).</p>
</div>
</div>
</section>
<!-- 6 -->
<section class="slide">
<p class="kicker">Framework 2 · Jaimes &amp; Chang (2000)</p>
<h2>A ten-level pyramid, plus non-visual facts</h2>
<div class="with-image">
<table class="levels defs">
<tr><th>level</th><th>what it covers</th><th>in the example</th></tr>
<tr class="form"><td>1&ndash;4&nbsp; syntactic</td><td class="def">how it looks: type and technique, global colour and texture, local shapes, composition</td><td>oil painting; bright blue at the centre; a wide frieze framed by curtains</td></tr>
<tr class="generic"><td>5&ndash;6&nbsp; generic</td><td class="def">everyday objects, people and scenes</td><td>woman, infant, shepherds, angels, ox, ass, sheaf of wheat; a stable</td></tr>
<tr class="specific"><td>7&ndash;8&nbsp; specific</td><td class="def">objects, people and scenes identified by name</td><td>the Virgin, Joseph, Christ, Isaiah; the Adoration of the Shepherds</td></tr>
<tr class="abstract"><td>9&ndash;10&nbsp; abstract</td><td class="def">what an object or the whole image symbolises or is about</td><td>the revelation of Christ, staged for the viewer</td></tr>
<tr class="catalogue"><td>B / A / P&nbsp; non-visual</td><td class="def">biographical (author, date), associated (text, context), physical (size, location)</td><td>Hugo van der Goes, ca. 1480; Gemäldegalerie Berlin; 97 × 245 cm</td></tr>
</table>
<div>
<p class="small">Of 300 queries</p>
<p class="stat">14%</p>
<p class="small">ask for something no level holds. Built to index images; semantic levels after Shatford (1986), who derives them from Panofsky.</p>
</div>
</div>
</section>
<!-- 7 -->
<section class="slide">
<p class="kicker">Framework 3 · Hollink et al. (2004)</p>
<h2>Built from how people describe images</h2>
<div class="with-image">
<table class="levels defs">
<tr><th>level</th><th>what it covers</th><th>in the example</th></tr>
<tr class="catalogue"><td>Non-visual</td><td class="def">creator, title, date, period, material, place, source</td><td>Hugo van der Goes; <em>Adoration of the Shepherds</em>; ca. 1480; oil on panel</td></tr>
<tr class="form"><td>Perceptual</td><td class="def">colour, shape, texture, composition, type and technique</td><td>bright blues and reds; a wide frieze; painting</td></tr>
<tr class="generic"><td>Conceptual · general</td><td class="def">generic objects, events, places and times</td><td>woman, infant, shepherds, angels; kneeling, rushing; a stable</td></tr>
<tr class="specific"><td>Conceptual · specific</td><td class="def">the same, identified by name</td><td>the Virgin, Joseph, Christ, Isaiah; the Adoration of the Shepherds</td></tr>
<tr class="abstract"><td>Conceptual · abstract</td><td class="def">the same, as meanings, emotions and ideas</td><td>awe, haste, devotion; revelation</td></tr>
</table>
<div>
<p class="small">Of 300 queries</p>
<p class="stat">19%</p>
<p class="small">ask for something no class holds. Each conceptual level is crossed with object, event, place, time and relation.</p>
</div>
</div>
</section>
<!-- 8 -->
<section class="slide">
<p class="kicker">Framework 4 · Yu et al. (2026)</p>
<h2>Five levels of depth in a critique</h2>
<div class="with-image">
<table class="levels defs">
<tr><th>level</th><th>what it covers</th><th>in the example</th></tr>
<tr class="form"><td>L1&nbsp; visual perception</td><td class="def">palette, line, composition, perspective, chiaroscuro</td><td>a bright palette; the still Virgin against the rush around her</td></tr>
<tr class="generic"><td>L2&nbsp; technical analysis</td><td class="def">brushwork, medium, subject and genre, style and school</td><td>oil on panel; a Nativity altarpiece; Early Netherlandish</td></tr>
<tr class="specific"><td>L3&nbsp; symbolic</td><td class="def">iconography, symbolism, inscriptions, patronage, literary references</td><td>the Adoration from Luke; Isaiah; the curtains as revelation</td></tr>
<tr class="catalogue"><td>L4&nbsp; historical context</td><td class="def">period, artist biography, influence, provenance</td><td>Hugo van der Goes, ca. 1480, after his withdrawal to a cloister</td></tr>
<tr class="abstract"><td>L5&nbsp; philosophical</td><td class="def">aesthetic ideals, emotion, cross-cultural exchange, value</td><td>worldly splendour and devotion in one object</td></tr>
</table>
<div>
<p class="small">Of 300 queries</p>
<p class="stat">4%</p>
<p class="small">ask for something no dimension holds. Made to score critiques by 15 VLMs, not to index or retrieve; L2 mixes medium with subject.</p>
</div>
</div>
</section>
<!-- 9 -->
<section class="slide">
<p class="kicker">Labels on 300 queries</p>
<h2>What the queries ask for, by framework</h2>
<table class="share">
<tr><th></th><th class="form">form</th><th class="generic">depicted</th><th class="specific">iconographic</th><th class="abstract">abstract</th><th class="catalogue">non-visual</th><th class="none">no place</th></tr>
<tr><th class="name">Panofsky</th><td class="empty">&mdash;</td><td class="generic"><div><span style="width:46.8%"></span><b>65%</b></div></td><td class="specific"><div><span style="width:44.6%"></span><b>62%</b></div></td><td class="abstract"><div><span style="width:0.7%"></span><b>1%</b></div></td><td class="empty">&mdash;</td><td class="none"><div><span style="width:72.0%"></span><b>100%</b></div></td></tr>
<tr><th class="name">J&amp;C</th><td class="form"><div><span style="width:70.6%"></span><b>98%</b></div></td><td class="generic"><div><span style="width:47.5%"></span><b>66%</b></div></td><td class="specific"><div><span style="width:41.0%"></span><b>57%</b></div></td><td class="abstract"><div><span style="width:11.5%"></span><b>16%</b></div></td><td class="catalogue"><div><span style="width:67.7%"></span><b>94%</b></div></td><td class="none"><div><span style="width:10.1%"></span><b>14%</b></div></td></tr>
<tr><th class="name">Hollink</th><td class="form"><div><span style="width:70.6%"></span><b>98%</b></div></td><td class="generic"><div><span style="width:51.8%"></span><b>72%</b></div></td><td class="specific"><div><span style="width:34.6%"></span><b>48%</b></div></td><td class="abstract"><div><span style="width:10.1%"></span><b>14%</b></div></td><td class="catalogue"><div><span style="width:69.8%"></span><b>97%</b></div></td><td class="none"><div><span style="width:13.7%"></span><b>19%</b></div></td></tr>
<tr><th class="name">L1&ndash;L5</th><td class="form"><div><span style="width:23.8%"></span><b>33%</b></div></td><td class="generic"><div><span style="width:70.6%"></span><b>98%</b></div></td><td class="specific"><div><span style="width:39.6%"></span><b>55%</b></div></td><td class="abstract"><div><span style="width:4.3%"></span><b>6%</b></div></td><td class="catalogue"><div><span style="width:67.0%"></span><b>93%</b></div></td><td class="none"><div><span style="width:2.9%"></span><b>4%</b></div></td></tr>
</table>
<p class="note">Share of 300 of the 372 benchmark queries asking for each kind of information
(a query usually asks for several, so rows do not sum to 100%). Each whole query labelled
by <code>gemini-3.6-flash</code> under each framework, blind across frameworks; unreviewed.
Almost every query names a medium and a non-visual fact (creator, date, source); few ask
at the abstract level. L1&ndash;L5 files medium under L2, counted here as depicted.</p>
</section>
<!-- 10 -->
<section class="slide">
<p class="kicker">Queries and images</p>
<h2>Each level draws on a different source</h2>
<table class="levels">
<tr><th>source</th><th>levels</th><th>in the example</th></tr>
<tr class="form"><td>the pixels</td><td>J&amp;C 1&ndash;4, Hollink perceptual, L1</td><td>oil painting, a wide frieze, bright blue</td></tr>
<tr class="generic"><td>the pixels</td><td>Panofsky I, J&amp;C 5&ndash;6, Hollink general, L2</td><td>shepherds, angels, manger, sheaf of wheat</td></tr>
<tr class="specific"><td>pixels + texts</td><td>Panofsky II, J&amp;C 7&ndash;8, Hollink specific, L3</td><td>the Adoration of the Shepherds; Isaiah</td></tr>
<tr class="abstract"><td>pixels + the culture</td><td>Panofsky III, J&amp;C 9&ndash;10, Hollink abstract, L5</td><td>revelation; splendour for an urban elite</td></tr>
<tr class="catalogue"><td>metadata only</td><td>J&amp;C B/A/P, Hollink non-visual, L4</td><td>Hugo van der Goes, ca. 1480, 97 × 245 cm, Berlin</td></tr>
</table>
</section>
</main>
<div class="progress" id="progress"></div>
<div class="nav"><button id="prev" aria-label="Previous slide">&larr;</button><button id="next" aria-label="Next slide">&rarr;</button><button id="full" aria-label="Full screen">⤢</button></div>
<div class="count" id="count"></div>
<script>
const deck = document.getElementById("deck");
const slides = [...deck.querySelectorAll(".slide")];
let i = 0;
function fit() {
const s = Math.min(window.innerWidth / 1280, window.innerHeight / 720);
deck.style.transform = `translate(-50%, -50%) scale(${s})`;
}
function show(n) {
i = Math.max(0, Math.min(slides.length - 1, n));
slides.forEach((s, k) => s.classList.toggle("active", k === i));
document.getElementById("count").textContent = `${i + 1} / ${slides.length}`;
document.getElementById("progress").style.width = `${((i + 1) / slides.length) * 100}%`;
try { history.replaceState(null, "", `#${i + 1}`); } catch (e) {}
}
document.addEventListener("keydown", (e) => {
if (["ArrowRight", "PageDown", " "].includes(e.key)) { e.preventDefault(); show(i + 1); }
else if (["ArrowLeft", "PageUp"].includes(e.key)) { e.preventDefault(); show(i - 1); }
else if (e.key === "Home") show(0);
else if (e.key === "End") show(slides.length - 1);
else if (e.key === "f") toggleFull();
});
deck.addEventListener("click", (e) => show(e.clientX > window.innerWidth / 2 ? i + 1 : i - 1));
document.getElementById("prev").onclick = () => show(i - 1);
document.getElementById("next").onclick = () => show(i + 1);
function toggleFull() {
if (document.fullscreenElement) document.exitFullscreen();
else document.documentElement.requestFullscreen?.();
}
document.getElementById("full").onclick = toggleFull;
// Follow the Space's light/dark toggle when embedded.
try {
const parentTheme = window.parent !== window && window.parent.document.documentElement.dataset.theme;
if (parentTheme) document.documentElement.dataset.theme = parentTheme;
} catch (e) {}
window.addEventListener("resize", fit);
fit();
show((parseInt(location.hash.slice(1), 10) || 1) - 1);
</script>
</body>
</html>