Release original LTF v3.3 epoch 20 inference model

demo/README.md

@@ -59,3 +59,30 @@ LTF_MODEL_DIR=../model_repo python app.py
 ```
 
 The web UI and its `predict` API both call the same original-file inference path.
+
+## Attention and tile inspection
+
+The Detect button additionally requests the model's existing `return_viz`
+outputs during the same forward pass. Temporary observation hooks collect the
+eight L1 across-tile aggregation heads. No model parameters are changed.
+
+- Exact, numbered local crop boxes, including edge-aligned overlap and small-image padding.
+- L1 tile-attention overlay: mean or any of the eight aggregation heads.
+- L2 local-view overlay with the global-view attention weight listed separately.
+- Top four local tiles, original crops paired with their 16×16 MIL patch attention.
+- A tile selector for inspecting any local tile without rerunning inference.
+- Exact view-weight table, embedding fusion gate and read-only model/inference parameters.
+- Historical training settings from the original saved configuration, explicitly labeled Training only.
+
+Colors use a separate min-max scale for each map (blue→yellow is low→high).
+Overlapping image pixels use the mean of covering tile weights. A patch map is
+normalized within its own tile. Raw normalized view weights are available in
+the table and prediction JSON; colors cannot be compared across images or tiles.
+These are model attention values, not pixel-level forgery probabilities or
+causal explanations. The eight selectable heads belong to the L1 across-tile
+aggregator; L1 patch pooling and L2 view pooling use gated MIL attention.
+
+The original five-output `predict` API is preserved. The new `explain` API
+returns prediction results plus visualization images, tables and attention metadata.
+Changing a head or inspected tile uses per-session cached data, retained for up
+to an hour. The unchanged checkpoint is used for both endpoints.

demo/app.py

@@ -22,6 +22,10 @@ from huggingface_hub import snapshot_download
 from PIL import Image, UnidentifiedImageError
 import torch
 
+sys.path.insert(0, str(Path(__file__).absolute().parent))
+from attention import (capture_prediction, draw_grid, head_view, parameter_rows,
+                       patch_view, public_attention, tile_boxes, tile_overlay, top_tiles)
+
 LOG = logging.getLogger(__name__)
 MAX_PIXELS = int(os.environ.get('LTF_MAX_PIXELS', '16000000'))
 MAX_VIEWS = int(os.environ.get('LTF_MAX_VIEWS', '400'))
@@ -108,7 +112,7 @@ def get_detector():
     return _detector
 
 
-def predict(image_path, branch='fusion', threshold=0.5):
+def _predict_request(image_path, branch='fusion', threshold=0.5, with_attention=False):
     source, width, height, views = validate_upload(image_path)
     if branch not in ('fusion', 'level1', 'level2'):
         raise gr.Error('Unknown model branch.')
@@ -118,7 +122,14 @@ def predict(image_path, branch='fusion', threshold=0.5):
     try:
         with MODEL_LOCK:
             detector = get_detector()
-            result = detector.predict(source, branch, float(threshold))
+            if with_attention:
+                result, attention = capture_prediction(detector, source, branch, float(threshold))
+                state = dict(path=str(source), attention=attention,
+                             boxes=tile_boxes(width, height, int(detector.model.input_resolution)),
+                             parameters=parameter_rows(detector, branch, float(threshold), views))
+            else:
+                result = detector.predict(source, branch, float(threshold))
+                state = None
     except torch.cuda.OutOfMemoryError:
         torch.cuda.empty_cache()
         raise gr.Error('GPU memory is insufficient. Please try a smaller image.') from None
@@ -144,7 +155,42 @@ def predict(image_path, branch='fusion', threshold=0.5):
     rows = [[name, result['probabilities'][name]['real'],
              result['probabilities'][name]['fake']]
             for name in ('fusion', 'level1', 'level2')]
-    return preview, summary, {'Real': scores['real'], 'AI-generated': scores['fake']}, rows, result
+    if state:
+        result['attention'] = public_attention(state)
+    return (preview, summary, {'Real': scores['real'], 'AI-generated': scores['fake']}, rows, result), state
+
+
+def predict(image_path, branch='fusion', threshold=0.5):
+    """Keep the original five-output public prediction API compatible."""
+    return _predict_request(image_path, branch, threshold)[0]
+
+
+def explain(image_path, branch='fusion', threshold=0.5):
+    base, state = _predict_request(image_path, branch, threshold, with_attention=True)
+    attention = state['attention']
+    count = len(state['boxes'])
+    if len(attention['l1']) != count or len(attention['l2']) != count + 1:
+        raise gr.Error('Attention and original image tile counts disagree.')
+    grid = draw_grid(state['path'], state['boxes'])
+    l1_image, l1_description = head_view(state, 'mean')
+    l2_image = tile_overlay(state['path'], state['boxes'], attention['l2'][1:])
+    description = (f'**L2 global-view weight: {attention["l2"][0]:.6f}** · '
+                   f'local-view weights sum: {float(attention["l2"][1:].sum()):.6f}.\n\n'
+                   'L2 图仅着色局部分块；全局权重单列，不铺满整张图。'
+                   ' / The L2 overlay shows local views only; the global weight is listed separately.\n\n'
+                   f'Fusion embedding gate: L1 × **{attention["gate"]:.4f}**, '
+                   f'L2 × **{1 - attention["gate"]:.4f}**. '
+                   '这是特征缩放系数，不是分支概率 / These scale embeddings, not branch probabilities.')
+    weight_rows = [['Global', 'whole image', None, float(attention['l2'][0])]]
+    weight_rows.extend([f'T{i + 1}', str(list(box)), float(attention['l1'][i]), float(attention['l2'][i + 1])]
+                       for i, box in enumerate(state['boxes']))
+    selected = int(attention['l1'].argmax()) + 1
+    tile, patch, patch_description = patch_view(state, str(selected))
+    return (*base, grid, l1_image, l1_description, l2_image, description, top_tiles(state),
+            state['parameters'], weight_rows, state,
+            gr.Dropdown(choices=[str(i + 1) for i in range(count)], value=str(selected)),
+            tile, patch, patch_description,
+            gr.Dropdown(choices=['mean'] + [f'head_{i + 1}' for i in range(attention['l1_heads'].shape[0])], value='mean'))
 
 
 def build_demo():
@@ -170,8 +216,53 @@ def build_demo():
             table = gr.Dataframe(headers=['Branch', 'Real score', 'AI-generated score'],
                                  datatype=['str', 'number', 'number'], interactive=False)
             details = gr.JSON(label='Prediction')
-        button.click(predict, [upload, branch, threshold], [preview, summary, scores, table, details],
-                     api_name='predict', concurrency_limit=1)
+        state = gr.State(value=None, time_to_live=3600)
+        with gr.Accordion('分块与 MIL Attention / Tiles and model attention', open=True):
+            gr.Markdown('以下权重直接来自本次模型前向计算。Attention 是相对关注程度，'
+                        '不能直接作为伪造区域或因果解释。'
+                        ' / These are actual forward-pass weights, not forgery probabilities or causal explanations.')
+            with gr.Tabs():
+                with gr.Tab('原图分块 / Tile grid'):
+                    grid = gr.Image(label='224×224 local tiles · T1…Tn · edge-aligned', interactive=False)
+                    gr.Markdown('分块编号按行排列，边缘对齐时允许重叠。全局视图独立于局部分块。'
+                                ' / Row-major IDs; edge-aligned tiles may overlap. The global view is separate.')
+                with gr.Tab('L1 分块注意力'):
+                    head = gr.Dropdown(['mean'], value='mean', label='Across-tile aggregation head')
+                    l1_image = gr.Image(label='L1 tile attention', interactive=False)
+                    l1_description = gr.Markdown()
+                with gr.Tab('L2 视图注意力'):
+                    l2_image = gr.Image(label='L2 local-view attention', interactive=False)
+                    attention_description = gr.Markdown()
+                    weight_table = gr.Dataframe(headers=['View', 'Original crop box', 'L1 weight', 'L2 weight'],
+                                                datatype=['str', 'str', 'number', 'number'], interactive=False)
+                with gr.Tab('Top-K 与块内 Patch Attention'):
+                    gallery = gr.Gallery(label='Top 4 L1 tiles: original + within-tile patch attention', columns=4,
+                                         object_fit='contain', height='auto')
+                    tile_number = gr.Dropdown(choices=[], label='Inspect local tile / 分块编号', allow_custom_value=False)
+                    with gr.Row():
+                        selected_tile = gr.Image(label='Original 224×224 crop', interactive=False)
+                        selected_patch = gr.Image(label='MIL patch attention (16×16)', interactive=False)
+                    patch_description = gr.Markdown()
+            gr.Markdown('热图分别归一化：蓝色低、黄色高，不能跨图片按颜色比较。'
+                        '重叠像素显示覆盖分块的平均权重，精确权重见表格。'
+                        ' / Heatmaps use separate min-max scales; overlap uses mean weights.')
+        with gr.Accordion('模型与超参数 / Model and hyperparameters (read only)', open=False):
+            parameters = gr.Dataframe(headers=['Scope', 'Parameter', 'Value'],
+                                      datatype=['str', 'str', 'str'], interactive=False)
+            gr.Markdown('Training only 是历史训练配置；Cross-shuffle、随机增强和 dropout '
+                        '在本次推理中不启用。Head 数量与网络结构随权重固定。')
+        button.click(explain, [upload, branch, threshold],
+                     [preview, summary, scores, table, details, grid, l1_image, l1_description,
+                      l2_image, attention_description, gallery, parameters, weight_table, state,
+                      tile_number, selected_tile, selected_patch, patch_description, head],
+                     api_name='explain', concurrency_limit=1, concurrency_id='ltf-inference')
+        head.change(head_view, [state, head], [l1_image, l1_description], api_name=False)
+        tile_number.change(patch_view, [state, tile_number],
+                           [selected_tile, selected_patch, patch_description], api_name=False)
+        # Preserve the original prediction endpoint for existing API clients.
+        api_button = gr.Button(visible=False)
+        api_button.click(predict, [upload, branch, threshold], [preview, summary, scores, table, details],
+                         api_name='predict', concurrency_limit=1, concurrency_id='ltf-inference')
         gr.Markdown('分数是模型输出，不能单独作为图片真伪的证明。'
                     ' / Model scores are not calibrated probabilities or proof of authenticity.')
     return demo.queue(max_size=16, default_concurrency_limit=1)

demo/attention.py

@@ -0,0 +1,226 @@
+"""Read actual LTF MIL attention and render its original-image coordinates."""
+from __future__ import annotations
+
+import json
+import math
+from pathlib import Path
+
+import numpy as np
+from PIL import Image, ImageDraw, ImageFont
+
+
+def capture_prediction(detector, path, branch, threshold):
+    """Observe one ordinary prediction; temporary hooks are always removed.
+
+    Call under the app's model lock. return_viz only appends existing model
+    tensors to the output; it does not change logits or rerun the backbone.
+    """
+    captured = {}
+
+    def enable_viz(module, args, kwargs):
+        return args, dict(kwargs, return_viz=True)
+
+    def collect(module, args, output):
+        viz = output['viz']
+        for key in ('tile_attn', 'level1_patch_attn', 'level2_attn', 'gate'):
+            captured[key] = viz[key].detach().float().cpu().numpy()
+
+    def collect_heads(module, args, output):
+        captured['head_attn'] = output[1][0, :, 0, :].detach().float().cpu().numpy()
+
+    handles = []
+    try:
+        handles.append(detector.model.register_forward_pre_hook(enable_viz, with_kwargs=True))
+        handles.append(detector.model.register_forward_hook(collect))
+        handles.append(detector.model.core.level1_aggregate_attn.register_forward_hook(collect_heads))
+        result = detector.predict(path, branch, threshold)
+    finally:
+        for handle in handles:
+            handle.remove()
+    nt = result['views'] - 1
+    patches = captured['level1_patch_attn'].reshape(nt, -1)
+    side = math.isqrt(patches.shape[1])
+    if side * side != patches.shape[1]:
+        raise ValueError('Patch attention does not form a square grid.')
+    attention = dict(
+        l1=captured['tile_attn'].reshape(nt),
+        l1_heads=captured['head_attn'],
+        l2=captured['level2_attn'].reshape(nt + 1),
+        patch=patches.reshape(nt, side, side),
+        gate=float(captured['gate'].reshape(-1)[0]),
+    )
+    for key in ('l1', 'l2', 'patch', 'l1_heads'):
+        weights = attention[key]
+        if not np.isfinite(weights).all() or (weights < 0).any():
+            raise ValueError('Attention contains invalid values.')
+    return result, attention
+
+
+def tile_boxes(width, height, tile_size=224):
+    # Use the exact same edge-aligned coordinates and row-major order as inference.
+    from utils.tile_dataset_srm import get_start_positions
+    return [(x, y, x + tile_size, y + tile_size)
+            for y in get_start_positions(height, tile_size)
+            for x in get_start_positions(width, tile_size)]
+
+
+def relative_scale(values):
+    values = np.asarray(values, dtype=np.float32)
+    low, high = float(values.min()), float(values.max())
+    if high - low < 1e-12:
+        return np.full_like(values, 0.5)
+    return (values - low) / (high - low)
+
+
+def colors(values):
+    """Continuous dark-blue → cyan → yellow scale; no probability semantics."""
+    x = np.clip(np.asarray(values), 0, 1)
+    stops = np.array([[35, 47, 110], [31, 164, 172], [253, 225, 77]], dtype=np.float32)
+    first = np.minimum((x * 2).astype(int), 1)
+    fraction = (x * 2 - first)[..., None]
+    return (stops[first] * (1 - fraction) + stops[first + 1] * fraction).astype(np.uint8)
+
+
+def preview_image(path):
+    with Image.open(path) as source:
+        image = source.convert('RGB')
+    image.thumbnail((1200, 1200))
+    return image
+
+
+def draw_grid(path, boxes):
+    image = preview_image(path)
+    with Image.open(path) as original:
+        sx, sy = image.width / original.width, image.height / original.height
+    draw = ImageDraw.Draw(image)
+    font = ImageFont.load_default(size=14)
+    labels = []
+    for index, (x0, y0, x1, y1) in enumerate(boxes, 1):
+        rect = (int(x0 * sx), int(y0 * sy), min(image.width - 1, int(x1 * sx) - 1),
+                min(image.height - 1, int(y1 * sy) - 1))
+        draw.rectangle(rect, outline=(255, 212, 74), width=2)
+        label = f'T{index}'
+        left, top = rect[0] + 3, rect[1] + 3
+        # Edge alignment can place two rows just one pixel apart. Keep their
+        # exact crop borders while moving text labels to avoid hiding earlier IDs.
+        for candidate_top in range(top, max(top + 1, rect[3] - 15), 18):
+            extent = draw.textbbox((left, candidate_top), label, font=font)
+            if not any(left < old[2] + 3 and extent[2] + 3 > old[0]
+                       and candidate_top < old[3] + 3 and extent[3] + 3 > old[1]
+                       for old in labels):
+                top = candidate_top
+                break
+        extent = draw.textbbox((left, top), label, font=font)
+        labels.append((left, top, extent[2], extent[3]))
+        draw.rectangle((left - 1, top - 1, extent[2] + 2, extent[3] + 2), fill=(18, 28, 45))
+        draw.text((left, top), label, fill='white', font=font)
+    return image
+
+
+def tile_overlay(path, boxes, weights):
+    image = preview_image(path)
+    with Image.open(path) as original:
+        sx, sy = image.width / original.width, image.height / original.height
+    summed = np.zeros((image.height, image.width), dtype=np.float32)
+    coverage = np.zeros_like(summed)
+    for box, weight in zip(boxes, weights, strict=True):
+        x0, y0, x1, y1 = box
+        x0, y0 = int(x0 * sx), int(y0 * sy)
+        x1, y1 = min(image.width, math.ceil(x1 * sx)), min(image.height, math.ceil(y1 * sy))
+        summed[y0:y1, x0:x1] += float(weight)
+        coverage[y0:y1, x0:x1] += 1
+    averaged = summed / np.maximum(coverage, 1)
+    heat = Image.fromarray(colors(relative_scale(averaged)))
+    return Image.blend(image, heat, 0.48)
+
+
+def patch_view(state, tile_number):
+    if not state:
+        return None, None, '请先检测图片 / Run detection first.'
+    index = int(tile_number) - 1
+    if index < 0 or index >= len(state['boxes']):
+        raise ValueError('Tile number is out of range.')
+    box = state['boxes'][index]
+    with Image.open(state['path']) as source:
+        tile = source.convert('RGB').crop(box)
+    weights = state['attention']['patch'][index]
+    heat = Image.fromarray(colors(relative_scale(weights))).resize(tile.size, Image.Resampling.NEAREST)
+    overlay = Image.blend(tile, heat, 0.48)
+    description = (f'T{index + 1} · original box {list(box)} · patch grid '
+                   f'{weights.shape[0]} × {weights.shape[1]} · '
+                   f'patch weights sum = {float(weights.sum()):.6f}. '
+                   '颜色在当前块内归一化；不能按颜色跨块比较。'
+                   ' / Colors are scaled within this tile only. '
+                   'Small images retain the model’s original black padding.')
+    return tile, overlay, description
+
+
+def top_tiles(state, k=4):
+    order = np.argsort(-state['attention']['l1'], kind='stable')[:k]
+    gallery = []
+    for index in order:
+        tile, overlay, _ = patch_view(state, int(index) + 1)
+        weight = float(state['attention']['l1'][index])
+        gallery.extend([(tile, f'T{index + 1} · L1 weight {weight:.6f}'),
+                        (overlay, f'T{index + 1} · within-tile patch attention')])
+    return gallery
+
+
+def head_view(state, selection):
+    if not state:
+        return None, '请先检测图片 / Run detection first.'
+    if selection == 'mean':
+        weights = state['attention']['l1']
+        label = 'Mean across L1 aggregation heads'
+    else:
+        index = int(selection.split('_')[-1]) - 1
+        weights = state['attention']['l1_heads'][index]
+        label = f'L1 aggregation head {index + 1}'
+    return tile_overlay(state['path'], state['boxes'], weights), (
+        f'{label} · sum = {float(weights.sum()):.6f}. '
+        '蓝→黄表示本图内相对权重低→高；重叠区域取覆盖分块权重的平均值。'
+        ' / Blue→yellow: lower→higher within this image; overlap uses mean weight. '
+        '精确权重见表格 / Exact weights are in the table.')
+
+
+def parameter_rows(detector, branch, threshold, views):
+    core, visual = detector.model.core, detector.model.visual
+    mil1, mil2 = core.mil_level1.head.head, core.mil_level2.head.head
+    rows = [
+        ['Inference', 'Branch / threshold', f'{branch} / {threshold:.2f}'],
+        ['Inference', 'Precision / device', f'{visual.conv1.weight.dtype} / {detector.device}'],
+        ['Inference', 'CLIP chunk size', str(detector.model.eval_chunk_size)],
+        ['Inference', 'Views: global + local', f'1 + {views - 1} = {views}'],
+        ['Architecture', 'Backbone', 'Frozen CLIP ViT-L/14'],
+        ['Architecture', 'Local tile size', f'{int(detector.model.input_resolution)} × {int(detector.model.input_resolution)}'],
+        ['Architecture', 'CLIP patch size', str(visual.conv1.kernel_size[0])],
+        ['Architecture', 'Feature dimension', str(detector.model.embed_dim)],
+        ['Architecture', 'L1 across-tile aggregation heads', str(core.level1_aggregate_attn.num_heads)],
+        ['Architecture', 'L1 MIL pooling / hidden', f'{mil1.pooling} / {mil1.attn_V.out_features} ({mil1.attn_bag_mode})'],
+        ['Architecture', 'L2 MIL pooling / hidden', f'{mil2.pooling} / {mil2.attn_V.out_features} ({mil2.attn_bag_mode})'],
+        ['Architecture', 'Fusion / gate hidden', f'pool-logit gated / {core.pool_gate[0].out_features}'],
+        ['Architecture', 'Fusion MLP hidden', str(core.fusion_head[0].out_features)],
+        ['Inference', 'Cross-shuffle / dropout', 'Disabled in evaluation mode'],
+    ]
+    reference = Path(__file__).with_name('training_reference.json')
+    if reference.is_file():
+        training = json.loads(reference.read_text())
+        rows.extend(['Training only', key, json.dumps(value, ensure_ascii=False)]
+                    for key, value in training.items())
+    return rows
+
+
+def public_attention(state):
+    weights = state['attention']
+    return dict(
+        meaning='Relative model attention, not a forgery probability or causal explanation.',
+        display='Each heatmap is min-max scaled independently; overlap uses mean tile weight.',
+        local_tile_boxes=[list(box) for box in state['boxes']],
+        l1_tile_weights=weights['l1'].tolist(),
+        l1_aggregation_head_weights=weights['l1_heads'].tolist(),
+        l2_view_weights=weights['l2'].tolist(),
+        l2_view_order=['global'] + [f'T{i + 1}' for i in range(len(state['boxes']))],
+        patches_per_tile=list(weights['patch'].shape[1:]),
+        fusion_embedding_gate_l1=weights['gate'],
+        fusion_embedding_gate_l2=1 - weights['gate'],
+    )

demo/training_reference.json

@@ -0,0 +1,21 @@
+{
+  "seed": 16,
+  "epochs": 20,
+  "batch_size": 64,
+  "learning_rate": 0.0004,
+  "weight_decay": 0.05,
+  "dropout": 0.1,
+  "default_train_tiles": 16,
+  "train_set_size": 50000,
+  "v2_l1_cross_shuffle_p": 0.9,
+  "v2_l1_replace_ratio_range": [
+    0.5,
+    0.9
+  ],
+  "v2_l2_cross_shuffle_p": 0.9,
+  "v2_l2_replace_ratio_range": [
+    0.1,
+    0.8
+  ],
+  "v2_bag_token_shuffle_p": 0.5
+}