Deploy Sensorflow Studio

app_backend.py

@@ -2447,10 +2447,16 @@ def pipeline_file(path: str):
 
 
 @app.get("/api/dataset/browse")
-def dataset_browse(source_path: str = "data", limit: int = 48):
-    """Validate Images Path by listing browsable local frames."""
+def dataset_browse(
+    source_path: str = "data",
+    limit: int = 48,
+    dataset_type: Optional[str] = None,
+):
+    """Validate Images Path by listing browsable local driving-relevant frames."""
     from sensorflow import pipeline_artifacts as artifacts
-    return artifacts.scan_source_images(source_path, limit=limit)
+
+    dtype = dataset_type or load_config().dataset_type
+    return artifacts.scan_source_images(source_path, limit=limit, dataset_type=dtype)
 
 
 @app.get("/api/mitl/queue")

data/README.md

@@ -0,0 +1,8 @@
+# Local data folder
+
+This folder may contain ad-hoc test assets. **Dataset browse and load filters** exclude
+non-driving media (README assets, logos, generic `test.jpg` files, docs paths, and files
+that do not match AV camera-frame layout when an official dataset type is selected).
+
+Place real camera frames under a nested path such as `data/images/train/` or
+`data/camera/front/` for Validate & Browse and Local ingest to pick them up.

sensorflow/adapters/local_adapter.py

@@ -23,29 +23,49 @@ VIDEO_EXTS = {".mp4", ".avi", ".mov", ".mkv", ".webm", ".m4v"}
 DEFAULT_MAX_FRAMES = 10_000
 
 
-def discover_media(path: Path) -> Tuple[List[Path], List[Path]]:
+def discover_media(
+    path: Path,
+    *,
+    dataset_type: Optional[str] = None,
+) -> Tuple[List[Path], List[Path]]:
     """Return (image_files, video_files) under path (file or directory)."""
+    from sensorflow.av_media_filter import (
+        IMAGE_EXTS as AV_IMAGE_EXTS,
+        VIDEO_EXTS as AV_VIDEO_EXTS,
+        partition_media_files,
+    )
+
     if path.is_file():
         ext = path.suffix.lower()
-        if ext in IMAGE_EXTS:
-            return [path], []
-        if ext in VIDEO_EXTS:
-            return [], [path]
+        if ext in AV_IMAGE_EXTS:
+            images, _ = partition_media_files([path], root=path.parent, dataset_type=dataset_type)
+            return images, []
+        if ext in AV_VIDEO_EXTS:
+            _, excluded = partition_media_files([path], root=path.parent, dataset_type=dataset_type, allow_videos=True)
+            return [], [] if excluded else [path]
         return [], []
 
     if not path.is_dir():
         return [], []
 
-    images: List[Path] = []
-    videos: List[Path] = []
+    raw_images: List[Path] = []
+    raw_videos: List[Path] = []
     for p in sorted(path.rglob("*")):
         if not p.is_file():
             continue
         ext = p.suffix.lower()
-        if ext in IMAGE_EXTS:
-            images.append(p)
-        elif ext in VIDEO_EXTS:
-            videos.append(p)
+        if ext in AV_IMAGE_EXTS:
+            raw_images.append(p)
+        elif ext in AV_VIDEO_EXTS:
+            raw_videos.append(p)
+
+    images, _ = partition_media_files(raw_images, root=path, dataset_type=dataset_type)
+    videos, _ = partition_media_files(
+        raw_videos,
+        root=path,
+        dataset_type=dataset_type,
+        allow_videos=True,
+    )
     return images, videos
 
 
@@ -111,7 +131,7 @@ class LocalSequenceAdapter(VendorAdapter):
                 "or a video file, then re-run ingest with Local enabled."
             )
 
-        images, videos = discover_media(path)
+        images, videos = discover_media(path, dataset_type=source.get("dataset_type"))
         frame_images = list(images)
 
         decode_dir = Path("runs/pipeline") / sequence_id / "decoded_frames"

sensorflow/av_media_filter.py

@@ -0,0 +1,286 @@
+"""Heuristic filters for driving / AV pipeline image and video discovery."""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+from typing import Any, Dict, Iterable, List, Optional, Sequence, Tuple
+
+IMAGE_EXTS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".tif", ".tiff"}
+VIDEO_EXTS = {".mp4", ".avi", ".mov", ".mkv", ".webm", ".m4v"}
+
+# Path segments that almost never contain drivable-scene media.
+EXCLUDED_DIR_SEGMENTS = {
+    ".git",
+    ".github",
+    ".venv",
+    "venv",
+    "__pycache__",
+    "node_modules",
+    "dist",
+    "build",
+    "docs",
+    "doc",
+    "documentation",
+    "marketing",
+    "assets",
+    "static/icons",
+    "icons",
+    "badges",
+    "screenshots",
+}
+
+# Filename stems that are commonly README / branding / test junk.
+JUNK_NAME_STEMS = {
+    "logo",
+    "icon",
+    "favicon",
+    "avatar",
+    "profile",
+    "readme",
+    "banner",
+    "badge",
+    "placeholder",
+    "sample",
+    "example",
+    "demo",
+    "test",
+    "screenshot",
+    "thumbnail",
+    "thumb",
+}
+
+# Substrings in relative paths that suggest AV camera frames.
+AV_PATH_MARKERS = (
+    "camera",
+    "cameras",
+    "images",
+    "frames",
+    "frame",
+    "front",
+    "rear",
+    "left",
+    "right",
+    "rgb",
+    "video",
+    "sequence",
+    "segment",
+    "train",
+    "val",
+    "training",
+    "validation",
+    "lidar",
+    "sensor",
+    "recording",
+    "drive",
+    "driving",
+    "scene",
+)
+
+OFFICIAL_AV_DATASETS = frozenset({"alpamayo", "waymo", "a2d2"})
+
+MIN_DRIVING_IMAGE_BYTES = 10 * 1024  # icons / tiny README assets
+
+EXCLUSION_CATEGORIES = (
+    "wrong_extension",
+    "excluded_path",
+    "junk_filename",
+    "too_small",
+    "non_av_layout",
+)
+
+
+def _normalize_parts(path: Path) -> Tuple[str, ...]:
+    return tuple(part.lower() for part in path.parts)
+
+
+def _relative_to_root(path: Path, root: Optional[Path]) -> Path:
+    if root is None:
+        return path
+    try:
+        return path.resolve().relative_to(root.resolve())
+    except ValueError:
+        return path
+
+
+def _has_excluded_dir(path: Path) -> bool:
+    parts = _normalize_parts(path)
+    for i, part in enumerate(parts):
+        if part in EXCLUDED_DIR_SEGMENTS:
+            return True
+        if part == "assets" and i + 1 < len(parts) and parts[i + 1] in {
+            "marketing",
+            "branding",
+            "icons",
+            "logos",
+        }:
+            return True
+    return False
+
+
+def _junk_filename(path: Path) -> bool:
+    stem = path.stem.lower()
+    if stem in JUNK_NAME_STEMS:
+        return True
+    if re.match(r"^(logo|icon|favicon|readme|banner|badge)[-_]?\d*$", stem):
+        return True
+    return False
+
+
+def _looks_like_av_layout(relative: Path) -> bool:
+    rel_str = str(relative).lower().replace("\\", "/")
+    if any(marker in rel_str for marker in AV_PATH_MARKERS):
+        return True
+    # Nested structure (e.g. data/sequences/001/front.jpg) — not a loose root file.
+    return len(relative.parts) >= 2
+
+
+def classify_media_file(
+    path: Path,
+    *,
+    root: Optional[Path] = None,
+    dataset_type: Optional[str] = None,
+    allow_videos: bool = True,
+) -> Tuple[bool, Optional[str]]:
+    """Return (accepted, exclusion_category)."""
+    if not path.is_file():
+        return False, "non_media"
+
+    ext = path.suffix.lower()
+    allowed = IMAGE_EXTS | (VIDEO_EXTS if allow_videos else set())
+    if ext not in allowed:
+        return False, "wrong_extension"
+
+    if _has_excluded_dir(path):
+        return False, "excluded_path"
+
+    if _junk_filename(path):
+        return False, "junk_filename"
+
+    if ext in IMAGE_EXTS:
+        try:
+            size = path.stat().st_size
+        except OSError:
+            return False, "non_media"
+        if size < MIN_DRIVING_IMAGE_BYTES:
+            return False, "too_small"
+
+    relative = _relative_to_root(path, root)
+    dtype = (dataset_type or "local").lower()
+    if dtype in OFFICIAL_AV_DATASETS and not _looks_like_av_layout(relative):
+        return False, "non_av_layout"
+
+    return True, None
+
+
+def partition_media_files(
+    paths: Iterable[Path],
+    *,
+    root: Optional[Path] = None,
+    dataset_type: Optional[str] = None,
+    allow_videos: bool = True,
+) -> Tuple[List[Path], List[Dict[str, Any]]]:
+    """Split paths into accepted driving media and excluded entries with reasons."""
+    accepted: List[Path] = []
+    excluded: List[Dict[str, Any]] = []
+    for path in paths:
+        ok, reason = classify_media_file(
+            path,
+            root=root,
+            dataset_type=dataset_type,
+            allow_videos=allow_videos,
+        )
+        if ok:
+            accepted.append(path)
+        else:
+            excluded.append(
+                {
+                    "path": str(path),
+                    "name": path.name,
+                    "reason": reason or "excluded_path",
+                }
+            )
+    accepted.sort(key=lambda p: str(p).lower())
+    return accepted, excluded
+
+
+def iter_candidate_files(
+    root: Path,
+    *,
+    allow_videos: bool = True,
+) -> List[Path]:
+    """Collect image/video candidates under root before AV heuristics."""
+    exts = set(IMAGE_EXTS)
+    if allow_videos:
+        exts |= VIDEO_EXTS
+
+    if root.is_file():
+        return [root] if root.suffix.lower() in exts else []
+
+    if not root.is_dir():
+        return []
+
+    out: List[Path] = []
+    for p in root.rglob("*"):
+        if p.is_file() and p.suffix.lower() in exts:
+            out.append(p)
+    return out
+
+
+def scan_driving_media(
+    source_path: str | Path,
+    *,
+    dataset_type: Optional[str] = None,
+    allow_videos: bool = True,
+    limit: Optional[int] = None,
+) -> Dict[str, Any]:
+    """Scan a path and return accepted media plus exclusion breakdown."""
+    root = Path(source_path)
+    if not root.is_absolute():
+        root = Path.cwd() / root
+
+    if not root.exists():
+        return {
+            "source_path": str(source_path),
+            "exists": False,
+            "accepted": [],
+            "excluded": [],
+            "count": 0,
+            "excluded_count": 0,
+            "excluded_by_reason": {},
+        }
+
+    candidates = iter_candidate_files(root, allow_videos=allow_videos)
+    scan_root = root if root.is_dir() else root.parent
+    accepted, excluded = partition_media_files(
+        candidates,
+        root=scan_root,
+        dataset_type=dataset_type,
+        allow_videos=allow_videos,
+    )
+
+    if limit is not None:
+        accepted = accepted[:limit]
+
+    by_reason: Dict[str, int] = {}
+    for item in excluded:
+        reason = item.get("reason") or "excluded_path"
+        by_reason[reason] = by_reason.get(reason, 0) + 1
+
+    return {
+        "source_path": str(source_path),
+        "exists": True,
+        "accepted": accepted,
+        "excluded": excluded,
+        "count": len(accepted),
+        "excluded_count": len(excluded),
+        "excluded_by_reason": by_reason,
+    }
+
+
+def summarize_exclusions(excluded: Sequence[Dict[str, Any]]) -> Dict[str, int]:
+    counts: Dict[str, int] = {}
+    for item in excluded:
+        reason = item.get("reason") or "excluded_path"
+        counts[reason] = counts.get(reason, 0) + 1
+    return counts

sensorflow/execution_ops.py

@@ -22,8 +22,15 @@ def _resolve(path: str | Path) -> Path:
     return p
 
 
-def discover_images(source_path: str, *, max_scan: int = 100_000) -> Dict[str, Any]:
-    """Discover images under a path; report corrupt/unreadable files."""
+def discover_images(
+    source_path: str,
+    *,
+    max_scan: int = 100_000,
+    dataset_type: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Discover driving-relevant images under a path; report corrupt/unreadable files."""
+    from sensorflow.av_media_filter import partition_media_files
+
     root = _resolve(source_path)
     result: Dict[str, Any] = {
         "source_path": str(root),
@@ -32,6 +39,9 @@ def discover_images(source_path: str, *, max_scan: int = 100_000) -> Dict[str, A
         "images_discovered": 0,
         "images_readable": 0,
         "images_corrupt": 0,
+        "images_excluded": 0,
+        "excluded_by_reason": {},
+        "excluded_samples": [],
         "annotations_discovered": 0,
         "annotation_objects": 0,
         "class_ids": [],
@@ -45,11 +55,11 @@ def discover_images(source_path: str, *, max_scan: int = 100_000) -> Dict[str, A
         result["errors"].append(f"Path does not exist: {source_path}")
         return result
 
-    images: List[Path] = []
+    raw_images: List[Path] = []
     labels: List[Path] = []
     if root.is_file():
         if root.suffix.lower() in IMAGE_EXTS:
-            images = [root]
+            raw_images = [root]
         elif root.suffix.lower() in LABEL_EXTS:
             labels = [root]
         else:
@@ -61,14 +71,31 @@ def discover_images(source_path: str, *, max_scan: int = 100_000) -> Dict[str, A
                 continue
             suf = p.suffix.lower()
             if suf in IMAGE_EXTS:
-                images.append(p)
+                raw_images.append(p)
             elif suf in LABEL_EXTS:
                 labels.append(p)
-            if len(images) + len(labels) >= max_scan:
+            if len(raw_images) + len(labels) >= max_scan:
                 result["warnings"].append(f"Scan capped at {max_scan} files")
                 break
 
-    images = sorted(images)
+    scan_root = root if root.is_dir() else root.parent
+    images, excluded = partition_media_files(
+        raw_images,
+        root=scan_root,
+        dataset_type=dataset_type,
+        allow_videos=False,
+    )
+    result["images_excluded"] = len(excluded)
+    result["excluded_by_reason"] = {
+        reason: sum(1 for e in excluded if e.get("reason") == reason)
+        for reason in sorted({e.get("reason", "excluded_path") for e in excluded})
+    }
+    result["excluded_samples"] = excluded[:50]
+    if excluded:
+        result["warnings"].append(
+            f"{len(excluded)} file(s) excluded as non-driving media "
+            f"({', '.join(f'{k}: {v}' for k, v in result['excluded_by_reason'].items())})"
+        )
     labels = sorted(labels)
     result["images_discovered"] = len(images)
     result["annotations_discovered"] = len(labels)
@@ -158,6 +185,9 @@ def write_load_manifest(discovery: Dict[str, Any], dest_dir: Path) -> Path:
         "images_discovered": discovery.get("images_discovered"),
         "images_readable": discovery.get("images_readable"),
         "images_corrupt": discovery.get("images_corrupt"),
+        "images_excluded": discovery.get("images_excluded"),
+        "excluded_by_reason": discovery.get("excluded_by_reason"),
+        "excluded_samples": discovery.get("excluded_samples"),
         "annotations_discovered": discovery.get("annotations_discovered"),
         "annotation_objects": discovery.get("annotation_objects"),
         "class_ids": discovery.get("class_ids"),
@@ -175,7 +205,7 @@ def load_and_preprocess(
     dataset_type: str = "local",
 ) -> Dict[str, Any]:
     """Real Load & Preprocess: discovery + optional YAML semantic checks."""
-    discovery = discover_images(source_path)
+    discovery = discover_images(source_path, dataset_type=dataset_type)
     yaml_report = None
     if yaml_path:
         ypath = Path(yaml_path)
@@ -230,6 +260,8 @@ def load_and_preprocess(
             "images_discovered": discovery["images_discovered"],
             "images_readable": discovery["images_readable"],
             "images_corrupt": discovery["images_corrupt"],
+            "images_excluded": discovery.get("images_excluded", 0),
+            "excluded_by_reason": discovery.get("excluded_by_reason", {}),
             "annotations_discovered": discovery["annotations_discovered"],
             "annotation_objects": discovery["annotation_objects"],
             "loaded_pct_of_discovered": loaded_pct,

sensorflow/pipeline_artifacts.py

@@ -253,9 +253,19 @@ def resolve_file(path_str: str) -> Tuple[Path, str]:
     return resolved, media_type or "application/octet-stream"
 
 
-def scan_source_images(source_path: str, limit: int = 48) -> Dict[str, Any]:
-    """List local images under Dataset Configuration Images Path for validation."""
+def scan_source_images(
+    source_path: str,
+    limit: int = 48,
+    *,
+    dataset_type: Optional[str] = None,
+) -> Dict[str, Any]:
+    """List local driving-relevant images under Dataset Configuration Images Path."""
+    from sensorflow.av_media_filter import OFFICIAL_AV_DATASETS, scan_driving_media
+
     path = Path(source_path)
+    if not path.is_absolute():
+        path = Path.cwd() / path
+
     if not path.exists():
         return {
             "status": "ok",
@@ -264,19 +274,13 @@ def scan_source_images(source_path: str, limit: int = 48) -> Dict[str, Any]:
             "browsable": False,
             "count": 0,
             "images": [],
+            "excluded_count": 0,
+            "excluded": [],
+            "excluded_by_reason": {},
             "empty_reason": f"Path does not exist: {source_path}",
         }
 
-    images: List[Path] = []
-    if path.is_file() and path.suffix.lower() in IMAGE_EXTS:
-        images = [path]
-    elif path.is_dir():
-        for p in sorted(path.rglob("*")):
-            if p.is_file() and p.suffix.lower() in IMAGE_EXTS:
-                images.append(p)
-                if len(images) >= max(limit, 1):
-                    break
-    else:
+    if path.is_file() and path.suffix.lower() not in IMAGE_EXTS:
         return {
             "status": "ok",
             "source_path": source_path,
@@ -284,16 +288,24 @@ def scan_source_images(source_path: str, limit: int = 48) -> Dict[str, Any]:
             "browsable": False,
             "count": 0,
             "images": [],
-            "empty_reason": "Not an image file or directory of images.",
+            "excluded_count": 0,
+            "excluded": [],
+            "excluded_by_reason": {},
+            "empty_reason": "Not an image file or directory of driving media.",
         }
 
-    # Full count for directories (cap walk cost for huge trees by reusing discovered list when small)
-    total = len(images)
-    if path.is_dir() and total >= limit:
-        total = sum(1 for p in path.rglob("*") if p.is_file() and p.suffix.lower() in IMAGE_EXTS)
+    scan = scan_driving_media(
+        path,
+        dataset_type=dataset_type,
+        allow_videos=False,
+    )
+    accepted = scan["accepted"]
+    excluded = scan["excluded"]
+    total = scan["count"]
+    dtype = (dataset_type or "local").lower()
 
     previews = []
-    for img in images[:limit]:
+    for img in accepted[:limit]:
         previews.append(
             {
                 "name": img.name,
@@ -302,6 +314,23 @@ def scan_source_images(source_path: str, limit: int = 48) -> Dict[str, Any]:
             }
         )
 
+    empty_reason = None
+    if total == 0:
+        if excluded:
+            empty_reason = (
+                f"No driving-relevant images found under {source_path} "
+                f"({len(excluded)} file(s) excluded as non-AV media)."
+            )
+        else:
+            empty_reason = "No driving-relevant images found under this path."
+
+    warnings: List[str] = []
+    if dtype in OFFICIAL_AV_DATASETS and excluded:
+        warnings.append(
+            f"Browsed path does not match typical {dtype} layout; "
+            "point Images Path at downloaded camera frames or enable Local ingest."
+        )
+
     return {
         "status": "ok",
         "source_path": source_path,
@@ -309,5 +338,9 @@ def scan_source_images(source_path: str, limit: int = 48) -> Dict[str, Any]:
         "browsable": total > 0,
         "count": total,
         "images": previews,
-        "empty_reason": None if total else "No images found under this path.",
+        "excluded_count": scan["excluded_count"],
+        "excluded": excluded[:200],
+        "excluded_by_reason": scan["excluded_by_reason"],
+        "warnings": warnings,
+        "empty_reason": empty_reason,
     }

static/app.js

@@ -922,13 +922,22 @@ document.addEventListener('DOMContentLoaded', () => {
     if (btnBrowseSource) {
         btnBrowseSource.addEventListener('click', async () => {
             const sourcePath = datasetSourcePath?.value || document.getElementById('input-source')?.value || 'data';
+            const datasetType = selectDatasetType?.value || 'local';
             const statusEl = document.getElementById('source-browse-status');
+            const excludedEl = document.getElementById('source-browse-excluded');
             const gallery = document.getElementById('source-browse-gallery');
             btnBrowseSource.disabled = true;
             btnBrowseSource.textContent = 'Scanning…';
             if (statusEl) statusEl.textContent = `Validating ${sourcePath}…`;
+            if (excludedEl) {
+                excludedEl.style.display = 'none';
+                excludedEl.innerHTML = '';
+            }
             try {
-                const res = await fetch(`${API_BASE}/api/dataset/browse?source_path=${encodeURIComponent(sourcePath)}&limit=48`);
+                const res = await fetch(
+                    `${API_BASE}/api/dataset/browse?source_path=${encodeURIComponent(sourcePath)}`
+                    + `&limit=48&dataset_type=${encodeURIComponent(datasetType)}`
+                );
                 const data = await res.json();
                 if (gallery) {
                     gallery.innerHTML = '';
@@ -942,20 +951,53 @@ document.addEventListener('DOMContentLoaded', () => {
                     });
                 }
                 if (statusEl) {
-                    statusEl.textContent = data.browsable
-                        ? `Browsable: ${data.count} image(s) under ${data.source_path}`
+                    let msg = data.browsable
+                        ? `Browsable: ${data.count} driving-relevant image(s) under ${data.source_path}`
                         : (data.empty_reason || 'Nothing browsable at this path.');
+                    if (Array.isArray(data.warnings) && data.warnings.length) {
+                        msg += ` — ${data.warnings[0]}`;
+                    }
+                    statusEl.textContent = msg;
+                }
+                if (excludedEl && data.excluded_count > 0) {
+                    excludedEl.style.display = 'block';
+                    const summary = document.createElement('div');
+                    summary.textContent = `${data.excluded_count} file(s) excluded (not driving media)`;
+                    const toggle = document.createElement('button');
+                    toggle.type = 'button';
+                    toggle.className = 'btn btn-secondary';
+                    toggle.style.marginTop = '8px';
+                    toggle.style.fontSize = '12px';
+                    toggle.textContent = 'Show excluded files';
+                    const list = document.createElement('ul');
+                    list.style.display = 'none';
+                    list.style.marginTop = '8px';
+                    list.style.fontSize = '12px';
+                    list.style.color = 'var(--text-muted)';
+                    (data.excluded || []).forEach((item) => {
+                        const li = document.createElement('li');
+                        li.textContent = `${item.name || item.path} (${item.reason || 'excluded'})`;
+                        list.appendChild(li);
+                    });
+                    toggle.addEventListener('click', () => {
+                        const open = list.style.display === 'none';
+                        list.style.display = open ? 'block' : 'none';
+                        toggle.textContent = open ? 'Hide excluded files' : 'Show excluded files';
+                    });
+                    excludedEl.appendChild(summary);
+                    excludedEl.appendChild(toggle);
+                    excludedEl.appendChild(list);
                 }
                 const badge = document.getElementById('ds-meta-badge');
                 if (badge) {
                     if (data.browsable) {
-                        badge.textContent = `${data.count} images browsable on disk`;
+                        badge.textContent = `${data.count} driving images browsable on disk`;
                         badge.style.background = 'rgba(0, 255, 170, 0.15)';
                         badge.style.color = '#00ffaa';
                         badge.style.border = '1px solid rgba(0, 255, 170, 0.3)';
                         document.getElementById('ds-kpi-loaded-rows').textContent = data.count;
                     } else {
-                        badge.textContent = 'Not browsable — path empty or missing';
+                        badge.textContent = 'Not browsable — no driving media at path';
                         badge.style.background = 'rgba(255, 68, 68, 0.12)';
                         badge.style.color = '#ff8888';
                         badge.style.border = '1px solid rgba(255, 68, 68, 0.35)';

static/index.html

@@ -153,6 +153,7 @@
                             </div>
                             <div id="config-exec-status" class="config-exec-status" style="margin-top:10px;font-size:12px;color:var(--text-muted);"></div>
                             <div id="source-browse-status" class="status-box" style="margin-top:12px;"></div>
+                            <div id="source-browse-excluded" class="status-box" style="margin-top:8px;display:none;"></div>
                             <div id="source-browse-gallery" class="output-browse-gallery" style="margin-top:12px;"></div>
                             <div class="evidence-card" id="evidence-dataset" data-stage="dataset">
                                 <h4>What actually happened?</h4>

tests/test_pipeline/test_av_media_filter.py

@@ -0,0 +1,140 @@
+"""Tests for AV driving-media browse filtering."""
+
+from __future__ import annotations
+
+import importlib
+import shutil
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+
+def _png_bytes() -> bytes:
+    return bytes.fromhex(
+        "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
+        "0000000a49444154789c63000100000500010d0a2db40000000049454e44ae426082"
+    )
+
+
+def _large_png_bytes(min_bytes: int = 11 * 1024) -> bytes:
+    """Valid PNG header with zero padding so size heuristics accept it in tests."""
+    base = _png_bytes()
+    if len(base) >= min_bytes:
+        return base
+    return base + b"\x00" * (min_bytes - len(base))
+
+
+def _write_driving_frame(root: Path, rel: str = "images/train/frame.png") -> Path:
+    path = root / rel
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_bytes(_large_png_bytes())
+    return path
+
+
+@pytest.fixture
+def client(tmp_path, monkeypatch):
+    monkeypatch.chdir(tmp_path)
+    import app_backend
+
+    importlib.reload(app_backend)
+    return TestClient(app_backend.app)
+
+
+def test_browse_excludes_junk_test_file_for_alpamayo(client, tmp_path):
+    data_dir = tmp_path / "data"
+    data_dir.mkdir()
+    (data_dir / "test.jpg").write_bytes(_large_png_bytes())  # wrong ext still image-like size
+    _write_driving_frame(data_dir, "camera/front/scene_001.jpg").write_bytes(_large_png_bytes())
+
+    res = client.get(
+        "/api/dataset/browse?source_path=data&dataset_type=alpamayo"
+    )
+    assert res.status_code == 200
+    body = res.json()
+    assert body["count"] == 1
+    assert body["browsable"] is True
+    assert body["excluded_count"] >= 1
+    names = {img["name"] for img in body["images"]}
+    assert "scene_001.jpg" in names
+    assert "test.jpg" not in names
+    reasons = {e["reason"] for e in body["excluded"]}
+    assert "junk_filename" in reasons or "non_av_layout" in reasons
+
+
+def test_browse_excludes_docs_and_logo_paths(client, tmp_path):
+    root = tmp_path / "data"
+    _write_driving_frame(root, "frames/road.png")
+    logo = root / "docs" / "logo.png"
+    logo.parent.mkdir(parents=True)
+    logo.write_bytes(_large_png_bytes())
+    icon = root / "assets" / "icon.png"
+    icon.parent.mkdir(parents=True)
+    icon.write_bytes(_large_png_bytes())
+
+    res = client.get("/api/dataset/browse?source_path=data&dataset_type=local")
+    body = res.json()
+    assert body["count"] == 1
+    assert body["excluded_count"] == 2
+    assert all(e["reason"] == "excluded_path" for e in body["excluded"])
+
+
+def test_browse_excludes_tiny_icons(client, tmp_path):
+    root = tmp_path / "data"
+    _write_driving_frame(root, "images/train/big.png")
+    tiny = root / "images" / "train" / "favicon.png"
+    tiny.write_bytes(_png_bytes())  # < 10KB and junk name
+
+    res = client.get("/api/dataset/browse?source_path=data&dataset_type=local")
+    body = res.json()
+    assert body["count"] == 1
+    assert body["excluded_count"] == 1
+    assert body["excluded"][0]["reason"] in {"too_small", "junk_filename"}
+
+
+def test_browse_lists_valid_driving_media(client, tmp_path):
+    root = tmp_path / "data"
+    _write_driving_frame(root, "camera/front/a.png")
+
+    res = client.get("/api/dataset/browse?source_path=data&dataset_type=local")
+    assert res.status_code == 200
+    body = res.json()
+    assert body["browsable"] is True
+    assert body["count"] == 1
+    assert body["images"][0]["preview_url"].startswith("/api/pipeline/file?")
+
+
+def test_load_reports_images_excluded(client, tmp_path):
+    root = tmp_path / "data"
+    _write_driving_frame(root, "images/train/good.png")
+    (root / "sample.jpg").write_bytes(_large_png_bytes())
+
+    res = client.post(
+        "/api/dataset/load",
+        json={"source_path": str(root), "dataset_type": "local"},
+    )
+    data = res.json()
+    assert data["metrics"]["images_discovered"] == 1
+    assert data["metrics"]["images_excluded"] == 1
+    assert data["discovery"]["excluded_by_reason"]
+
+
+def test_repo_data_test_jpg_excluded_when_alpamayo_selected(client, tmp_path, monkeypatch):
+    """Regression: loose data/test.jpg must not appear for official AV dataset types."""
+    repo_data = Path(__file__).resolve().parents[2] / "data"
+    if not (repo_data / "test.jpg").exists():
+        pytest.skip("repo data/test.jpg not present")
+
+    monkeypatch.chdir(tmp_path)
+    shutil.copytree(repo_data, tmp_path / "data")
+
+    import app_backend
+
+    importlib.reload(app_backend)
+    c = TestClient(app_backend.app)
+    res = c.get("/api/dataset/browse?source_path=data&dataset_type=alpamayo")
+    body = res.json()
+    assert body["count"] == 0
+    assert body["browsable"] is False
+    assert body["excluded_count"] >= 1
+    assert not body["images"]

tests/test_pipeline/test_pipeline_browse.py

@@ -6,6 +6,14 @@ import pytest
 from fastapi.testclient import TestClient
 
 
+def _large_png_bytes(min_bytes: int = 11 * 1024) -> bytes:
+    base = bytes.fromhex(
+        "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
+        "0000000a49444154789c63000100000500010d0a2db40000000049454e44ae426082"
+    )
+    return base if len(base) >= min_bytes else base + b"\x00" * (min_bytes - len(base))
+
+
 @pytest.fixture
 def client(tmp_path, monkeypatch):
     monkeypatch.chdir(tmp_path)
@@ -118,14 +126,10 @@ def test_load_all_not_executed_without_stub(client, tmp_path):
 
 
 def test_load_all_local_real_when_path_has_images(client, tmp_path):
-    data_dir = tmp_path / "data"
-    data_dir.mkdir()
-    (data_dir / "a.png").write_bytes(
-        bytes.fromhex(
-            "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
-            "0000000a49444154789c63000100000500010d0a2db40000000049454e44ae426082"
-        )
-    )
+    data_dir = tmp_path / "data" / "images" / "train"
+    data_dir.mkdir(parents=True)
+    png = _large_png_bytes()
+    (data_dir / "a.png").write_bytes(png)
     res = client.post(
         "/api/dataset/load-all",
         json={
@@ -174,14 +178,9 @@ def test_pipeline_file_blocks_path_traversal(client, tmp_path):
 
 
 def test_dataset_browse_lists_images(client, tmp_path):
-    data_dir = tmp_path / "data"
-    data_dir.mkdir()
-    (data_dir / "a.png").write_bytes(
-        bytes.fromhex(
-            "89504e470d0a1a0a0000000d49484452000000010000000108060000001f15c489"
-            "0000000a49444154789c63000100000500010d0a2db40000000049454e44ae426082"
-        )
-    )
+    data_dir = tmp_path / "data" / "camera" / "front"
+    data_dir.mkdir(parents=True)
+    (data_dir / "a.png").write_bytes(_large_png_bytes())
     res = client.get("/api/dataset/browse?source_path=data")
     assert res.status_code == 200
     body = res.json()