feat: image in ui

src/ink/output.ts

@@ -704,29 +704,67 @@ function writeLineToScreen(
               break
             }
           }
+        } else if (nextChar === '_') {
+          // APC (Application Program Command): pass through to screen
+          // buffer. Terminal receives the raw \x1b_G...\x1b\\ sequence
+          // and interprets it as a Kitty graphics protocol command,
+          // rendering the native image at this position within Ink's
+          // layout — no \x1b[H or writeRaw positioning needed.
+          setCellAt(screen, offsetX, y, {
+            char: '\x1b',
+            styleId: stylePool.none,
+            width: CellWidth.Narrow,
+            hyperlink: undefined,
+          })
+          offsetX++
+          setCellAt(screen, offsetX, y, {
+            char: '_',
+            styleId: stylePool.none,
+            width: CellWidth.Narrow,
+            hyperlink: undefined,
+          })
+          offsetX++
+          charIdx++ // skip past ESC
+          while (charIdx < characters.length - 1) {
+            charIdx++
+            const c = characters[charIdx]?.value
+            if (c === undefined) break
+            setCellAt(screen, offsetX, y, {
+              char: c,
+              styleId: stylePool.none,
+              width: CellWidth.Narrow,
+              hyperlink: undefined,
+            })
+            offsetX++
+            if (c === '\x07') break
+            if (c === '\x1b') {
+              const nextC = characters[charIdx + 1]?.value
+              if (nextC === '\\') {
+                charIdx++
+                setCellAt(screen, offsetX, y, {
+                  char: '\\',
+                  styleId: stylePool.none,
+                  width: CellWidth.Narrow,
+                  hyperlink: undefined,
+                })
+                offsetX++
+                break
+              }
+            }
+          }
         } else if (
           nextChar === ']' ||
           nextChar === 'P' ||
-          nextChar === '_' ||
           nextChar === '^' ||
           nextChar === 'X'
         ) {
-          // String-based sequences terminated by BEL (0x07) or ST (ESC \):
-          // - OSC: ESC ] ... (Operating System Command)
-          // - DCS: ESC P ... (Device Control String)
-          // - APC: ESC _ ... (Application Program Command)
-          // - PM:  ESC ^ ... (Privacy Message)
-          // - SOS: ESC X ... (Start of String)
+          // String-based sequences (OSC, DCS, PM, SOS) terminated by
+          // BEL (0x07) or ST (ESC \): skip as before.
           charIdx++ // skip the introducer char
           while (charIdx < characters.length - 1) {
             charIdx++
             const c = characters[charIdx]?.value
-            // BEL (0x07) terminates the sequence
-            if (c === '\x07') {
-              break
-            }
-            // ST (String Terminator) is ESC \
-            // When we see ESC, check if next char is backslash
+            if (c === '\x07') break
             if (c === '\x1b') {
               const nextC = characters[charIdx + 1]?.value
               if (nextC === '\\') {

src/tools/ImageShowTool/ImageShowTool.tsx

@@ -1,9 +1,8 @@
 import { readFile } from 'fs/promises'
 import { homedir } from 'os'
-import React, { useContext, useEffect, useRef } from 'react'
+import React from 'react'
 import { z } from 'zod/v4'
 import { RawAnsi, Text } from '../../ink.js'
-import { TerminalWriteContext } from '../../ink/useTerminalNotification.js'
 import { wrapForMultiplexer } from '../../ink/termio/osc.js'
 import { buildTool, type ToolDef } from '../../Tool.js'
 import { logForDebugging } from '../../utils/debug.js'
@@ -40,118 +39,18 @@ function detectFormat(url: string): string {
   return 'png'
 }
 
-/** Read a local file and return its buffer + detected format */
-async function readLocalImage(url: string): Promise<{ buffer: Buffer; format: string } | null> {
-  try {
-    const format = detectFormat(url)
-    const buffer = await readFile(url)
-    return { buffer, format }
-  } catch {
-    return null
-  }
-}
-
-/** Download an image from URL and return its buffer + detected format */
-async function fetchImage(url: string): Promise<{ buffer: Buffer; format: string } | null> {
-  try {
-    const response = await fetch(url, {
-      headers: {
-        'User-Agent': 'Mozilla/5.0 (compatible; VersperClaw/1.0)',
-      },
-      redirect: 'follow',
-    })
-
-    if (!response.ok) {
-      logForDebugging(`ImageShow: HTTP ${response.status} for ${url}`)
-      return null
-    }
-
-    const contentType = response.headers.get('content-type') ?? ''
-    const format = detectFormat(url !== contentType ? url : contentType)
-
-    const reader = response.body?.getReader()
-    if (!reader) return null
-
-    const chunks: Uint8Array[] = []
-    let totalSize = 0
-    const MAX_SIZE = 10_000_000
-
-    while (true) {
-      const { done, value } = await reader.read()
-      if (done) break
-      totalSize += value.byteLength
-      if (totalSize > MAX_SIZE) {
-        logForDebugging(`ImageShow: image too large (${totalSize} bytes) for ${url}`)
-        reader.cancel()
-        return null
-      }
-      chunks.push(value)
-    }
-
-    const combinedLength = chunks.reduce((acc, c) => acc + c.byteLength, 0)
-    const combined = new Uint8Array(combinedLength)
-    let offset = 0
-    for (const chunk of chunks) {
-      combined.set(chunk, offset)
-      offset += chunk.byteLength
-    }
-
-    return { buffer: Buffer.from(combined.buffer), format }
-  } catch (err) {
-    logForDebugging(`ImageShow: fetch error ${err} for ${url}`)
-    return null
-  }
-}
-
-/** Load image: local file or remote URL */
-async function loadImage(url: string): Promise<{ buffer: Buffer; format: string } | null> {
-  // Expand ~ to home directory
-  const normalizedUrl = url.startsWith('~')
-    ? url.replace(/^~(?=$|\/)/, homedir())
-    : url
-  if (normalizedUrl.startsWith('file://') || normalizedUrl.startsWith('/') || normalizedUrl.startsWith('.')) {
-    const path = normalizedUrl.startsWith('file://') ? normalizedUrl.slice(7) : normalizedUrl
-    return readLocalImage(path)
-  }
-  return fetchImage(normalizedUrl)
-}
-
 function getToolUseSummary(input: Partial<Input>): string | null {
   return input?.url ? `Show: ${input.url.split('/').pop() ?? input.url}` : null
 }
 
-/**
- * Renders a pre-generated Kitty protocol image sequence by writing it
- * directly to stdout via writeRaw (bypassing Ink's virtual DOM so the
- * native image protocol reaches the terminal cleanly).
- */
-function TerminalImageDisplay({
-  sequence,
-  message,
-}: {
-  sequence: string
-  message: string
-}): React.ReactNode {
-  const writeRaw = useContext(TerminalWriteContext)
-  const renderedRef = useRef(false)
-
-  useEffect(() => {
-    if (renderedRef.current || !writeRaw) return
-    renderedRef.current = true
-    writeRaw(sequence + '\n')
-    logForDebugging('ImageShow: displayed via Kitty protocol (timg)')
-  }, [sequence, writeRaw])
-
-  return <Text dimColor>{message}</Text>
-}
-
 export const ImageShowTool = buildTool({
   name: IMAGE_TOOL_NAME,
   description:
     'Display an image (PNG/JPEG/GIF/WebP) directly in the terminal. ' +
-    'Uses the Kitty graphics protocol via timg when the terminal supports it, ' +
-    'or falls back to timg Unicode-block rendering for universal compatibility. ' +
-    'Supports both URLs (https://) and local file paths. Images are shown inline above the tool result.',
+    'Uses the Kitty graphics protocol via timg for native quality when the ' +
+    'terminal supports it, with a Unicode-block fallback for universal compatibility. ' +
+    'The block-mode output is rendered within Ink\'s virtual DOM so the cursor ' +
+    'stays in sync and the image scrolls with conversation content.',
 
   getToolUseSummary,
   getActivityDescription(input) {
@@ -229,8 +128,6 @@ export const ImageShowTool = buildTool({
     content: {
       success: boolean
       message: string
-      base64?: string
-      format?: string
       kittyOutput?: string
       timgOutput?: string
     },
@@ -241,31 +138,22 @@ export const ImageShowTool = buildTool({
       return null
     }
 
-    // Kitty protocol output rendered via writeRaw (out-of-band) for native
-    // terminal image rendering. The text message tells Ink the image exists
-    // so it doesn't leave a blank gap.
-    if (content.kittyOutput) {
-      return (
-        <TerminalImageDisplay
-          sequence={content.kittyOutput}
-          message={content.message}
-        />
-      )
-    }
-
-    // timg block-mode output rendered in-band via RawAnsi so Ink knows the
-    // image dimensions and its virtual cursor stays in sync with the terminal.
+    // Block-mode placeholder rendered in-band via RawAnsi so Ink knows the
+    // image dimensions and its virtual cursor stays in sync. The Kitty protocol
+    // APC escape sequence (when available) is prepended to the first block-mode
+    // line; output.ts's writeLineToScreen passes APC through to the screen
+    // buffer, so the terminal receives the native image at this position.
     if (content.timgOutput) {
-      // Strip cursor hide/show sequences that timg adds
       const cleaned = content.timgOutput.replace(/\x1b\[\?25[hl]/g, '')
       const lines = cleaned.split('\n').filter(l => l.length > 0)
-      if (lines.length === 0) {
-        return <Text dimColor>{content.message}</Text>
+      if (lines.length > 0) {
+        const ansiStrip = (s: string) => s.replace(/\x1b\[[0-9;?]*[a-zA-Z]/g, '')
+        const width = Math.max(...lines.map(l => ansiStrip(l).length))
+        if (content.kittyOutput) {
+          lines[0] = content.kittyOutput + lines[0]
+        }
+        return <RawAnsi lines={lines} width={width} />
       }
-      // Measure visible width (strip ANSI escape codes)
-      const ansiStrip = (s: string) => s.replace(/\x1b\[[0-9;?]*[a-zA-Z]/g, '')
-      const width = Math.max(...lines.map(l => ansiStrip(l).length))
-      return <RawAnsi lines={lines} width={width} />
     }
 
     return <Text dimColor>{content.message}</Text>
@@ -287,9 +175,60 @@ export const ImageShowTool = buildTool({
 
     logForDebugging(`ImageShow: loading ${url}`)
 
-    const result = await loadImage(url)
+    // Expand ~ to home directory for local files
+    const normalizedUrl = url.startsWith('~')
+      ? url.replace(/^~(?=$|\/)/, homedir())
+      : url
+
+    let imgResult: { buffer: Buffer; format: string } | null = null
+
+    if (normalizedUrl.startsWith('file://') || normalizedUrl.startsWith('/') || normalizedUrl.startsWith('.')) {
+      const path = normalizedUrl.startsWith('file://') ? normalizedUrl.slice(7) : normalizedUrl
+      try {
+        const format = detectFormat(path)
+        const buffer = await readFile(path)
+        imgResult = { buffer, format }
+      } catch {
+        // fall through
+      }
+    } else {
+      try {
+        const response = await fetch(normalizedUrl, {
+          headers: { 'User-Agent': 'Mozilla/5.0 (compatible; VersperClaw/1.0)' },
+          redirect: 'follow',
+        })
+        if (response.ok) {
+          const contentType = response.headers.get('content-type') ?? ''
+          const format = detectFormat(normalizedUrl !== contentType ? normalizedUrl : contentType)
+          const reader = response.body?.getReader()
+          if (reader) {
+            const chunks: Uint8Array[] = []
+            let totalSize = 0
+            while (true) {
+              const { done, value } = await reader.read()
+              if (done) break
+              totalSize += value.byteLength
+              if (totalSize > 10_000_000) {
+                reader.cancel()
+                break
+              }
+              chunks.push(value)
+            }
+            const combined = new Uint8Array(chunks.reduce((acc, c) => acc + c.byteLength, 0))
+            let offset = 0
+            for (const chunk of chunks) {
+              combined.set(chunk, offset)
+              offset += chunk.byteLength
+            }
+            imgResult = { buffer: Buffer.from(combined.buffer), format }
+          }
+        }
+      } catch (err) {
+        logForDebugging(`ImageShow: fetch error ${err} for ${normalizedUrl}`)
+      }
+    }
 
-    if (!result) {
+    if (!imgResult) {
       return {
         data: {
           success: false,
@@ -298,24 +237,18 @@ export const ImageShowTool = buildTool({
       }
     }
 
-    const { buffer, format } = result
-
+    const { buffer, format } = imgResult
     const mediaType =
-      format === 'jpeg'
-        ? 'image/jpeg'
-        : format === 'gif'
-          ? 'image/gif'
-          : format === 'webp'
-            ? 'image/webp'
+      format === 'jpeg' ? 'image/jpeg'
+        : format === 'gif' ? 'image/gif'
+          : format === 'webp' ? 'image/webp'
             : 'image/png'
 
     logForDebugging(`ImageShow: loaded ${buffer.length} byte ${format} image`)
 
-    // Try native Kitty protocol rendering via timg when the terminal supports it.
-    // timg's -p kitty produces Kitty protocol escape sequences natively,
-    // which are then wrapped for tmux passthrough if needed and written
-    // directly to stdout via writeRaw in TerminalImageDisplay.
-    // Falls back to Unicode block rendering for universal compatibility.
+    // Hybrid rendering:
+    // 1. Kitty protocol via timg for native-quality overlay (when supported)
+    // 2. Block-mode via timg for Ink DOM cursor tracking (always, as fallback)
     const protocol = detectImageProtocol()
     let kittyOutput: string | undefined
     let timgOutput: string | undefined
@@ -324,12 +257,13 @@ export const ImageShowTool = buildTool({
       const rawKitty = renderImageWithTimgSync(buffer, format, undefined, undefined, 'kitty')
       if (rawKitty) {
         kittyOutput = wrapForMultiplexer(rawKitty)
-        logForDebugging('ImageShow: generated via timg Kitty protocol')
+        logForDebugging('ImageShow: generated Kitty protocol output via timg')
       }
     }
 
-    if (!kittyOutput) {
-      timgOutput = renderImageWithTimgSync(buffer, format, undefined, undefined, 'blocks') ?? undefined
+    timgOutput = renderImageWithTimgSync(buffer, format, undefined, undefined, 'blocks') ?? undefined
+    if (timgOutput) {
+      logForDebugging('ImageShow: generated block-mode output via timg')
     }
 
     return {