fix: render timg images within Ink's virtual DOM to prevent cursor desync
Browse filestimg Unicode-block output was written to stdout via process.stdout.write(),
bypassing Ink's rendering pipeline. Ink's virtual cursor didn't know about
the image rows, causing subsequent UI (prompt, input bar) to render at wrong
positions — overlapping the image and duplicating input bars.
Now the timg ANSI output is passed through the tool data and rendered via
the RawAnsi component within Ink's virtual DOM. Ink's Yoga layout accounts
for the image's width × height, keeping cursor position in sync.
Co-Authored-By: Claude Big Pickle <noreply@anthropic.com>
src/tools/ImageShowTool/ImageShowTool.tsx
CHANGED
|
@@ -2,7 +2,7 @@ import { readFile } from 'fs/promises'
|
|
| 2 |
import { homedir } from 'os'
|
| 3 |
import React, { useContext, useEffect, useRef } from 'react'
|
| 4 |
import { z } from 'zod/v4'
|
| 5 |
-
import { Text } from '../../ink.js'
|
| 6 |
import { TerminalWriteContext } from '../../ink/useTerminalNotification.js'
|
| 7 |
import { buildTool, type ToolDef } from '../../Tool.js'
|
| 8 |
import { logForDebugging } from '../../utils/debug.js'
|
|
@@ -10,7 +10,7 @@ import {
|
|
| 10 |
detectImageProtocol,
|
| 11 |
encodeKittyImage,
|
| 12 |
isInsideTmux,
|
| 13 |
-
|
| 14 |
} from '../../utils/terminalImage.js'
|
| 15 |
|
| 16 |
const IMAGE_TOOL_NAME = 'ImageShow'
|
|
@@ -122,16 +122,11 @@ function getToolUseSummary(input: Partial<Input>): string | null {
|
|
| 122 |
}
|
| 123 |
|
| 124 |
/**
|
| 125 |
-
* React component that
|
| 126 |
-
*
|
| 127 |
-
*
|
| 128 |
-
*
|
| 129 |
-
*
|
| 130 |
-
* every modern terminal (Unicode + 24-bit color required).
|
| 131 |
-
* 3. Plain text message – when neither method is available.
|
| 132 |
-
*
|
| 133 |
-
* The image is rendered via Ink's writeRaw so the escape sequences stay
|
| 134 |
-
* synchronized with Ink's render cycle and avoid cursor-position races.
|
| 135 |
*/
|
| 136 |
function TerminalImageDisplay({
|
| 137 |
base64,
|
|
@@ -146,37 +141,22 @@ function TerminalImageDisplay({
|
|
| 146 |
const renderedRef = useRef(false)
|
| 147 |
|
| 148 |
useEffect(() => {
|
| 149 |
-
if (renderedRef.current || !
|
| 150 |
renderedRef.current = true
|
| 151 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 152 |
const protocol = detectImageProtocol()
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
// Path A: Kitty protocol – best quality, works natively in Kitty,
|
| 156 |
-
// Ghostty, WezTerm, Konsole, foot outside tmux.
|
| 157 |
-
if (protocol === 'kitty' && !isInsideTmux()) {
|
| 158 |
const sequence = encodeKittyImage(buf, format)
|
| 159 |
if (sequence) {
|
| 160 |
-
writeRaw(sequence)
|
| 161 |
logForDebugging('ImageShow: displayed via Kitty protocol')
|
| 162 |
}
|
| 163 |
-
return
|
| 164 |
}
|
| 165 |
-
|
| 166 |
-
// Path B: timg Unicode-block rendering – reliable in all modern
|
| 167 |
-
// terminals including inside tmux where Kitty passthrough may be blocked.
|
| 168 |
-
renderImageWithTimg(buf, format)
|
| 169 |
-
.then(output => {
|
| 170 |
-
if (output) {
|
| 171 |
-
writeRaw(output)
|
| 172 |
-
logForDebugging('ImageShow: displayed via timg Unicode blocks')
|
| 173 |
-
} else {
|
| 174 |
-
logForDebugging('ImageShow: timg not available, showing text only')
|
| 175 |
-
}
|
| 176 |
-
})
|
| 177 |
-
.catch(() => {
|
| 178 |
-
logForDebugging('ImageShow: timg render error, showing text only')
|
| 179 |
-
})
|
| 180 |
}, [base64, format, writeRaw])
|
| 181 |
|
| 182 |
return <Text dimColor>{message}</Text>
|
|
@@ -268,6 +248,7 @@ export const ImageShowTool = buildTool({
|
|
| 268 |
message: string
|
| 269 |
base64?: string
|
| 270 |
format?: string
|
|
|
|
| 271 |
},
|
| 272 |
_progressMessages,
|
| 273 |
_options,
|
|
@@ -275,6 +256,22 @@ export const ImageShowTool = buildTool({
|
|
| 275 |
if (!content.success) {
|
| 276 |
return null
|
| 277 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 278 |
return (
|
| 279 |
<TerminalImageDisplay
|
| 280 |
base64={content.base64}
|
|
@@ -291,6 +288,7 @@ export const ImageShowTool = buildTool({
|
|
| 291 |
imageData?: { base64: string; mediaType: string }
|
| 292 |
base64?: string
|
| 293 |
format?: string
|
|
|
|
| 294 |
}
|
| 295 |
}> {
|
| 296 |
const url = input.url
|
|
@@ -322,8 +320,20 @@ export const ImageShowTool = buildTool({
|
|
| 322 |
|
| 323 |
logForDebugging(`ImageShow: loaded ${buffer.length} byte ${format} image`)
|
| 324 |
|
| 325 |
-
//
|
| 326 |
-
//
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 327 |
return {
|
| 328 |
data: {
|
| 329 |
success: true,
|
|
@@ -334,6 +344,7 @@ export const ImageShowTool = buildTool({
|
|
| 334 |
},
|
| 335 |
base64: buffer.toString('base64'),
|
| 336 |
format,
|
|
|
|
| 337 |
},
|
| 338 |
}
|
| 339 |
},
|
|
|
|
| 2 |
import { homedir } from 'os'
|
| 3 |
import React, { useContext, useEffect, useRef } from 'react'
|
| 4 |
import { z } from 'zod/v4'
|
| 5 |
+
import { RawAnsi, Text } from '../../ink.js'
|
| 6 |
import { TerminalWriteContext } from '../../ink/useTerminalNotification.js'
|
| 7 |
import { buildTool, type ToolDef } from '../../Tool.js'
|
| 8 |
import { logForDebugging } from '../../utils/debug.js'
|
|
|
|
| 10 |
detectImageProtocol,
|
| 11 |
encodeKittyImage,
|
| 12 |
isInsideTmux,
|
| 13 |
+
renderImageWithTimgSync,
|
| 14 |
} from '../../utils/terminalImage.js'
|
| 15 |
|
| 16 |
const IMAGE_TOOL_NAME = 'ImageShow'
|
|
|
|
| 122 |
}
|
| 123 |
|
| 124 |
/**
|
| 125 |
+
* React component that renders the image via the Kitty graphics protocol
|
| 126 |
+
* when the terminal naturally supports it (Kitty/Ghostty/WezTerm outside
|
| 127 |
+
* tmux). timg fallback is rendered in-band via RawAnsi (in
|
| 128 |
+
* renderToolResultMessage), so Ink knows the image dimensions and its
|
| 129 |
+
* virtual cursor stays in sync with the terminal.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 130 |
*/
|
| 131 |
function TerminalImageDisplay({
|
| 132 |
base64,
|
|
|
|
| 141 |
const renderedRef = useRef(false)
|
| 142 |
|
| 143 |
useEffect(() => {
|
| 144 |
+
if (renderedRef.current || !writeRaw) return
|
| 145 |
renderedRef.current = true
|
| 146 |
|
| 147 |
+
// Kitty protocol: timg fallback is rendered in renderToolResultMessage
|
| 148 |
+
// via RawAnsi (in-band with Ink's virtual DOM), so Ink knows the image
|
| 149 |
+
// dimensions and cursor position stays correct.
|
| 150 |
+
// Only use Kitty when it can reach the terminal natively.
|
| 151 |
const protocol = detectImageProtocol()
|
| 152 |
+
if (protocol === 'kitty' && !isInsideTmux() && base64 && format) {
|
| 153 |
+
const buf = Buffer.from(base64, 'base64')
|
|
|
|
|
|
|
|
|
|
| 154 |
const sequence = encodeKittyImage(buf, format)
|
| 155 |
if (sequence) {
|
| 156 |
+
writeRaw(sequence + '\n')
|
| 157 |
logForDebugging('ImageShow: displayed via Kitty protocol')
|
| 158 |
}
|
|
|
|
| 159 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 160 |
}, [base64, format, writeRaw])
|
| 161 |
|
| 162 |
return <Text dimColor>{message}</Text>
|
|
|
|
| 248 |
message: string
|
| 249 |
base64?: string
|
| 250 |
format?: string
|
| 251 |
+
timgOutput?: string
|
| 252 |
},
|
| 253 |
_progressMessages,
|
| 254 |
_options,
|
|
|
|
| 256 |
if (!content.success) {
|
| 257 |
return null
|
| 258 |
}
|
| 259 |
+
|
| 260 |
+
// timg output rendered in-band via RawAnsi so Ink knows the image
|
| 261 |
+
// dimensions and its virtual cursor stays in sync with the terminal.
|
| 262 |
+
if (content.timgOutput) {
|
| 263 |
+
// Strip cursor hide/show sequences that timg adds
|
| 264 |
+
const cleaned = content.timgOutput.replace(/\x1b\[\?25[hl]/g, '')
|
| 265 |
+
const lines = cleaned.split('\n').filter(l => l.length > 0)
|
| 266 |
+
if (lines.length === 0) {
|
| 267 |
+
return <Text dimColor>{content.message}</Text>
|
| 268 |
+
}
|
| 269 |
+
// Measure visible width (strip ANSI escape codes)
|
| 270 |
+
const ansiStrip = (s: string) => s.replace(/\x1b\[[0-9;?]*[a-zA-Z]/g, '')
|
| 271 |
+
const width = Math.max(...lines.map(l => ansiStrip(l).length))
|
| 272 |
+
return <RawAnsi lines={lines} width={width} />
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
return (
|
| 276 |
<TerminalImageDisplay
|
| 277 |
base64={content.base64}
|
|
|
|
| 288 |
imageData?: { base64: string; mediaType: string }
|
| 289 |
base64?: string
|
| 290 |
format?: string
|
| 291 |
+
timgOutput?: string
|
| 292 |
}
|
| 293 |
}> {
|
| 294 |
const url = input.url
|
|
|
|
| 320 |
|
| 321 |
logForDebugging(`ImageShow: loaded ${buffer.length} byte ${format} image`)
|
| 322 |
|
| 323 |
+
// Generate timg Unicode-block rendering. The output is passed through
|
| 324 |
+
// to the Ink virtual DOM via RawAnsi in renderToolResultMessage, so
|
| 325 |
+
// Ink knows the image dimensions and its cursor stays in sync.
|
| 326 |
+
// Kitty protocol is separately handled in TerminalImageDisplay via
|
| 327 |
+
// writeRaw (only when the terminal natively supports it outside tmux).
|
| 328 |
+
const protocol = detectImageProtocol()
|
| 329 |
+
let timgOutput: string | undefined
|
| 330 |
+
if (isInsideTmux() || protocol !== 'kitty') {
|
| 331 |
+
timgOutput = renderImageWithTimgSync(buffer, format) ?? undefined
|
| 332 |
+
if (timgOutput) {
|
| 333 |
+
logForDebugging('ImageShow: displayed via timg Unicode blocks (in-band with Ink)')
|
| 334 |
+
}
|
| 335 |
+
}
|
| 336 |
+
|
| 337 |
return {
|
| 338 |
data: {
|
| 339 |
success: true,
|
|
|
|
| 344 |
},
|
| 345 |
base64: buffer.toString('base64'),
|
| 346 |
format,
|
| 347 |
+
timgOutput,
|
| 348 |
},
|
| 349 |
}
|
| 350 |
},
|
src/utils/terminalImage.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
import { mkdtempSync, rmSync, writeFileSync } from 'fs'
|
| 2 |
import { tmpdir } from 'os'
|
| 3 |
import { join } from 'path'
|
|
@@ -5,7 +6,7 @@ import { Buffer } from 'buffer'
|
|
| 5 |
import { env } from './env.js'
|
| 6 |
import { execFileNoThrow } from './execFileNoThrow.js'
|
| 7 |
import { wrapForMultiplexer } from '../ink/termio/osc.js'
|
| 8 |
-
import { which } from './which.js'
|
| 9 |
|
| 10 |
export type ImageProtocol = 'kitty' | null
|
| 11 |
|
|
@@ -203,3 +204,70 @@ export async function renderImageWithTimg(
|
|
| 203 |
return null
|
| 204 |
}
|
| 205 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import { execFileSync } from 'child_process'
|
| 2 |
import { mkdtempSync, rmSync, writeFileSync } from 'fs'
|
| 3 |
import { tmpdir } from 'os'
|
| 4 |
import { join } from 'path'
|
|
|
|
| 6 |
import { env } from './env.js'
|
| 7 |
import { execFileNoThrow } from './execFileNoThrow.js'
|
| 8 |
import { wrapForMultiplexer } from '../ink/termio/osc.js'
|
| 9 |
+
import { which, whichSync } from './which.js'
|
| 10 |
|
| 11 |
export type ImageProtocol = 'kitty' | null
|
| 12 |
|
|
|
|
| 204 |
return null
|
| 205 |
}
|
| 206 |
}
|
| 207 |
+
|
| 208 |
+
/**
|
| 209 |
+
* Synchronous version of renderImageWithTimg. Blocks the event loop while
|
| 210 |
+
* spawning timg, which prevents Ink from rendering between the tool call
|
| 211 |
+
* and the image output — eliminating cursor-position races.
|
| 212 |
+
*
|
| 213 |
+
* @see renderImageWithTimg
|
| 214 |
+
*/
|
| 215 |
+
export function renderImageWithTimgSync(
|
| 216 |
+
buffer: Buffer,
|
| 217 |
+
format: string,
|
| 218 |
+
columns?: number,
|
| 219 |
+
rows?: number,
|
| 220 |
+
): string | null {
|
| 221 |
+
try {
|
| 222 |
+
const timgPath = whichSync('timg')
|
| 223 |
+
if (!timgPath) return null
|
| 224 |
+
|
| 225 |
+
const cols = columns ?? process.stdout.columns ?? 80
|
| 226 |
+
const termRows = rows ?? process.stdout.rows ?? 40
|
| 227 |
+
const maxRows = Math.max(10, Math.floor(termRows * 0.5))
|
| 228 |
+
const tmpDir = mkdtempSync(join(tmpdir(), 'versperclaw-timg-'))
|
| 229 |
+
const ext = format === 'jpeg' ? 'jpg' : format
|
| 230 |
+
const tmpFile = join(tmpDir, `image.${ext}`)
|
| 231 |
+
let result: string | null = null
|
| 232 |
+
|
| 233 |
+
try {
|
| 234 |
+
writeFileSync(tmpFile, buffer)
|
| 235 |
+
|
| 236 |
+
// Try quarter blocks first (4 pixels per cell, better quality)
|
| 237 |
+
try {
|
| 238 |
+
const stdout = execFileSync(
|
| 239 |
+
timgPath,
|
| 240 |
+
['-p', 'q', '-g', `${cols}x${maxRows}`, tmpFile],
|
| 241 |
+
{ encoding: 'utf8', timeout: 15000, maxBuffer: 10 * 1024 * 1024 },
|
| 242 |
+
)
|
| 243 |
+
if (stdout) result = stdout
|
| 244 |
+
} catch {
|
| 245 |
+
// fall through to half blocks
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
// Fallback to half blocks (2 pixels per cell, max compatibility)
|
| 249 |
+
if (!result) {
|
| 250 |
+
try {
|
| 251 |
+
const stdout = execFileSync(
|
| 252 |
+
timgPath,
|
| 253 |
+
['-p', 'h', '-g', `${cols}x${maxRows}`, tmpFile],
|
| 254 |
+
{ encoding: 'utf8', timeout: 15000, maxBuffer: 10 * 1024 * 1024 },
|
| 255 |
+
)
|
| 256 |
+
if (stdout) result = stdout
|
| 257 |
+
} catch {
|
| 258 |
+
// ignored
|
| 259 |
+
}
|
| 260 |
+
}
|
| 261 |
+
} finally {
|
| 262 |
+
try {
|
| 263 |
+
rmSync(tmpDir, { recursive: true, force: true })
|
| 264 |
+
} catch {
|
| 265 |
+
// ignore cleanup errors
|
| 266 |
+
}
|
| 267 |
+
}
|
| 268 |
+
|
| 269 |
+
return result
|
| 270 |
+
} catch {
|
| 271 |
+
return null
|
| 272 |
+
}
|
| 273 |
+
}
|