chenbhao Claude Big Pickle commited on
Commit
80a03ed
·
1 Parent(s): f03a328

fix: render timg images within Ink's virtual DOM to prevent cursor desync

Browse files

timg Unicode-block output was written to stdout via process.stdout.write(),
bypassing Ink's rendering pipeline. Ink's virtual cursor didn't know about
the image rows, causing subsequent UI (prompt, input bar) to render at wrong
positions — overlapping the image and duplicating input bars.

Now the timg ANSI output is passed through the tool data and rendered via
the RawAnsi component within Ink's virtual DOM. Ink's Yoga layout accounts
for the image's width × height, keeping cursor position in sync.

Co-Authored-By: Claude Big Pickle <noreply@anthropic.com>

src/tools/ImageShowTool/ImageShowTool.tsx CHANGED
@@ -2,7 +2,7 @@ import { readFile } from 'fs/promises'
2
  import { homedir } from 'os'
3
  import React, { useContext, useEffect, useRef } from 'react'
4
  import { z } from 'zod/v4'
5
- import { Text } from '../../ink.js'
6
  import { TerminalWriteContext } from '../../ink/useTerminalNotification.js'
7
  import { buildTool, type ToolDef } from '../../Tool.js'
8
  import { logForDebugging } from '../../utils/debug.js'
@@ -10,7 +10,7 @@ import {
10
  detectImageProtocol,
11
  encodeKittyImage,
12
  isInsideTmux,
13
- renderImageWithTimg,
14
  } from '../../utils/terminalImage.js'
15
 
16
  const IMAGE_TOOL_NAME = 'ImageShow'
@@ -122,16 +122,11 @@ function getToolUseSummary(input: Partial<Input>): string | null {
122
  }
123
 
124
  /**
125
- * React component that displays an image in the terminal using the best
126
- * available method:
127
- * 1. Kitty graphics protocol when the terminal supports it AND we're
128
- * NOT inside tmux (where Kitty passthrough may be blocked).
129
- * 2. `timg` Unicode blocks reliable fallback that works in virtually
130
- * every modern terminal (Unicode + 24-bit color required).
131
- * 3. Plain text message – when neither method is available.
132
- *
133
- * The image is rendered via Ink's writeRaw so the escape sequences stay
134
- * synchronized with Ink's render cycle and avoid cursor-position races.
135
  */
136
  function TerminalImageDisplay({
137
  base64,
@@ -146,37 +141,22 @@ function TerminalImageDisplay({
146
  const renderedRef = useRef(false)
147
 
148
  useEffect(() => {
149
- if (renderedRef.current || !base64 || !format || !writeRaw) return
150
  renderedRef.current = true
151
 
 
 
 
 
152
  const protocol = detectImageProtocol()
153
- const buf = Buffer.from(base64, 'base64')
154
-
155
- // Path A: Kitty protocol – best quality, works natively in Kitty,
156
- // Ghostty, WezTerm, Konsole, foot outside tmux.
157
- if (protocol === 'kitty' && !isInsideTmux()) {
158
  const sequence = encodeKittyImage(buf, format)
159
  if (sequence) {
160
- writeRaw(sequence)
161
  logForDebugging('ImageShow: displayed via Kitty protocol')
162
  }
163
- return
164
  }
165
-
166
- // Path B: timg Unicode-block rendering – reliable in all modern
167
- // terminals including inside tmux where Kitty passthrough may be blocked.
168
- renderImageWithTimg(buf, format)
169
- .then(output => {
170
- if (output) {
171
- writeRaw(output)
172
- logForDebugging('ImageShow: displayed via timg Unicode blocks')
173
- } else {
174
- logForDebugging('ImageShow: timg not available, showing text only')
175
- }
176
- })
177
- .catch(() => {
178
- logForDebugging('ImageShow: timg render error, showing text only')
179
- })
180
  }, [base64, format, writeRaw])
181
 
182
  return <Text dimColor>{message}</Text>
@@ -268,6 +248,7 @@ export const ImageShowTool = buildTool({
268
  message: string
269
  base64?: string
270
  format?: string
 
271
  },
272
  _progressMessages,
273
  _options,
@@ -275,6 +256,22 @@ export const ImageShowTool = buildTool({
275
  if (!content.success) {
276
  return null
277
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
278
  return (
279
  <TerminalImageDisplay
280
  base64={content.base64}
@@ -291,6 +288,7 @@ export const ImageShowTool = buildTool({
291
  imageData?: { base64: string; mediaType: string }
292
  base64?: string
293
  format?: string
 
294
  }
295
  }> {
296
  const url = input.url
@@ -322,8 +320,20 @@ export const ImageShowTool = buildTool({
322
 
323
  logForDebugging(`ImageShow: loaded ${buffer.length} byte ${format} image`)
324
 
325
- // Return data with base64 and format so renderToolResultMessage can
326
- // pass them to KittyImageDisplay for writeRaw-based terminal output.
 
 
 
 
 
 
 
 
 
 
 
 
327
  return {
328
  data: {
329
  success: true,
@@ -334,6 +344,7 @@ export const ImageShowTool = buildTool({
334
  },
335
  base64: buffer.toString('base64'),
336
  format,
 
337
  },
338
  }
339
  },
 
2
  import { homedir } from 'os'
3
  import React, { useContext, useEffect, useRef } from 'react'
4
  import { z } from 'zod/v4'
5
+ import { RawAnsi, Text } from '../../ink.js'
6
  import { TerminalWriteContext } from '../../ink/useTerminalNotification.js'
7
  import { buildTool, type ToolDef } from '../../Tool.js'
8
  import { logForDebugging } from '../../utils/debug.js'
 
10
  detectImageProtocol,
11
  encodeKittyImage,
12
  isInsideTmux,
13
+ renderImageWithTimgSync,
14
  } from '../../utils/terminalImage.js'
15
 
16
  const IMAGE_TOOL_NAME = 'ImageShow'
 
122
  }
123
 
124
  /**
125
+ * React component that renders the image via the Kitty graphics protocol
126
+ * when the terminal naturally supports it (Kitty/Ghostty/WezTerm outside
127
+ * tmux). timg fallback is rendered in-band via RawAnsi (in
128
+ * renderToolResultMessage), so Ink knows the image dimensions and its
129
+ * virtual cursor stays in sync with the terminal.
 
 
 
 
 
130
  */
131
  function TerminalImageDisplay({
132
  base64,
 
141
  const renderedRef = useRef(false)
142
 
143
  useEffect(() => {
144
+ if (renderedRef.current || !writeRaw) return
145
  renderedRef.current = true
146
 
147
+ // Kitty protocol: timg fallback is rendered in renderToolResultMessage
148
+ // via RawAnsi (in-band with Ink's virtual DOM), so Ink knows the image
149
+ // dimensions and cursor position stays correct.
150
+ // Only use Kitty when it can reach the terminal natively.
151
  const protocol = detectImageProtocol()
152
+ if (protocol === 'kitty' && !isInsideTmux() && base64 && format) {
153
+ const buf = Buffer.from(base64, 'base64')
 
 
 
154
  const sequence = encodeKittyImage(buf, format)
155
  if (sequence) {
156
+ writeRaw(sequence + '\n')
157
  logForDebugging('ImageShow: displayed via Kitty protocol')
158
  }
 
159
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
160
  }, [base64, format, writeRaw])
161
 
162
  return <Text dimColor>{message}</Text>
 
248
  message: string
249
  base64?: string
250
  format?: string
251
+ timgOutput?: string
252
  },
253
  _progressMessages,
254
  _options,
 
256
  if (!content.success) {
257
  return null
258
  }
259
+
260
+ // timg output rendered in-band via RawAnsi so Ink knows the image
261
+ // dimensions and its virtual cursor stays in sync with the terminal.
262
+ if (content.timgOutput) {
263
+ // Strip cursor hide/show sequences that timg adds
264
+ const cleaned = content.timgOutput.replace(/\x1b\[\?25[hl]/g, '')
265
+ const lines = cleaned.split('\n').filter(l => l.length > 0)
266
+ if (lines.length === 0) {
267
+ return <Text dimColor>{content.message}</Text>
268
+ }
269
+ // Measure visible width (strip ANSI escape codes)
270
+ const ansiStrip = (s: string) => s.replace(/\x1b\[[0-9;?]*[a-zA-Z]/g, '')
271
+ const width = Math.max(...lines.map(l => ansiStrip(l).length))
272
+ return <RawAnsi lines={lines} width={width} />
273
+ }
274
+
275
  return (
276
  <TerminalImageDisplay
277
  base64={content.base64}
 
288
  imageData?: { base64: string; mediaType: string }
289
  base64?: string
290
  format?: string
291
+ timgOutput?: string
292
  }
293
  }> {
294
  const url = input.url
 
320
 
321
  logForDebugging(`ImageShow: loaded ${buffer.length} byte ${format} image`)
322
 
323
+ // Generate timg Unicode-block rendering. The output is passed through
324
+ // to the Ink virtual DOM via RawAnsi in renderToolResultMessage, so
325
+ // Ink knows the image dimensions and its cursor stays in sync.
326
+ // Kitty protocol is separately handled in TerminalImageDisplay via
327
+ // writeRaw (only when the terminal natively supports it outside tmux).
328
+ const protocol = detectImageProtocol()
329
+ let timgOutput: string | undefined
330
+ if (isInsideTmux() || protocol !== 'kitty') {
331
+ timgOutput = renderImageWithTimgSync(buffer, format) ?? undefined
332
+ if (timgOutput) {
333
+ logForDebugging('ImageShow: displayed via timg Unicode blocks (in-band with Ink)')
334
+ }
335
+ }
336
+
337
  return {
338
  data: {
339
  success: true,
 
344
  },
345
  base64: buffer.toString('base64'),
346
  format,
347
+ timgOutput,
348
  },
349
  }
350
  },
src/utils/terminalImage.ts CHANGED
@@ -1,3 +1,4 @@
 
1
  import { mkdtempSync, rmSync, writeFileSync } from 'fs'
2
  import { tmpdir } from 'os'
3
  import { join } from 'path'
@@ -5,7 +6,7 @@ import { Buffer } from 'buffer'
5
  import { env } from './env.js'
6
  import { execFileNoThrow } from './execFileNoThrow.js'
7
  import { wrapForMultiplexer } from '../ink/termio/osc.js'
8
- import { which } from './which.js'
9
 
10
  export type ImageProtocol = 'kitty' | null
11
 
@@ -203,3 +204,70 @@ export async function renderImageWithTimg(
203
  return null
204
  }
205
  }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { execFileSync } from 'child_process'
2
  import { mkdtempSync, rmSync, writeFileSync } from 'fs'
3
  import { tmpdir } from 'os'
4
  import { join } from 'path'
 
6
  import { env } from './env.js'
7
  import { execFileNoThrow } from './execFileNoThrow.js'
8
  import { wrapForMultiplexer } from '../ink/termio/osc.js'
9
+ import { which, whichSync } from './which.js'
10
 
11
  export type ImageProtocol = 'kitty' | null
12
 
 
204
  return null
205
  }
206
  }
207
+
208
+ /**
209
+ * Synchronous version of renderImageWithTimg. Blocks the event loop while
210
+ * spawning timg, which prevents Ink from rendering between the tool call
211
+ * and the image output — eliminating cursor-position races.
212
+ *
213
+ * @see renderImageWithTimg
214
+ */
215
+ export function renderImageWithTimgSync(
216
+ buffer: Buffer,
217
+ format: string,
218
+ columns?: number,
219
+ rows?: number,
220
+ ): string | null {
221
+ try {
222
+ const timgPath = whichSync('timg')
223
+ if (!timgPath) return null
224
+
225
+ const cols = columns ?? process.stdout.columns ?? 80
226
+ const termRows = rows ?? process.stdout.rows ?? 40
227
+ const maxRows = Math.max(10, Math.floor(termRows * 0.5))
228
+ const tmpDir = mkdtempSync(join(tmpdir(), 'versperclaw-timg-'))
229
+ const ext = format === 'jpeg' ? 'jpg' : format
230
+ const tmpFile = join(tmpDir, `image.${ext}`)
231
+ let result: string | null = null
232
+
233
+ try {
234
+ writeFileSync(tmpFile, buffer)
235
+
236
+ // Try quarter blocks first (4 pixels per cell, better quality)
237
+ try {
238
+ const stdout = execFileSync(
239
+ timgPath,
240
+ ['-p', 'q', '-g', `${cols}x${maxRows}`, tmpFile],
241
+ { encoding: 'utf8', timeout: 15000, maxBuffer: 10 * 1024 * 1024 },
242
+ )
243
+ if (stdout) result = stdout
244
+ } catch {
245
+ // fall through to half blocks
246
+ }
247
+
248
+ // Fallback to half blocks (2 pixels per cell, max compatibility)
249
+ if (!result) {
250
+ try {
251
+ const stdout = execFileSync(
252
+ timgPath,
253
+ ['-p', 'h', '-g', `${cols}x${maxRows}`, tmpFile],
254
+ { encoding: 'utf8', timeout: 15000, maxBuffer: 10 * 1024 * 1024 },
255
+ )
256
+ if (stdout) result = stdout
257
+ } catch {
258
+ // ignored
259
+ }
260
+ }
261
+ } finally {
262
+ try {
263
+ rmSync(tmpDir, { recursive: true, force: true })
264
+ } catch {
265
+ // ignore cleanup errors
266
+ }
267
+ }
268
+
269
+ return result
270
+ } catch {
271
+ return null
272
+ }
273
+ }