chenbhao Claude Big Pickle commited on
Commit
f03a328
·
1 Parent(s): 3fa929f

feat: add inline terminal image rendering with Kitty protocol support

Browse files

Introduce InlineImage component, ImageShow tool, and image caching/detection
utilities for rendering images directly in the terminal via the Kitty graphics
protocol. Falls back to clickable hyperlinks when protocol is unavailable.

Co-Authored-By: Claude Big Pickle <noreply@anthropic.com>

src/components/InlineImage.tsx ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import React, { useContext, useEffect, useRef, useState } from 'react'
2
+ import Link from '../ink/components/Link.js'
3
+ import { supportsHyperlinks } from '../ink/supports-hyperlinks.js'
4
+ import { Box, Text } from '../ink.js'
5
+ import { TerminalWriteContext } from '../ink/useTerminalNotification.js'
6
+ import { downloadImage, type CachedImage } from '../utils/imageUrlCache.js'
7
+ import {
8
+ detectImageProtocol,
9
+ encodeImageForTerminal,
10
+ isInsideTmux,
11
+ type ImageProtocol,
12
+ } from '../utils/terminalImage.js'
13
+
14
+ type Props = {
15
+ url: string
16
+ alt?: string
17
+ }
18
+
19
+ type Status = 'loading' | 'ready' | 'error'
20
+
21
+ /**
22
+ * Extract a display filename from a URL (for fallback text).
23
+ */
24
+ function filenameFromUrl(url: string): string {
25
+ try {
26
+ const pathname = new URL(url).pathname
27
+ const parts = pathname.split('/')
28
+ const last = parts[parts.length - 1]
29
+ return last || 'image'
30
+ } catch {
31
+ return 'image'
32
+ }
33
+ }
34
+
35
+ /**
36
+ * Renders an inline image in the terminal using the Kitty graphics protocol
37
+ * when available. Falls back to a clickable hyperlink for terminals without
38
+ * image protocol support.
39
+ *
40
+ * Behavior:
41
+ * - loading: dimmed "[Loading image...]" text + tmux passthrough notice
42
+ * - ready + kitty protocol: writes escape sequence via Ink's writeRaw,
43
+ * shows clickable [Image] link
44
+ * - ready + no protocol: shows clickable link with the URL
45
+ * - error: shows clickable link with plain text fallback
46
+ */
47
+ export function InlineImage({ url, alt }: Props) {
48
+ const writeRaw = useContext(TerminalWriteContext)
49
+ const [status, setStatus] = useState<Status>('loading')
50
+ const [imageData, setImageData] = useState<CachedImage | null>(null)
51
+ const [protocol, setProtocol] = useState<ImageProtocol>(() =>
52
+ detectImageProtocol(),
53
+ )
54
+ const protocolRef = useRef(protocol)
55
+ protocolRef.current = protocol
56
+ const imageWrittenRef = useRef(false)
57
+
58
+ const insideTmux = isInsideTmux()
59
+
60
+ // Download image on mount
61
+ useEffect(() => {
62
+ let cancelled = false
63
+
64
+ setStatus('loading')
65
+ setImageData(null)
66
+ setProtocol(detectImageProtocol())
67
+ imageWrittenRef.current = false
68
+ console.error(`[InlineImage] Starting download: ${url}`)
69
+
70
+ downloadImage(url)
71
+ .then(result => {
72
+ if (cancelled) return
73
+ console.error(`[InlineImage] Download result for ${url}:`, result ? `OK (${result.buffer.length} bytes, ${result.format})` : 'FAILED')
74
+ if (result) {
75
+ setImageData(result)
76
+ setStatus('ready')
77
+ } else {
78
+ setStatus('error')
79
+ }
80
+ })
81
+ .catch(err => {
82
+ if (!cancelled) {
83
+ console.error(`[InlineImage] Download error for ${url}:`, err)
84
+ setStatus('error')
85
+ }
86
+ })
87
+
88
+ return () => {
89
+ cancelled = true
90
+ }
91
+ }, [url])
92
+
93
+ // Write terminal image escape sequence via Ink's writeRaw (not raw stdout)
94
+ // so it stays synchronous with Ink's output and avoids cursor-position races.
95
+ useEffect(() => {
96
+ if (status !== 'ready' || !imageData || !protocol || imageWrittenRef.current) return
97
+
98
+ const sequence = encodeImageForTerminal(imageData.buffer, imageData.format)
99
+ if (sequence && writeRaw) {
100
+ console.error(`[InlineImage] Writing ${imageData.buffer.length} byte ${imageData.format} image via writeRaw`)
101
+ writeRaw(sequence)
102
+ imageWrittenRef.current = true
103
+ } else if (!writeRaw) {
104
+ console.error(`[InlineImage] writeRaw not available!`)
105
+ } else {
106
+ console.error(`[InlineImage] No sequence generated`)
107
+ }
108
+ }, [status, imageData, protocol, writeRaw])
109
+
110
+ const displayName = alt || filenameFromUrl(url)
111
+
112
+ // -- Render states --
113
+
114
+ // Loading: dimmed placeholder text
115
+ if (status === 'loading') {
116
+ if (insideTmux) {
117
+ return (
118
+ <Text dimColor>
119
+ {`[Loading ${displayName}...] (tmux may need allow-passthrough on)`}
120
+ </Text>
121
+ )
122
+ }
123
+ return <Text dimColor>{`[Loading ${displayName}...]`}</Text>
124
+ }
125
+
126
+ // Error: clickable link or plain text
127
+ if (status === 'error') {
128
+ if (supportsHyperlinks()) {
129
+ return (
130
+ <Link url={url}>
131
+ <Text dimColor>{`[${displayName}]`}</Text>
132
+ </Link>
133
+ )
134
+ }
135
+ return <Text dimColor>{`[${displayName}: ${url}]`}</Text>
136
+ }
137
+
138
+ // Ready: if we have a protocol and wrote the sequence, render a small label.
139
+ // The image was already written to the terminal via writeRaw above.
140
+ const link = supportsHyperlinks() ? (
141
+ <Link url={url}>
142
+ <Text dimColor>{`[${displayName}]`}</Text>
143
+ </Link>
144
+ ) : (
145
+ <Text dimColor>{`[${displayName}: ${url}]`}</Text>
146
+ )
147
+
148
+ return (
149
+ <Box flexDirection="column">
150
+ {link}
151
+ {insideTmux && protocol && (
152
+ <Text dimColor>{` ⚠ tmux: add \`set -g allow-passthrough on\` to ~/.tmux.conf`}</Text>
153
+ )}
154
+ </Box>
155
+ )
156
+ }
src/components/Markdown.tsx CHANGED
@@ -7,6 +7,7 @@ import { type CliHighlight, getCliHighlightPromise } from '../utils/cliHighlight
7
  import { hashContent } from '../utils/hash.js';
8
  import { configureMarked, formatToken } from '../utils/markdown.js';
9
  import { stripPromptXMLTags } from '../utils/messages.js';
 
10
  import { MarkdownTable } from './MarkdownTable.js';
11
  type Props = {
12
  children: string;
@@ -144,9 +145,12 @@ function MarkdownBody(t0) {
144
  if (token.type === "table") {
145
  flushNonTableContent();
146
  elements.push(<MarkdownTable key={elements.length} token={token as Tokens.Table} highlight={highlight} />);
 
 
 
 
147
  } else {
148
  nonTableContent = nonTableContent + formatToken(token, theme, 0, null, null, highlight);
149
- nonTableContent;
150
  }
151
  }
152
  flushNonTableContent();
 
7
  import { hashContent } from '../utils/hash.js';
8
  import { configureMarked, formatToken } from '../utils/markdown.js';
9
  import { stripPromptXMLTags } from '../utils/messages.js';
10
+ import { InlineImage } from './InlineImage.js';
11
  import { MarkdownTable } from './MarkdownTable.js';
12
  type Props = {
13
  children: string;
 
145
  if (token.type === "table") {
146
  flushNonTableContent();
147
  elements.push(<MarkdownTable key={elements.length} token={token as Tokens.Table} highlight={highlight} />);
148
+ } else if (token.type === "image") {
149
+ flushNonTableContent();
150
+ console.error(`[MarkdownBody] Found image token: href=${token.href} text=${token.text}`)
151
+ elements.push(<InlineImage key={elements.length} url={token.href} alt={token.text} />);
152
  } else {
153
  nonTableContent = nonTableContent + formatToken(token, theme, 0, null, null, highlight);
 
154
  }
155
  }
156
  flushNonTableContent();
src/tools.ts CHANGED
@@ -80,6 +80,7 @@ import { ReadMcpResourceTool } from './tools/ReadMcpResourceTool/ReadMcpResource
80
  import { ToolSearchTool } from './tools/ToolSearchTool/ToolSearchTool.js'
81
  import { DebugSessionTool } from './tools/DebugSessionTool.js'
82
  import { FriendEmotionTool } from './tools/FriendEmotionTool.js'
 
83
  import { LocationTool } from './tools/LocationTool/LocationTool.js'
84
  // Friend ScreenObserve removed — voice + emotion only
85
  import { EnterPlanModeTool } from './tools/EnterPlanModeTool/EnterPlanModeTool.js'
@@ -253,6 +254,8 @@ export function getAllBaseTools(): Tools {
253
  DebugSessionTool,
254
  // Friend VRM desktop pet tools — enabled when the plugin is active
255
  FriendEmotionTool,
 
 
256
  // Location & mapping tool — uses Amap (China) or Google Maps (international)
257
  LocationTool,
258
  ListMcpResourcesTool,
 
80
  import { ToolSearchTool } from './tools/ToolSearchTool/ToolSearchTool.js'
81
  import { DebugSessionTool } from './tools/DebugSessionTool.js'
82
  import { FriendEmotionTool } from './tools/FriendEmotionTool.js'
83
+ import { ImageShowTool } from './tools/ImageShowTool/ImageShowTool.js'
84
  import { LocationTool } from './tools/LocationTool/LocationTool.js'
85
  // Friend ScreenObserve removed — voice + emotion only
86
  import { EnterPlanModeTool } from './tools/EnterPlanModeTool/EnterPlanModeTool.js'
 
254
  DebugSessionTool,
255
  // Friend VRM desktop pet tools — enabled when the plugin is active
256
  FriendEmotionTool,
257
+ // ImageShow — display images in terminal via Kitty graphics protocol
258
+ ImageShowTool,
259
  // Location & mapping tool — uses Amap (China) or Google Maps (international)
260
  LocationTool,
261
  ListMcpResourcesTool,
src/tools/ImageShowTool/ImageShowTool.tsx ADDED
@@ -0,0 +1,340 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { readFile } from 'fs/promises'
2
+ import { homedir } from 'os'
3
+ import React, { useContext, useEffect, useRef } from 'react'
4
+ import { z } from 'zod/v4'
5
+ import { Text } from '../../ink.js'
6
+ import { TerminalWriteContext } from '../../ink/useTerminalNotification.js'
7
+ import { buildTool, type ToolDef } from '../../Tool.js'
8
+ import { logForDebugging } from '../../utils/debug.js'
9
+ import {
10
+ detectImageProtocol,
11
+ encodeKittyImage,
12
+ isInsideTmux,
13
+ renderImageWithTimg,
14
+ } from '../../utils/terminalImage.js'
15
+
16
+ const IMAGE_TOOL_NAME = 'ImageShow'
17
+
18
+ const inputSchema = () =>
19
+ z.strictObject({
20
+ url: z
21
+ .string()
22
+ .describe(
23
+ 'Image URL (https://...) or local file path (e.g. /tmp/image.png). ' +
24
+ 'Supported formats: PNG, JPEG, GIF, WebP.',
25
+ ),
26
+ alt: z
27
+ .string()
28
+ .optional()
29
+ .describe('Alt text shown as link when image cannot be displayed.'),
30
+ })
31
+
32
+ type Input = z.infer<ReturnType<typeof inputSchema>>
33
+
34
+ /** Detect format from URL/file extension */
35
+ function detectFormat(url: string): string {
36
+ const clean = url.split('?')[0]!.split('#')[0]!
37
+ const ext = clean.split('.').pop()?.toLowerCase() ?? ''
38
+ if (['png', 'jpg', 'jpeg', 'gif', 'webp'].includes(ext)) {
39
+ return ext === 'jpg' ? 'jpeg' : ext
40
+ }
41
+ return 'png'
42
+ }
43
+
44
+ /** Read a local file and return its buffer + detected format */
45
+ async function readLocalImage(url: string): Promise<{ buffer: Buffer; format: string } | null> {
46
+ try {
47
+ const format = detectFormat(url)
48
+ const buffer = await readFile(url)
49
+ return { buffer, format }
50
+ } catch {
51
+ return null
52
+ }
53
+ }
54
+
55
+ /** Download an image from URL and return its buffer + detected format */
56
+ async function fetchImage(url: string): Promise<{ buffer: Buffer; format: string } | null> {
57
+ try {
58
+ const response = await fetch(url, {
59
+ headers: {
60
+ 'User-Agent': 'Mozilla/5.0 (compatible; VersperClaw/1.0)',
61
+ },
62
+ redirect: 'follow',
63
+ })
64
+
65
+ if (!response.ok) {
66
+ logForDebugging(`ImageShow: HTTP ${response.status} for ${url}`)
67
+ return null
68
+ }
69
+
70
+ const contentType = response.headers.get('content-type') ?? ''
71
+ const format = detectFormat(url !== contentType ? url : contentType)
72
+
73
+ const reader = response.body?.getReader()
74
+ if (!reader) return null
75
+
76
+ const chunks: Uint8Array[] = []
77
+ let totalSize = 0
78
+ const MAX_SIZE = 10_000_000
79
+
80
+ while (true) {
81
+ const { done, value } = await reader.read()
82
+ if (done) break
83
+ totalSize += value.byteLength
84
+ if (totalSize > MAX_SIZE) {
85
+ logForDebugging(`ImageShow: image too large (${totalSize} bytes) for ${url}`)
86
+ reader.cancel()
87
+ return null
88
+ }
89
+ chunks.push(value)
90
+ }
91
+
92
+ const combinedLength = chunks.reduce((acc, c) => acc + c.byteLength, 0)
93
+ const combined = new Uint8Array(combinedLength)
94
+ let offset = 0
95
+ for (const chunk of chunks) {
96
+ combined.set(chunk, offset)
97
+ offset += chunk.byteLength
98
+ }
99
+
100
+ return { buffer: Buffer.from(combined.buffer), format }
101
+ } catch (err) {
102
+ logForDebugging(`ImageShow: fetch error ${err} for ${url}`)
103
+ return null
104
+ }
105
+ }
106
+
107
+ /** Load image: local file or remote URL */
108
+ async function loadImage(url: string): Promise<{ buffer: Buffer; format: string } | null> {
109
+ // Expand ~ to home directory
110
+ const normalizedUrl = url.startsWith('~')
111
+ ? url.replace(/^~(?=$|\/)/, homedir())
112
+ : url
113
+ if (normalizedUrl.startsWith('file://') || normalizedUrl.startsWith('/') || normalizedUrl.startsWith('.')) {
114
+ const path = normalizedUrl.startsWith('file://') ? normalizedUrl.slice(7) : normalizedUrl
115
+ return readLocalImage(path)
116
+ }
117
+ return fetchImage(normalizedUrl)
118
+ }
119
+
120
+ function getToolUseSummary(input: Partial<Input>): string | null {
121
+ return input?.url ? `Show: ${input.url.split('/').pop() ?? input.url}` : null
122
+ }
123
+
124
+ /**
125
+ * React component that displays an image in the terminal using the best
126
+ * available method:
127
+ * 1. Kitty graphics protocol – when the terminal supports it AND we're
128
+ * NOT inside tmux (where Kitty passthrough may be blocked).
129
+ * 2. `timg` Unicode blocks – reliable fallback that works in virtually
130
+ * every modern terminal (Unicode + 24-bit color required).
131
+ * 3. Plain text message – when neither method is available.
132
+ *
133
+ * The image is rendered via Ink's writeRaw so the escape sequences stay
134
+ * synchronized with Ink's render cycle and avoid cursor-position races.
135
+ */
136
+ function TerminalImageDisplay({
137
+ base64,
138
+ format,
139
+ message,
140
+ }: {
141
+ base64?: string
142
+ format?: string
143
+ message: string
144
+ }): React.ReactNode {
145
+ const writeRaw = useContext(TerminalWriteContext)
146
+ const renderedRef = useRef(false)
147
+
148
+ useEffect(() => {
149
+ if (renderedRef.current || !base64 || !format || !writeRaw) return
150
+ renderedRef.current = true
151
+
152
+ const protocol = detectImageProtocol()
153
+ const buf = Buffer.from(base64, 'base64')
154
+
155
+ // Path A: Kitty protocol – best quality, works natively in Kitty,
156
+ // Ghostty, WezTerm, Konsole, foot outside tmux.
157
+ if (protocol === 'kitty' && !isInsideTmux()) {
158
+ const sequence = encodeKittyImage(buf, format)
159
+ if (sequence) {
160
+ writeRaw(sequence)
161
+ logForDebugging('ImageShow: displayed via Kitty protocol')
162
+ }
163
+ return
164
+ }
165
+
166
+ // Path B: timg Unicode-block rendering – reliable in all modern
167
+ // terminals including inside tmux where Kitty passthrough may be blocked.
168
+ renderImageWithTimg(buf, format)
169
+ .then(output => {
170
+ if (output) {
171
+ writeRaw(output)
172
+ logForDebugging('ImageShow: displayed via timg Unicode blocks')
173
+ } else {
174
+ logForDebugging('ImageShow: timg not available, showing text only')
175
+ }
176
+ })
177
+ .catch(() => {
178
+ logForDebugging('ImageShow: timg render error, showing text only')
179
+ })
180
+ }, [base64, format, writeRaw])
181
+
182
+ return <Text dimColor>{message}</Text>
183
+ }
184
+
185
+ export const ImageShowTool = buildTool({
186
+ name: IMAGE_TOOL_NAME,
187
+ description:
188
+ 'Display an image (PNG/JPEG/GIF/WebP) directly in the terminal. ' +
189
+ 'Uses the Kitty graphics protocol when available outside tmux, ' +
190
+ 'or falls back to timg Unicode-block rendering for universal compatibility. ' +
191
+ 'Supports both URLs (https://) and local file paths. Images are shown inline above the tool result.',
192
+
193
+ getToolUseSummary,
194
+ getActivityDescription(input) {
195
+ return input?.url ? `Showing image: ${input.url}` : 'Showing image'
196
+ },
197
+
198
+ isEnabled() {
199
+ return true
200
+ },
201
+
202
+ get inputSchema() {
203
+ return inputSchema()
204
+ },
205
+
206
+ async validateInput(input) {
207
+ if (!input?.url) {
208
+ return { result: false, message: 'Missing url', errorCode: 1 }
209
+ }
210
+ return { result: true }
211
+ },
212
+
213
+ async prompt(_options): Promise<string> {
214
+ return `ImageShow displays a PNG/JPEG/GIF/WebP image directly in the terminal. Uses the Kitty graphics protocol when available (outside tmux), or falls back to timg Unicode-block rendering for universal terminal compatibility. Supports local file paths (e.g. /tmp/image.png) and HTTPS URLs (e.g. https://example.com/image.png). The image is shown inline above the tool result.`
215
+ },
216
+
217
+ async checkPermissions(): Promise<{ behavior: 'allow' }> {
218
+ return { behavior: 'allow' }
219
+ },
220
+
221
+ isReadOnly() {
222
+ return true
223
+ },
224
+
225
+ isConcurrencySafe() {
226
+ return true
227
+ },
228
+
229
+ mapToolResultToToolResultBlockParam(
230
+ content: {
231
+ success: boolean
232
+ message: string
233
+ imageData?: { base64: string; mediaType: string }
234
+ },
235
+ toolUseID: string,
236
+ ) {
237
+ if (content.success && content.imageData) {
238
+ return {
239
+ tool_use_id: toolUseID,
240
+ type: 'tool_result' as const,
241
+ content: [
242
+ {
243
+ type: 'image' as const,
244
+ source: {
245
+ type: 'base64' as const,
246
+ data: content.imageData.base64,
247
+ media_type: content.imageData.mediaType as 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp',
248
+ },
249
+ },
250
+ ],
251
+ }
252
+ }
253
+ return {
254
+ tool_use_id: toolUseID,
255
+ type: 'tool_result' as const,
256
+ content: [
257
+ {
258
+ type: 'text' as const,
259
+ text: content.message,
260
+ },
261
+ ],
262
+ }
263
+ },
264
+
265
+ renderToolResultMessage(
266
+ content: {
267
+ success: boolean
268
+ message: string
269
+ base64?: string
270
+ format?: string
271
+ },
272
+ _progressMessages,
273
+ _options,
274
+ ): React.ReactNode {
275
+ if (!content.success) {
276
+ return null
277
+ }
278
+ return (
279
+ <TerminalImageDisplay
280
+ base64={content.base64}
281
+ format={content.format}
282
+ message={content.message}
283
+ />
284
+ )
285
+ },
286
+
287
+ async call(input: Input): Promise<{
288
+ data: {
289
+ success: boolean
290
+ message: string
291
+ imageData?: { base64: string; mediaType: string }
292
+ base64?: string
293
+ format?: string
294
+ }
295
+ }> {
296
+ const url = input.url
297
+ const alt = input.alt ?? url.split('/').pop() ?? 'image'
298
+
299
+ logForDebugging(`ImageShow: loading ${url}`)
300
+
301
+ const result = await loadImage(url)
302
+
303
+ if (!result) {
304
+ return {
305
+ data: {
306
+ success: false,
307
+ message: `Failed to load image from: ${url}`,
308
+ },
309
+ }
310
+ }
311
+
312
+ const { buffer, format } = result
313
+
314
+ const mediaType =
315
+ format === 'jpeg'
316
+ ? 'image/jpeg'
317
+ : format === 'gif'
318
+ ? 'image/gif'
319
+ : format === 'webp'
320
+ ? 'image/webp'
321
+ : 'image/png'
322
+
323
+ logForDebugging(`ImageShow: loaded ${buffer.length} byte ${format} image`)
324
+
325
+ // Return data with base64 and format so renderToolResultMessage can
326
+ // pass them to KittyImageDisplay for writeRaw-based terminal output.
327
+ return {
328
+ data: {
329
+ success: true,
330
+ message: `Displayed: ${alt} (${buffer.length} bytes, ${format})`,
331
+ imageData: {
332
+ base64: buffer.toString('base64'),
333
+ mediaType,
334
+ },
335
+ base64: buffer.toString('base64'),
336
+ format,
337
+ },
338
+ }
339
+ },
340
+ }) satisfies ToolDef<any, any>
src/utils/imageUrlCache.ts ADDED
@@ -0,0 +1,256 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { createHash } from 'crypto'
2
+ import { mkdir, open, rm } from 'fs/promises'
3
+ import { join } from 'path'
4
+ import { getClaudeConfigHomeDir } from './envUtils.js'
5
+ import { logForDebugging } from './debug.js'
6
+
7
+ const IMAGE_CACHE_DIR = 'image-cache'
8
+ const URL_CACHE_SUBDIR = 'url-cache'
9
+ const MAX_URL_CACHED_IMAGES = 100
10
+ const MAX_DOWNLOAD_SIZE_BYTES = 10_000_000 // 10 MB limit
11
+
12
+ export interface CachedImage {
13
+ buffer: Buffer
14
+ format: string
15
+ mediaType: string
16
+ path: string
17
+ }
18
+
19
+ // In-memory cache to avoid repeated downloads
20
+ const urlImageCache = new Map<string, CachedImage>()
21
+
22
+ /**
23
+ * Get the cache directory for URL-downloaded images.
24
+ */
25
+ function getUrlCacheDir(): string {
26
+ return join(getClaudeConfigHomeDir(), IMAGE_CACHE_DIR, URL_CACHE_SUBDIR)
27
+ }
28
+
29
+ /**
30
+ * Ensure the URL cache directory exists.
31
+ */
32
+ async function ensureUrlCacheDir(): Promise<void> {
33
+ await mkdir(getUrlCacheDir(), { recursive: true })
34
+ }
35
+
36
+ /**
37
+ * Compute a hash of a URL for use as a cache filename.
38
+ */
39
+ function hashUrl(url: string): string {
40
+ return createHash('sha256').update(url).digest('hex').slice(0, 16)
41
+ }
42
+
43
+ /**
44
+ * Determine image format from Content-Type header or URL extension.
45
+ */
46
+ function detectFormat(url: string, contentType: string): string {
47
+ // Prefer Content-Type header
48
+ if (contentType) {
49
+ const mime = contentType.split('/')[1]?.toLowerCase()
50
+ if (mime && ['png', 'jpeg', 'jpg', 'gif', 'webp'].includes(mime)) {
51
+ return mime === 'jpg' ? 'jpeg' : mime
52
+ }
53
+ }
54
+
55
+ // Fall back to URL extension
56
+ const cleanUrl = url.split('?')[0]!.split('#')[0]!
57
+ const ext = cleanUrl.split('.').pop()?.toLowerCase() ?? ''
58
+ if (['png', 'jpg', 'jpeg', 'gif', 'webp'].includes(ext)) {
59
+ return ext === 'jpg' ? 'jpeg' : ext
60
+ }
61
+
62
+ return 'png' // default
63
+ }
64
+
65
+ /**
66
+ * Determine media type from format string.
67
+ */
68
+ function formatToMediaType(format: string): string {
69
+ return `image/${format === 'jpeg' ? 'jpeg' : format}`
70
+ }
71
+
72
+ /**
73
+ * Download an image from a URL, cache it to disk, and return the buffer,
74
+ * format, and local path.
75
+ *
76
+ * Skips download if the image is already in the in-memory or disk cache.
77
+ * Limits download size to MAX_DOWNLOAD_SIZE_BYTES.
78
+ *
79
+ * @param url - The image URL to download
80
+ * @returns CachedImage or null if download fails
81
+ */
82
+ export async function downloadImage(url: string): Promise<CachedImage | null> {
83
+ // Check in-memory cache first
84
+ const cached = urlImageCache.get(url)
85
+ if (cached) {
86
+ return cached
87
+ }
88
+
89
+ // Check disk cache
90
+ try {
91
+ const cacheDir = getUrlCacheDir()
92
+ const hash = hashUrl(url)
93
+ const files = await readdirSafe(cacheDir)
94
+ for (const file of files) {
95
+ if (file.startsWith(hash)) {
96
+ // Found cached file
97
+ const parts = file.split('.')
98
+ const format = parts.length > 1 ? parts[parts.length - 1]! : 'png'
99
+ const fh = await open(join(cacheDir, file), 'r')
100
+ try {
101
+ const buffer = await fh.readFile()
102
+ const result: CachedImage = {
103
+ buffer,
104
+ format,
105
+ mediaType: formatToMediaType(format),
106
+ path: join(cacheDir, file),
107
+ }
108
+ urlImageCache.set(url, result)
109
+ logForDebugging(`Image URL cache hit: ${url}`)
110
+ return result
111
+ } finally {
112
+ await fh.close()
113
+ }
114
+ }
115
+ }
116
+ } catch {
117
+ // Disk cache miss or directory doesn't exist — proceed to download
118
+ }
119
+
120
+ // Download the image
121
+ try {
122
+ logForDebugging(`Downloading image: ${url}`)
123
+ const response = await fetch(url, {
124
+ headers: {
125
+ // Some image hosts block requests without a browser-like User-Agent
126
+ 'User-Agent':
127
+ 'Mozilla/5.0 (compatible; VersperClaw/1.0; +https://versperai.dev)',
128
+ },
129
+ // Follow up to 5 redirects (e.g. Wikimedia CDN redirects)
130
+ redirect: 'follow',
131
+ })
132
+
133
+ if (!response.ok) {
134
+ // Surface the error so users can see it during testing
135
+ console.error(`[InlineImage] Download failed (HTTP ${response.status}): ${url}`)
136
+ logForDebugging(`Image download failed (${response.status}): ${url}`)
137
+ return null
138
+ }
139
+
140
+ // Check content-length if available
141
+ const contentLength = response.headers.get('content-length')
142
+ if (contentLength && parseInt(contentLength, 10) > MAX_DOWNLOAD_SIZE_BYTES) {
143
+ logForDebugging(`Image too large (${contentLength} bytes): ${url}`)
144
+ return null
145
+ }
146
+
147
+ const contentType = response.headers.get('content-type') ?? ''
148
+ const format = detectFormat(url, contentType)
149
+
150
+ // Stream the response with a size limit
151
+ const reader = response.body?.getReader()
152
+ if (!reader) {
153
+ return null
154
+ }
155
+
156
+ const chunks: Uint8Array[] = []
157
+ let totalSize = 0
158
+
159
+ while (true) {
160
+ const { done, value } = await reader.read()
161
+ if (done) break
162
+ totalSize += value.byteLength
163
+ if (totalSize > MAX_DOWNLOAD_SIZE_BYTES) {
164
+ logForDebugging(`Image download exceeded size limit: ${url}`)
165
+ reader.cancel()
166
+ return null
167
+ }
168
+ chunks.push(value)
169
+ }
170
+
171
+ // Combine chunks
172
+ const combinedLength = chunks.reduce((acc, c) => acc + c.byteLength, 0)
173
+ const combined = new Uint8Array(combinedLength)
174
+ let offset = 0
175
+ for (const chunk of chunks) {
176
+ combined.set(chunk, offset)
177
+ offset += chunk.byteLength
178
+ }
179
+
180
+ const buffer = Buffer.from(combined.buffer)
181
+
182
+ // Cache to disk
183
+ try {
184
+ await ensureUrlCacheDir()
185
+ const ext = format
186
+ const cachePath = join(getUrlCacheDir(), `${hashUrl(url)}.${ext}`)
187
+ const fh = await open(cachePath, 'w', 0o600)
188
+ try {
189
+ await fh.writeFile(buffer)
190
+ await fh.datasync()
191
+ } finally {
192
+ await fh.close()
193
+ }
194
+
195
+ const result: CachedImage = {
196
+ buffer,
197
+ format,
198
+ mediaType: formatToMediaType(format),
199
+ path: cachePath,
200
+ }
201
+
202
+ // Store in memory cache
203
+ evictOldestIfAtCap()
204
+ urlImageCache.set(url, result)
205
+
206
+ logForDebugging(`Cached image: ${url} -> ${cachePath}`)
207
+ return result
208
+ } catch (cacheError) {
209
+ // Cache write failed, still return the buffer
210
+ logForDebugging(`Failed to cache image to disk: ${cacheError}`)
211
+ return {
212
+ buffer,
213
+ format,
214
+ mediaType: formatToMediaType(format),
215
+ path: '',
216
+ }
217
+ }
218
+ } catch (error) {
219
+ logForDebugging(`Image download error: ${error}`)
220
+ return null
221
+ }
222
+ }
223
+
224
+ /**
225
+ * Read directory contents safely (returns empty array on error).
226
+ */
227
+ async function readdirSafe(dir: string): Promise<string[]> {
228
+ try {
229
+ const { readdir } = await import('fs/promises')
230
+ const entries = await readdir(dir, { withFileTypes: false })
231
+ return entries as string[]
232
+ } catch {
233
+ return []
234
+ }
235
+ }
236
+
237
+ /**
238
+ * Evict oldest in-memory cache entry if at capacity.
239
+ */
240
+ function evictOldestIfAtCap(): void {
241
+ while (urlImageCache.size >= MAX_URL_CACHED_IMAGES) {
242
+ const oldest = urlImageCache.keys().next().value
243
+ if (oldest !== undefined) {
244
+ urlImageCache.delete(oldest)
245
+ } else {
246
+ break
247
+ }
248
+ }
249
+ }
250
+
251
+ /**
252
+ * Clear the in-memory URL image cache.
253
+ */
254
+ export function clearUrlImageCache(): void {
255
+ urlImageCache.clear()
256
+ }
src/utils/markdown.ts CHANGED
@@ -137,6 +137,8 @@ export function formatToken(
137
  case 'hr':
138
  return '---'
139
  case 'image':
 
 
140
  return token.href
141
  case 'link': {
142
  // Prevent mailto links from being displayed as clickable links
@@ -286,6 +288,14 @@ export function formatToken(
286
  // only) so hostnames like docs.github.io/guide#42 don't false-positive. Repo
287
  // segment allows dots (e.g. cc.kurs.web). Lookbehind is avoided — it defeats
288
  // YARR JIT in JSC.
 
 
 
 
 
 
 
 
289
  const ISSUE_REF_PATTERN =
290
  /(^|[^\w./-])([A-Za-z0-9][\w-]*\/[A-Za-z0-9][\w.-]*)#(\d+)\b/g
291
 
 
137
  case 'hr':
138
  return '---'
139
  case 'image':
140
+ // Fallback: Markdown.tsx intercepts image tokens to render <InlineImage>,
141
+ // but if formatToken is called directly (e.g. from applyMarkdown), show URL.
142
  return token.href
143
  case 'link': {
144
  // Prevent mailto links from being displayed as clickable links
 
288
  // only) so hostnames like docs.github.io/guide#42 don't false-positive. Repo
289
  // segment allows dots (e.g. cc.kurs.web). Lookbehind is avoided — it defeats
290
  // YARR JIT in JSC.
291
+ /**
292
+ * Regex for detecting bare image URLs in text.
293
+ * Matches URLs ending in common image extensions.
294
+ * Used by Markdown.tsx to split text tokens containing image URLs.
295
+ */
296
+ export const IMAGE_URL_RE =
297
+ /https?:\/\/[^\s()<>]+?\.(?:png|jpg|jpeg|gif|webp)(?:[?#][^\s()<>]*?)?/gi
298
+
299
  const ISSUE_REF_PATTERN =
300
  /(^|[^\w./-])([A-Za-z0-9][\w-]*\/[A-Za-z0-9][\w.-]*)#(\d+)\b/g
301
 
src/utils/terminalImage.ts ADDED
@@ -0,0 +1,205 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import { mkdtempSync, rmSync, writeFileSync } from 'fs'
2
+ import { tmpdir } from 'os'
3
+ import { join } from 'path'
4
+ import { Buffer } from 'buffer'
5
+ import { env } from './env.js'
6
+ import { execFileNoThrow } from './execFileNoThrow.js'
7
+ import { wrapForMultiplexer } from '../ink/termio/osc.js'
8
+ import { which } from './which.js'
9
+
10
+ export type ImageProtocol = 'kitty' | null
11
+
12
+ /** Is the terminal inside tmux? */
13
+ export function isInsideTmux(): boolean {
14
+ return !!process.env.TMUX
15
+ }
16
+
17
+ const KITTY_IMAGE_CHUNK_SIZE = 4096
18
+
19
+ // Image format codes for Kitty protocol
20
+ const KITTY_FORMAT_CODES: Record<string, number> = {
21
+ png: 100,
22
+ jpeg: 101,
23
+ jpg: 101,
24
+ gif: 102,
25
+ webp: 103,
26
+ }
27
+
28
+ /**
29
+ * Detect whether the current terminal supports inline image display via
30
+ * the Kitty graphics protocol. Kitty, WezTerm, Konsole, foot, and
31
+ * Ghostty all support it.
32
+ */
33
+ export function supportsImageProtocol(): boolean {
34
+ return detectImageProtocol() !== null
35
+ }
36
+
37
+ /**
38
+ * Detect the best available terminal image protocol.
39
+ * Currently only supports the Kitty protocol.
40
+ *
41
+ * Returns `'kitty'` or `null` if no protocol is available.
42
+ */
43
+ export function detectImageProtocol(): ImageProtocol {
44
+ const term = env.terminal ?? ''
45
+
46
+ // Explicit TERM check for Kitty (TERM=xterm-kitty or TERM contains kitty)
47
+ if (process.env.TERM?.includes('kitty')) return 'kitty'
48
+ if (process.env.KITTY_WINDOW_ID) return 'kitty'
49
+
50
+ // TERM_PROGRAM-based detection
51
+ switch (term) {
52
+ case 'kitty':
53
+ case 'WezTerm':
54
+ case 'konsole':
55
+ case 'ghostty':
56
+ case 'foot':
57
+ return 'kitty'
58
+ }
59
+
60
+ return null
61
+ }
62
+
63
+ /**
64
+ * Get the Kitty protocol format code for a given image format string.
65
+ * Defaults to PNG (100) for unknown formats.
66
+ */
67
+ function getKittyFormatCode(format: string): number {
68
+ return KITTY_FORMAT_CODES[format.toLowerCase()] ?? 100
69
+ }
70
+
71
+ /**
72
+ * Encode an image buffer into Kitty graphics protocol escape sequences.
73
+ *
74
+ * Large images are split into chunks of KITTY_IMAGE_CHUNK_SIZE bytes
75
+ * to avoid overflowing terminal input buffers.
76
+ *
77
+ * @param buffer - Raw image data
78
+ * @param format - Image format (png, jpeg, gif, webp)
79
+ * @returns Kitty protocol escape sequence
80
+ */
81
+ export function encodeKittyImage(buffer: Buffer, format: string): string {
82
+ const f = getKittyFormatCode(format)
83
+ const b64 = buffer.toString('base64')
84
+ const parts: string[] = []
85
+
86
+ if (b64.length <= KITTY_IMAGE_CHUNK_SIZE) {
87
+ // Single chunk — no splitting needed
88
+ parts.push(`\x1b_Ga=d,f=${f},m=0;${b64}\x1b\\`)
89
+ } else {
90
+ // Split into multiple chunks
91
+ let offset = 0
92
+ while (offset < b64.length) {
93
+ const chunk = b64.slice(offset, offset + KITTY_IMAGE_CHUNK_SIZE)
94
+ const isLast = offset + KITTY_IMAGE_CHUNK_SIZE >= b64.length
95
+ parts.push(`\x1b_Ga=d,f=${f},m=${isLast ? 0 : 1};${chunk}\x1b\\`)
96
+ offset += KITTY_IMAGE_CHUNK_SIZE
97
+ }
98
+ }
99
+
100
+ const sequence = parts.join('')
101
+
102
+ // Wrap for tmux/screen multiplexer passthrough if needed
103
+ return wrapForMultiplexer(sequence)
104
+ }
105
+
106
+ /**
107
+ * Generate a terminal image display sequence for an image buffer.
108
+ * Picks the best protocol for the current terminal.
109
+ *
110
+ * @param buffer - Raw image data
111
+ * @param format - Image format (png, jpeg, gif, webp)
112
+ * @returns The escape sequence to write to stdout, or null if no protocol available
113
+ */
114
+ export function encodeImageForTerminal(
115
+ buffer: Buffer,
116
+ format: string,
117
+ ): string | null {
118
+ const protocol = detectImageProtocol()
119
+ if (protocol === 'kitty') {
120
+ return encodeKittyImage(buffer, format)
121
+ }
122
+ return null
123
+ }
124
+
125
+ /**
126
+ * Image protocol capabilities summary (for debugging/logging).
127
+ */
128
+ export function getImageProtocolSummary(): string {
129
+ const protocol = detectImageProtocol()
130
+ if (protocol === 'kitty') return 'kitty'
131
+ return 'none'
132
+ }
133
+
134
+ /**
135
+ * Render an image using the `timg` utility with Unicode block characters.
136
+ * Works in any terminal that supports Unicode and 24-bit color (virtually all
137
+ * modern terminals). Falls back gracefully if timg is not installed.
138
+ *
139
+ * The image is rendered using quarter-block characters ("pixelation q") for
140
+ * the best quality-to-compatibility ratio. If timg fails, half-blocks are
141
+ * tried as a fallback.
142
+ *
143
+ * @param buffer - The raw image buffer (decoded)
144
+ * @param format - Image format (png, jpeg, gif, webp)
145
+ * @param columns - Optional terminal width in character columns (auto-detected)
146
+ * @param rows - Optional terminal height in character rows (auto-detected)
147
+ * @returns ANSI escape sequence string for rendering, or null on failure
148
+ */
149
+ export async function renderImageWithTimg(
150
+ buffer: Buffer,
151
+ format: string,
152
+ columns?: number,
153
+ rows?: number,
154
+ ): Promise<string | null> {
155
+ try {
156
+ const timgPath = await which('timg')
157
+ if (!timgPath) return null
158
+
159
+ const cols = columns ?? process.stdout.columns ?? 80
160
+ const termRows = rows ?? process.stdout.rows ?? 40
161
+ const tmpDir = mkdtempSync(join(tmpdir(), 'versperclaw-timg-'))
162
+ const ext = format === 'jpeg' ? 'jpg' : format
163
+ const tmpFile = join(tmpDir, `image.${ext}`)
164
+ let result: string | null = null
165
+
166
+ // Use at most half the terminal height so the image doesn't dominate
167
+ const maxRows = Math.max(10, Math.floor(termRows * 0.5))
168
+
169
+ try {
170
+ writeFileSync(tmpFile, buffer)
171
+
172
+ // Try quarter blocks first (4 pixels per cell, better quality)
173
+ const quarter = await execFileNoThrow(
174
+ timgPath,
175
+ ['-p', 'q', '-g', `${cols}x${maxRows}`, tmpFile],
176
+ { timeout: 15000, preserveOutputOnError: true },
177
+ )
178
+ if (quarter.code === 0 && quarter.stdout) {
179
+ result = quarter.stdout
180
+ }
181
+
182
+ // Fallback to half blocks (2 pixels per cell, max compatibility)
183
+ if (!result) {
184
+ const half = await execFileNoThrow(
185
+ timgPath,
186
+ ['-p', 'h', '-g', `${cols}x${maxRows}`, tmpFile],
187
+ { timeout: 15000, preserveOutputOnError: true },
188
+ )
189
+ if (half.code === 0 && half.stdout) {
190
+ result = half.stdout
191
+ }
192
+ }
193
+ } finally {
194
+ try {
195
+ rmSync(tmpDir, { recursive: true, force: true })
196
+ } catch {
197
+ // ignore cleanup errors
198
+ }
199
+ }
200
+
201
+ return result
202
+ } catch {
203
+ return null
204
+ }
205
+ }