chenbhao Claude Big Pickle commited on
Commit
43b76b7
Β·
1 Parent(s): f0588e7

feat: render image directly to terminal fd, keep empty placeholder for TUI layout

Browse files

Replace ink-picture's Image (ImageBox + Kitty placement via Ink) with a
DirectImageDisplay that writes Kitty protocol directly to process.stdout.fd,
completely bypassing Ink's rendering loop. The Inku TUI renders an empty Box
placeholder for correct layout while the image is drawn on the terminal
framebuffer.

Follows the proven approach from commit 3f3bed3 (old timg): decouple image
rendering from TUI layout so Ink's frame writes never overwrite the picture.

Co-Authored-By: Claude Big Pickle <noreply@anthropic.com>

src/tools/ImageShowTool/DirectImageDisplay.tsx ADDED
@@ -0,0 +1,176 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import fs from "node:fs";
2
+ import React, { useCallback, useEffect, useRef, useState } from "react";
3
+ import { Box } from "../../ink.js";
4
+ import type { DOMElement } from "../../ink.js";
5
+ import { useOnRender } from "../../ink-picture/InkPictureProvider.js";
6
+ import { cursorForward } from "../../ink-picture/utils/ansiEscapes.js";
7
+ import { fetchImage, getPngBuffer } from "../../ink-picture/utils/image.js";
8
+ import generateKittyId from "../../ink-picture/utils/generateKittyId.js";
9
+ import {
10
+ makeKittyDeletion,
11
+ makeKittyPlacement,
12
+ makeKittyTransmitChunks,
13
+ } from "../../ink-picture/renderers/kitty.js";
14
+ import usePosition from "../../ink-picture/hooks/usePosition.js";
15
+
16
+ /**
17
+ * Direct terminal image display that bypasses Ink's rendering loop.
18
+ *
19
+ * Follows the approach from commit f6a6fdc (old timg):
20
+ * - Writes Kitty protocol directly to process.stdout.fd (not through Ink's stdout)
21
+ * - useEffect fires AFTER Ink's frame write (post-commit), so first placement
22
+ * runs after Ink's initial output
23
+ * - useOnRender + setTimeout(0) handles re-placement on subsequent renders
24
+ * (useOnRender fires DURING commit, before Ink writes; setTimeout defers
25
+ * placement until after)
26
+ * - Periodic re-placement via setInterval combats Ink overwrites from other
27
+ * re-renders (where our subtree may not re-render)
28
+ * - Renders empty Box placeholder in Ink for correct TUI layout
29
+ */
30
+ export function DirectImageDisplay({
31
+ src,
32
+ width,
33
+ height,
34
+ pixelWidth,
35
+ pixelHeight,
36
+ }: {
37
+ src: string;
38
+ width: number;
39
+ height: number;
40
+ pixelWidth: number;
41
+ pixelHeight: number;
42
+ }) {
43
+ const containerRef = useRef<DOMElement | null>(null);
44
+ const position = usePosition(containerRef);
45
+
46
+ const [imageData, setImageData] = useState<Buffer | undefined>(undefined);
47
+
48
+ // ── Load image via Jimp ──
49
+ useEffect(() => {
50
+ let cancelled = false;
51
+ (async () => {
52
+ try {
53
+ const image = await fetchImage(src);
54
+ if (!image || cancelled) return;
55
+ image.cover({ w: pixelWidth, h: pixelHeight });
56
+ const png = await getPngBuffer(image);
57
+ if (!cancelled) {
58
+ setImageData(png.data);
59
+ }
60
+ } catch {
61
+ // Image load failed β€” nothing to display
62
+ }
63
+ })();
64
+ return () => {
65
+ cancelled = true;
66
+ };
67
+ }, [src, pixelWidth, pixelHeight]);
68
+
69
+ // ── Refs for values used in placement callback (stable identity) ──
70
+ const imageIdRef = useRef<number | undefined>(undefined);
71
+ const positionRef = useRef(position);
72
+ positionRef.current = position;
73
+ const dimsRef = useRef({ w: width, h: height });
74
+ dimsRef.current = { w: width, h: height };
75
+
76
+ // ── Transmit image data to terminal GPU memory ──
77
+ // Uses fs.writeSync to bypass Ink's stdout buffering.
78
+ useEffect(() => {
79
+ if (!imageData) return;
80
+
81
+ const id = generateKittyId();
82
+ const base64Data = imageData.toString("base64");
83
+ const chunks = makeKittyTransmitChunks(id, base64Data);
84
+ const fd = process.stdout.fd;
85
+ for (const chunk of chunks) {
86
+ fs.writeSync(fd, chunk);
87
+ }
88
+ imageIdRef.current = id;
89
+ }, [imageData]);
90
+
91
+ // ── Place image with position-aware cursor movement ──
92
+ const placeImage = useCallback(() => {
93
+ const id = imageIdRef.current;
94
+ if (!id) return;
95
+
96
+ const pos = positionRef.current;
97
+ if (!pos) return;
98
+
99
+ const { w, h } = dimsRef.current;
100
+ if (h <= 0) return;
101
+
102
+ // Calculate cursor-up distance (same logic as cursorUp() in ansiEscapes.ts)
103
+ // Note: terminalHeight is in character rows (process.stdout.rows),
104
+ // NOT terminalInfo.terminalHeight which is in pixels.
105
+ const terminalHeight = process.stdout.rows;
106
+ const cursorUpCount = pos.appHeight - pos.row;
107
+ const movementCount =
108
+ pos.appHeight >= terminalHeight ? cursorUpCount - 1 : cursorUpCount;
109
+
110
+ const fd = process.stdout.fd;
111
+ const parts: Buffer[] = [
112
+ Buffer.from(`\x1b7`), // save cursor (DECSC)
113
+ ];
114
+ if (movementCount > 0) {
115
+ parts.push(Buffer.from(`\x1b[${movementCount}A`)); // cursor up to image row
116
+ }
117
+ parts.push(
118
+ Buffer.from(`\r`), // carriage return to col 0
119
+ Buffer.from(cursorForward(pos.col)), // forward to image column
120
+ Buffer.from(makeKittyPlacement(id, 1, w, h)),
121
+ Buffer.from(`\x1b8`), // restore cursor (DECRC)
122
+ );
123
+ fs.writeSync(fd, Buffer.concat(parts));
124
+ }, []);
125
+
126
+ // ── First placement: after Ink writes its initial frame ──
127
+ // useEffect fires AFTER React's commit phase (which includes Ink's terminal
128
+ // output). So calling placeImage() directly here runs after Ink's frame write.
129
+ useEffect(() => {
130
+ if (!imageIdRef.current) return;
131
+ if (!position) return;
132
+
133
+ placeImage();
134
+ }, [imageData, position, placeImage]);
135
+
136
+ // ── Re-place after each React render ──
137
+ // useOnRender fires DURING the commit phase (via Profiler), BEFORE Ink writes
138
+ // its frame to the terminal. So we use setTimeout(0) to defer the placement
139
+ // until after Ink's synchronous frame write completes.
140
+ useOnRender(() => {
141
+ const id = imageIdRef.current;
142
+ if (!id) return;
143
+
144
+ setTimeout(() => {
145
+ placeImage();
146
+ }, 0);
147
+ });
148
+
149
+ // ── Periodic re-placement to combat Ink overwrites from OTHER re-renders ──
150
+ // When other parts of Codev re-render (e.g. new message arrives), Ink rewrites
151
+ // the entire frame. Our subtree might not re-render, so useOnRender won't fire.
152
+ // This interval ensures the image stays visible.
153
+ useEffect(() => {
154
+ const interval = setInterval(() => {
155
+ placeImage();
156
+ }, 500);
157
+
158
+ return () => clearInterval(interval);
159
+ }, [placeImage]);
160
+
161
+ // ── Cleanup: delete image from terminal GPU memory ──
162
+ useEffect(() => {
163
+ return () => {
164
+ const id = imageIdRef.current;
165
+ if (id) {
166
+ const fd = process.stdout.fd;
167
+ fs.writeSync(fd, makeKittyDeletion(id));
168
+ }
169
+ };
170
+ }, []);
171
+
172
+ // ── Empty placeholder for correct TUI layout ──
173
+ // Renders invisible Box with correct height so Ink reserves the right
174
+ // number of character cells. The actual image is drawn via Kitty protocol.
175
+ return <Box ref={containerRef} height={height} flexDirection="column" />;
176
+ }
src/tools/ImageShowTool/UI.tsx CHANGED
@@ -1,19 +1,16 @@
1
  import React, { useMemo } from 'react'
2
- import Image, { InkPictureProvider } from '../../ink-picture/index.js'
3
  import { Box, Text } from '../../ink.js'
4
  import { MessageResponse } from '../../components/MessageResponse.js'
 
5
  import type { ImageShowOutput } from './ImageShowTool.js'
6
  import { detectTerminalCaps } from './detectTerminal.js'
 
7
 
8
  // ── Image display component ──
9
- // Renders a placeholder in Ink's TUI and uses Kitty/Sixel protocol to draw
10
- // the full-resolution image directly on the terminal framebuffer (bypassing
11
- // the Ink render cycle). The placeholder reserves character cells so the TUI
12
- // layout isn't broken; ink-picture's useDirectRenderer repositions the image
13
- // after each Ink screen refresh.
14
- //
15
- // Terminal detection uses environment variables (TERM, TERM_PROGRAM, etc.)
16
- // instead of ANSI escape queries, avoiding stdin conflicts with Ink's TUI.
17
 
18
  export function ImageDisplay({ src, width, height, pixelWidth, pixelHeight }: {
19
  src: string
@@ -24,16 +21,25 @@ export function ImageDisplay({ src, width, height, pixelWidth, pixelHeight }: {
24
  }) {
25
  const terminalInfo = useMemo(() => detectTerminalCaps(), [])
26
 
 
 
 
 
 
 
 
 
 
 
27
  return (
28
  <Box flexDirection="column">
29
  <InkPictureProvider terminalInfo={terminalInfo}>
30
- <Image
31
  src={src}
32
  width={width}
33
  height={height}
34
  pixelWidth={pixelWidth}
35
  pixelHeight={pixelHeight}
36
- alt={typeof src === 'string' ? src : 'image'}
37
  />
38
  </InkPictureProvider>
39
  </Box>
 
1
  import React, { useMemo } from 'react'
 
2
  import { Box, Text } from '../../ink.js'
3
  import { MessageResponse } from '../../components/MessageResponse.js'
4
+ import { InkPictureProvider } from '../../ink-picture/InkPictureProvider.js'
5
  import type { ImageShowOutput } from './ImageShowTool.js'
6
  import { detectTerminalCaps } from './detectTerminal.js'
7
+ import { DirectImageDisplay } from './DirectImageDisplay.js'
8
 
9
  // ── Image display component ──
10
+ // Follows the old timg approach (commit f6a6fdc):
11
+ // - Writes Kitty protocol directly to the terminal fd (bypassing Ink)
12
+ // - Renders empty placeholder in Ink for correct TUI layout
13
+ // - The image is drawn on the terminal framebuffer without affecting layout
 
 
 
 
14
 
15
  export function ImageDisplay({ src, width, height, pixelWidth, pixelHeight }: {
16
  src: string
 
21
  }) {
22
  const terminalInfo = useMemo(() => detectTerminalCaps(), [])
23
 
24
+ const supportsKittyGraphics = terminalInfo.supportsKittyGraphics === true
25
+
26
+ if (!supportsKittyGraphics) {
27
+ return (
28
+ <MessageResponse height={1}>
29
+ <Text dimColor>Image: {src}</Text>
30
+ </MessageResponse>
31
+ )
32
+ }
33
+
34
  return (
35
  <Box flexDirection="column">
36
  <InkPictureProvider terminalInfo={terminalInfo}>
37
+ <DirectImageDisplay
38
  src={src}
39
  width={width}
40
  height={height}
41
  pixelWidth={pixelWidth}
42
  pixelHeight={pixelHeight}
 
43
  />
44
  </InkPictureProvider>
45
  </Box>