feat: render image directly to terminal fd, keep empty placeholder for TUI layout
Browse filesReplace ink-picture's Image (ImageBox + Kitty placement via Ink) with a
DirectImageDisplay that writes Kitty protocol directly to process.stdout.fd,
completely bypassing Ink's rendering loop. The Inku TUI renders an empty Box
placeholder for correct layout while the image is drawn on the terminal
framebuffer.
Follows the proven approach from commit 3f3bed3 (old timg): decouple image
rendering from TUI layout so Ink's frame writes never overwrite the picture.
Co-Authored-By: Claude Big Pickle <noreply@anthropic.com>
src/tools/ImageShowTool/DirectImageDisplay.tsx
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import fs from "node:fs";
|
| 2 |
+
import React, { useCallback, useEffect, useRef, useState } from "react";
|
| 3 |
+
import { Box } from "../../ink.js";
|
| 4 |
+
import type { DOMElement } from "../../ink.js";
|
| 5 |
+
import { useOnRender } from "../../ink-picture/InkPictureProvider.js";
|
| 6 |
+
import { cursorForward } from "../../ink-picture/utils/ansiEscapes.js";
|
| 7 |
+
import { fetchImage, getPngBuffer } from "../../ink-picture/utils/image.js";
|
| 8 |
+
import generateKittyId from "../../ink-picture/utils/generateKittyId.js";
|
| 9 |
+
import {
|
| 10 |
+
makeKittyDeletion,
|
| 11 |
+
makeKittyPlacement,
|
| 12 |
+
makeKittyTransmitChunks,
|
| 13 |
+
} from "../../ink-picture/renderers/kitty.js";
|
| 14 |
+
import usePosition from "../../ink-picture/hooks/usePosition.js";
|
| 15 |
+
|
| 16 |
+
/**
|
| 17 |
+
* Direct terminal image display that bypasses Ink's rendering loop.
|
| 18 |
+
*
|
| 19 |
+
* Follows the approach from commit f6a6fdc (old timg):
|
| 20 |
+
* - Writes Kitty protocol directly to process.stdout.fd (not through Ink's stdout)
|
| 21 |
+
* - useEffect fires AFTER Ink's frame write (post-commit), so first placement
|
| 22 |
+
* runs after Ink's initial output
|
| 23 |
+
* - useOnRender + setTimeout(0) handles re-placement on subsequent renders
|
| 24 |
+
* (useOnRender fires DURING commit, before Ink writes; setTimeout defers
|
| 25 |
+
* placement until after)
|
| 26 |
+
* - Periodic re-placement via setInterval combats Ink overwrites from other
|
| 27 |
+
* re-renders (where our subtree may not re-render)
|
| 28 |
+
* - Renders empty Box placeholder in Ink for correct TUI layout
|
| 29 |
+
*/
|
| 30 |
+
export function DirectImageDisplay({
|
| 31 |
+
src,
|
| 32 |
+
width,
|
| 33 |
+
height,
|
| 34 |
+
pixelWidth,
|
| 35 |
+
pixelHeight,
|
| 36 |
+
}: {
|
| 37 |
+
src: string;
|
| 38 |
+
width: number;
|
| 39 |
+
height: number;
|
| 40 |
+
pixelWidth: number;
|
| 41 |
+
pixelHeight: number;
|
| 42 |
+
}) {
|
| 43 |
+
const containerRef = useRef<DOMElement | null>(null);
|
| 44 |
+
const position = usePosition(containerRef);
|
| 45 |
+
|
| 46 |
+
const [imageData, setImageData] = useState<Buffer | undefined>(undefined);
|
| 47 |
+
|
| 48 |
+
// ββ Load image via Jimp ββ
|
| 49 |
+
useEffect(() => {
|
| 50 |
+
let cancelled = false;
|
| 51 |
+
(async () => {
|
| 52 |
+
try {
|
| 53 |
+
const image = await fetchImage(src);
|
| 54 |
+
if (!image || cancelled) return;
|
| 55 |
+
image.cover({ w: pixelWidth, h: pixelHeight });
|
| 56 |
+
const png = await getPngBuffer(image);
|
| 57 |
+
if (!cancelled) {
|
| 58 |
+
setImageData(png.data);
|
| 59 |
+
}
|
| 60 |
+
} catch {
|
| 61 |
+
// Image load failed β nothing to display
|
| 62 |
+
}
|
| 63 |
+
})();
|
| 64 |
+
return () => {
|
| 65 |
+
cancelled = true;
|
| 66 |
+
};
|
| 67 |
+
}, [src, pixelWidth, pixelHeight]);
|
| 68 |
+
|
| 69 |
+
// ββ Refs for values used in placement callback (stable identity) ββ
|
| 70 |
+
const imageIdRef = useRef<number | undefined>(undefined);
|
| 71 |
+
const positionRef = useRef(position);
|
| 72 |
+
positionRef.current = position;
|
| 73 |
+
const dimsRef = useRef({ w: width, h: height });
|
| 74 |
+
dimsRef.current = { w: width, h: height };
|
| 75 |
+
|
| 76 |
+
// ββ Transmit image data to terminal GPU memory ββ
|
| 77 |
+
// Uses fs.writeSync to bypass Ink's stdout buffering.
|
| 78 |
+
useEffect(() => {
|
| 79 |
+
if (!imageData) return;
|
| 80 |
+
|
| 81 |
+
const id = generateKittyId();
|
| 82 |
+
const base64Data = imageData.toString("base64");
|
| 83 |
+
const chunks = makeKittyTransmitChunks(id, base64Data);
|
| 84 |
+
const fd = process.stdout.fd;
|
| 85 |
+
for (const chunk of chunks) {
|
| 86 |
+
fs.writeSync(fd, chunk);
|
| 87 |
+
}
|
| 88 |
+
imageIdRef.current = id;
|
| 89 |
+
}, [imageData]);
|
| 90 |
+
|
| 91 |
+
// ββ Place image with position-aware cursor movement ββ
|
| 92 |
+
const placeImage = useCallback(() => {
|
| 93 |
+
const id = imageIdRef.current;
|
| 94 |
+
if (!id) return;
|
| 95 |
+
|
| 96 |
+
const pos = positionRef.current;
|
| 97 |
+
if (!pos) return;
|
| 98 |
+
|
| 99 |
+
const { w, h } = dimsRef.current;
|
| 100 |
+
if (h <= 0) return;
|
| 101 |
+
|
| 102 |
+
// Calculate cursor-up distance (same logic as cursorUp() in ansiEscapes.ts)
|
| 103 |
+
// Note: terminalHeight is in character rows (process.stdout.rows),
|
| 104 |
+
// NOT terminalInfo.terminalHeight which is in pixels.
|
| 105 |
+
const terminalHeight = process.stdout.rows;
|
| 106 |
+
const cursorUpCount = pos.appHeight - pos.row;
|
| 107 |
+
const movementCount =
|
| 108 |
+
pos.appHeight >= terminalHeight ? cursorUpCount - 1 : cursorUpCount;
|
| 109 |
+
|
| 110 |
+
const fd = process.stdout.fd;
|
| 111 |
+
const parts: Buffer[] = [
|
| 112 |
+
Buffer.from(`\x1b7`), // save cursor (DECSC)
|
| 113 |
+
];
|
| 114 |
+
if (movementCount > 0) {
|
| 115 |
+
parts.push(Buffer.from(`\x1b[${movementCount}A`)); // cursor up to image row
|
| 116 |
+
}
|
| 117 |
+
parts.push(
|
| 118 |
+
Buffer.from(`\r`), // carriage return to col 0
|
| 119 |
+
Buffer.from(cursorForward(pos.col)), // forward to image column
|
| 120 |
+
Buffer.from(makeKittyPlacement(id, 1, w, h)),
|
| 121 |
+
Buffer.from(`\x1b8`), // restore cursor (DECRC)
|
| 122 |
+
);
|
| 123 |
+
fs.writeSync(fd, Buffer.concat(parts));
|
| 124 |
+
}, []);
|
| 125 |
+
|
| 126 |
+
// ββ First placement: after Ink writes its initial frame ββ
|
| 127 |
+
// useEffect fires AFTER React's commit phase (which includes Ink's terminal
|
| 128 |
+
// output). So calling placeImage() directly here runs after Ink's frame write.
|
| 129 |
+
useEffect(() => {
|
| 130 |
+
if (!imageIdRef.current) return;
|
| 131 |
+
if (!position) return;
|
| 132 |
+
|
| 133 |
+
placeImage();
|
| 134 |
+
}, [imageData, position, placeImage]);
|
| 135 |
+
|
| 136 |
+
// ββ Re-place after each React render ββ
|
| 137 |
+
// useOnRender fires DURING the commit phase (via Profiler), BEFORE Ink writes
|
| 138 |
+
// its frame to the terminal. So we use setTimeout(0) to defer the placement
|
| 139 |
+
// until after Ink's synchronous frame write completes.
|
| 140 |
+
useOnRender(() => {
|
| 141 |
+
const id = imageIdRef.current;
|
| 142 |
+
if (!id) return;
|
| 143 |
+
|
| 144 |
+
setTimeout(() => {
|
| 145 |
+
placeImage();
|
| 146 |
+
}, 0);
|
| 147 |
+
});
|
| 148 |
+
|
| 149 |
+
// ββ Periodic re-placement to combat Ink overwrites from OTHER re-renders ββ
|
| 150 |
+
// When other parts of Codev re-render (e.g. new message arrives), Ink rewrites
|
| 151 |
+
// the entire frame. Our subtree might not re-render, so useOnRender won't fire.
|
| 152 |
+
// This interval ensures the image stays visible.
|
| 153 |
+
useEffect(() => {
|
| 154 |
+
const interval = setInterval(() => {
|
| 155 |
+
placeImage();
|
| 156 |
+
}, 500);
|
| 157 |
+
|
| 158 |
+
return () => clearInterval(interval);
|
| 159 |
+
}, [placeImage]);
|
| 160 |
+
|
| 161 |
+
// ββ Cleanup: delete image from terminal GPU memory ββ
|
| 162 |
+
useEffect(() => {
|
| 163 |
+
return () => {
|
| 164 |
+
const id = imageIdRef.current;
|
| 165 |
+
if (id) {
|
| 166 |
+
const fd = process.stdout.fd;
|
| 167 |
+
fs.writeSync(fd, makeKittyDeletion(id));
|
| 168 |
+
}
|
| 169 |
+
};
|
| 170 |
+
}, []);
|
| 171 |
+
|
| 172 |
+
// ββ Empty placeholder for correct TUI layout ββ
|
| 173 |
+
// Renders invisible Box with correct height so Ink reserves the right
|
| 174 |
+
// number of character cells. The actual image is drawn via Kitty protocol.
|
| 175 |
+
return <Box ref={containerRef} height={height} flexDirection="column" />;
|
| 176 |
+
}
|
src/tools/ImageShowTool/UI.tsx
CHANGED
|
@@ -1,19 +1,16 @@
|
|
| 1 |
import React, { useMemo } from 'react'
|
| 2 |
-
import Image, { InkPictureProvider } from '../../ink-picture/index.js'
|
| 3 |
import { Box, Text } from '../../ink.js'
|
| 4 |
import { MessageResponse } from '../../components/MessageResponse.js'
|
|
|
|
| 5 |
import type { ImageShowOutput } from './ImageShowTool.js'
|
| 6 |
import { detectTerminalCaps } from './detectTerminal.js'
|
|
|
|
| 7 |
|
| 8 |
// ββ Image display component ββ
|
| 9 |
-
//
|
| 10 |
-
//
|
| 11 |
-
//
|
| 12 |
-
//
|
| 13 |
-
// after each Ink screen refresh.
|
| 14 |
-
//
|
| 15 |
-
// Terminal detection uses environment variables (TERM, TERM_PROGRAM, etc.)
|
| 16 |
-
// instead of ANSI escape queries, avoiding stdin conflicts with Ink's TUI.
|
| 17 |
|
| 18 |
export function ImageDisplay({ src, width, height, pixelWidth, pixelHeight }: {
|
| 19 |
src: string
|
|
@@ -24,16 +21,25 @@ export function ImageDisplay({ src, width, height, pixelWidth, pixelHeight }: {
|
|
| 24 |
}) {
|
| 25 |
const terminalInfo = useMemo(() => detectTerminalCaps(), [])
|
| 26 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 27 |
return (
|
| 28 |
<Box flexDirection="column">
|
| 29 |
<InkPictureProvider terminalInfo={terminalInfo}>
|
| 30 |
-
<
|
| 31 |
src={src}
|
| 32 |
width={width}
|
| 33 |
height={height}
|
| 34 |
pixelWidth={pixelWidth}
|
| 35 |
pixelHeight={pixelHeight}
|
| 36 |
-
alt={typeof src === 'string' ? src : 'image'}
|
| 37 |
/>
|
| 38 |
</InkPictureProvider>
|
| 39 |
</Box>
|
|
|
|
| 1 |
import React, { useMemo } from 'react'
|
|
|
|
| 2 |
import { Box, Text } from '../../ink.js'
|
| 3 |
import { MessageResponse } from '../../components/MessageResponse.js'
|
| 4 |
+
import { InkPictureProvider } from '../../ink-picture/InkPictureProvider.js'
|
| 5 |
import type { ImageShowOutput } from './ImageShowTool.js'
|
| 6 |
import { detectTerminalCaps } from './detectTerminal.js'
|
| 7 |
+
import { DirectImageDisplay } from './DirectImageDisplay.js'
|
| 8 |
|
| 9 |
// ββ Image display component ββ
|
| 10 |
+
// Follows the old timg approach (commit f6a6fdc):
|
| 11 |
+
// - Writes Kitty protocol directly to the terminal fd (bypassing Ink)
|
| 12 |
+
// - Renders empty placeholder in Ink for correct TUI layout
|
| 13 |
+
// - The image is drawn on the terminal framebuffer without affecting layout
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
export function ImageDisplay({ src, width, height, pixelWidth, pixelHeight }: {
|
| 16 |
src: string
|
|
|
|
| 21 |
}) {
|
| 22 |
const terminalInfo = useMemo(() => detectTerminalCaps(), [])
|
| 23 |
|
| 24 |
+
const supportsKittyGraphics = terminalInfo.supportsKittyGraphics === true
|
| 25 |
+
|
| 26 |
+
if (!supportsKittyGraphics) {
|
| 27 |
+
return (
|
| 28 |
+
<MessageResponse height={1}>
|
| 29 |
+
<Text dimColor>Image: {src}</Text>
|
| 30 |
+
</MessageResponse>
|
| 31 |
+
)
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
return (
|
| 35 |
<Box flexDirection="column">
|
| 36 |
<InkPictureProvider terminalInfo={terminalInfo}>
|
| 37 |
+
<DirectImageDisplay
|
| 38 |
src={src}
|
| 39 |
width={width}
|
| 40 |
height={height}
|
| 41 |
pixelWidth={pixelWidth}
|
| 42 |
pixelHeight={pixelHeight}
|
|
|
|
| 43 |
/>
|
| 44 |
</InkPictureProvider>
|
| 45 |
</Box>
|