Download src/memory/embedding-chunk-limits.ts from shimen/shinclaw: direct link, hf CLI and curl.
- Browser
- Download file 894 Bytes
-
https://huggingface.co/spaces/shimen/shinclaw/resolve/main/src/memory/embedding-chunk-limits.ts
- Command line
-
hf download hf://spaces/shimen/shinclaw/src/memory/embedding-chunk-limits.ts
-
curl -L -o embedding-chunk-limits.ts https://huggingface.co/spaces/shimen/shinclaw/resolve/main/src/memory/embedding-chunk-limits.ts
894 Bytes
| import type { EmbeddingProvider } from "./embeddings.js"; | |
| import { estimateUtf8Bytes, splitTextToUtf8ByteLimit } from "./embedding-input-limits.js"; | |
| import { resolveEmbeddingMaxInputTokens } from "./embedding-model-limits.js"; | |
| import { hashText, type MemoryChunk } from "./internal.js"; | |
| export function enforceEmbeddingMaxInputTokens( | |
| provider: EmbeddingProvider, | |
| chunks: MemoryChunk[], | |
| ): MemoryChunk[] { | |
| const maxInputTokens = resolveEmbeddingMaxInputTokens(provider); | |
| const out: MemoryChunk[] = []; | |
| for (const chunk of chunks) { | |
| if (estimateUtf8Bytes(chunk.text) <= maxInputTokens) { | |
| out.push(chunk); | |
| continue; | |
| } | |
| for (const text of splitTextToUtf8ByteLimit(chunk.text, maxInputTokens)) { | |
| out.push({ | |
| startLine: chunk.startLine, | |
| endLine: chunk.endLine, | |
| text, | |
| hash: hashText(text), | |
| }); | |
| } | |
| } | |
| return out; | |
| } | |