Spaces:
Running
Running
Sync from felladrin/MiniSearch@c75b7b9
Browse files- Dockerfile +11 -1
- client/modules/config.test.ts +50 -0
- client/modules/config.ts +26 -3
- client/modules/search.test.ts +83 -0
- client/modules/search.ts +72 -24
- docs/configuration.md +2 -5
- docs/page-content.md +18 -0
Dockerfile
CHANGED
|
@@ -79,7 +79,8 @@ RUN chmod 644 $SEARXNG_SETTINGS_PATH && \
|
|
| 79 |
/usr/local/searxng/searxng-venv/bin/pip install --no-build-isolation -e . && \
|
| 80 |
/usr/local/searxng/searxng-venv/bin/pip uninstall -y wheel setuptools && \
|
| 81 |
/usr/local/searxng/searxng-venv/bin/pip uninstall -y pip && \
|
| 82 |
-
/usr/local/searxng/searxng-venv/bin/python -c "import searx.webapp"
|
|
|
|
| 83 |
|
| 84 |
# The runtime never pip-installs, and pip itself vendors flagged copies of
|
| 85 |
# msgpack and setuptools (`pip/_vendor/vendor.txt`) that no released pip
|
|
@@ -89,6 +90,15 @@ RUN chmod 644 $SEARXNG_SETTINGS_PATH && \
|
|
| 89 |
# the `import searx.webapp` build check above proves the package imports
|
| 90 |
# after the removal. The editable install finder is plain importlib.
|
| 91 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
# Create the app directory while still root and hand it to the app user:
|
| 93 |
# the legacy (non-BuildKit) builder creates WORKDIR directories as root even
|
| 94 |
# under USER, which then breaks `npm ci`'s mkdir of node_modules.
|
|
|
|
| 79 |
/usr/local/searxng/searxng-venv/bin/pip install --no-build-isolation -e . && \
|
| 80 |
/usr/local/searxng/searxng-venv/bin/pip uninstall -y wheel setuptools && \
|
| 81 |
/usr/local/searxng/searxng-venv/bin/pip uninstall -y pip && \
|
| 82 |
+
/usr/local/searxng/searxng-venv/bin/python -c "import searx.webapp" && \
|
| 83 |
+
rm -f /tmp/sxng_cache_*
|
| 84 |
|
| 85 |
# The runtime never pip-installs, and pip itself vendors flagged copies of
|
| 86 |
# msgpack and setuptools (`pip/_vendor/vendor.txt`) that no released pip
|
|
|
|
| 90 |
# the `import searx.webapp` build check above proves the package imports
|
| 91 |
# after the removal. The editable install finder is plain importlib.
|
| 92 |
|
| 93 |
+
# That same import also creates SearXNG's SQLite caches in the temp directory,
|
| 94 |
+
# owned by root because the build runs as root. The container runs as `node`,
|
| 95 |
+
# and SearXNG wipes and rebuilds those caches whenever `server.secret_key`
|
| 96 |
+
# differs from the one written above, so shipping them makes an instance with
|
| 97 |
+
# its own key die at startup with `attempt to write a readonly database`, with
|
| 98 |
+
# no search and a container that still reports healthy (#2732). Clearing them
|
| 99 |
+
# lets the running user create its own on first start; the `-shm` and `-wal`
|
| 100 |
+
# sidecars go with them, or SQLite fails on those instead.
|
| 101 |
+
|
| 102 |
# Create the app directory while still root and hand it to the app user:
|
| 103 |
# the legacy (non-BuildKit) builder creates WORKDIR directories as root even
|
| 104 |
# under USER, which then breaks `npm ci`'s mkdir of node_modules.
|
client/modules/config.test.ts
CHANGED
|
@@ -64,6 +64,56 @@ describe("Config Module", () => {
|
|
| 64 |
await expect(getConfig()).rejects.toThrow("Network error");
|
| 65 |
});
|
| 66 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 67 |
it("retries after a failure instead of caching it", async () => {
|
| 68 |
const { getConfig } = await importConfigModule();
|
| 69 |
mockFetch.mockRejectedValueOnce(new Error("Network error"));
|
|
|
|
| 64 |
await expect(getConfig()).rejects.toThrow("Network error");
|
| 65 |
});
|
| 66 |
|
| 67 |
+
// The search token is per-process, so a restart leaves this page holding one
|
| 68 |
+
// the server never issued; invalidating is how a loaded page gets the new one.
|
| 69 |
+
it("refetches after the cache is invalidated", async () => {
|
| 70 |
+
const { getConfig, invalidateConfig } = await importConfigModule();
|
| 71 |
+
mockFetch.mockResolvedValue({
|
| 72 |
+
ok: true,
|
| 73 |
+
json: vi.fn().mockResolvedValue(serverConfig),
|
| 74 |
+
});
|
| 75 |
+
|
| 76 |
+
await getConfig();
|
| 77 |
+
|
| 78 |
+
const rotatedConfig = { ...serverConfig, searchToken: "b".repeat(64) };
|
| 79 |
+
mockFetch.mockResolvedValue({
|
| 80 |
+
ok: true,
|
| 81 |
+
json: vi.fn().mockResolvedValue(rotatedConfig),
|
| 82 |
+
});
|
| 83 |
+
|
| 84 |
+
invalidateConfig();
|
| 85 |
+
|
| 86 |
+
await expect(getConfig()).resolves.toEqual(rotatedConfig);
|
| 87 |
+
expect(mockFetch).toHaveBeenCalledTimes(2);
|
| 88 |
+
});
|
| 89 |
+
|
| 90 |
+
// Otherwise the discarded response lands in the cache a moment after the
|
| 91 |
+
// invalidation, and the page goes back to hashing the token it just dropped.
|
| 92 |
+
it("does not let a fetch started before the invalidation repopulate the cache", async () => {
|
| 93 |
+
const { getConfig, invalidateConfig } = await importConfigModule();
|
| 94 |
+
let resolveFirstFetch: (config: ServerConfig) => void = () => {};
|
| 95 |
+
mockFetch.mockReturnValueOnce(
|
| 96 |
+
new Promise((resolve) => {
|
| 97 |
+
resolveFirstFetch = (config) =>
|
| 98 |
+
resolve({ ok: true, json: vi.fn().mockResolvedValue(config) });
|
| 99 |
+
}),
|
| 100 |
+
);
|
| 101 |
+
|
| 102 |
+
const firstCall = getConfig();
|
| 103 |
+
invalidateConfig();
|
| 104 |
+
resolveFirstFetch(serverConfig);
|
| 105 |
+
await expect(firstCall).resolves.toEqual(serverConfig);
|
| 106 |
+
|
| 107 |
+
const rotatedConfig = { ...serverConfig, searchToken: "b".repeat(64) };
|
| 108 |
+
mockFetch.mockResolvedValue({
|
| 109 |
+
ok: true,
|
| 110 |
+
json: vi.fn().mockResolvedValue(rotatedConfig),
|
| 111 |
+
});
|
| 112 |
+
|
| 113 |
+
await expect(getConfig()).resolves.toEqual(rotatedConfig);
|
| 114 |
+
expect(mockFetch).toHaveBeenCalledTimes(2);
|
| 115 |
+
});
|
| 116 |
+
|
| 117 |
it("retries after a failure instead of caching it", async () => {
|
| 118 |
const { getConfig } = await importConfigModule();
|
| 119 |
mockFetch.mockRejectedValueOnce(new Error("Network error"));
|
client/modules/config.ts
CHANGED
|
@@ -38,6 +38,7 @@ const FETCH_TIMEOUT_MS = 5000;
|
|
| 38 |
|
| 39 |
let cachedConfig: ServerConfig | null = null;
|
| 40 |
let pendingFetch: Promise<ServerConfig> | null = null;
|
|
|
|
| 41 |
|
| 42 |
async function fetchConfig(): Promise<ServerConfig> {
|
| 43 |
const response = await fetch("/api/config", {
|
|
@@ -63,14 +64,17 @@ export async function getConfig(): Promise<ServerConfig> {
|
|
| 63 |
if (cachedConfig) return cachedConfig;
|
| 64 |
|
| 65 |
if (!pendingFetch) {
|
|
|
|
| 66 |
pendingFetch = fetchConfig().then(
|
| 67 |
(config) => {
|
| 68 |
-
|
| 69 |
-
|
|
|
|
|
|
|
| 70 |
return config;
|
| 71 |
},
|
| 72 |
(error) => {
|
| 73 |
-
pendingFetch = null;
|
| 74 |
throw error;
|
| 75 |
},
|
| 76 |
);
|
|
@@ -78,3 +82,22 @@ export async function getConfig(): Promise<ServerConfig> {
|
|
| 78 |
|
| 79 |
return pendingFetch;
|
| 80 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 38 |
|
| 39 |
let cachedConfig: ServerConfig | null = null;
|
| 40 |
let pendingFetch: Promise<ServerConfig> | null = null;
|
| 41 |
+
let configGeneration = 0;
|
| 42 |
|
| 43 |
async function fetchConfig(): Promise<ServerConfig> {
|
| 44 |
const response = await fetch("/api/config", {
|
|
|
|
| 64 |
if (cachedConfig) return cachedConfig;
|
| 65 |
|
| 66 |
if (!pendingFetch) {
|
| 67 |
+
const generation = configGeneration;
|
| 68 |
pendingFetch = fetchConfig().then(
|
| 69 |
(config) => {
|
| 70 |
+
if (generation === configGeneration) {
|
| 71 |
+
cachedConfig = config;
|
| 72 |
+
pendingFetch = null;
|
| 73 |
+
}
|
| 74 |
return config;
|
| 75 |
},
|
| 76 |
(error) => {
|
| 77 |
+
if (generation === configGeneration) pendingFetch = null;
|
| 78 |
throw error;
|
| 79 |
},
|
| 80 |
);
|
|
|
|
| 82 |
|
| 83 |
return pendingFetch;
|
| 84 |
}
|
| 85 |
+
|
| 86 |
+
/**
|
| 87 |
+
* Drops the cached config so the next `getConfig` call goes back to
|
| 88 |
+
* /api/config.
|
| 89 |
+
*
|
| 90 |
+
* Each server process mints its own search token, so a restart leaves an open
|
| 91 |
+
* page holding a token the server never issued. The 401 handler in
|
| 92 |
+
* `search.ts` calls this before asking for the token again, which is the only
|
| 93 |
+
* way a page that is already loaded can find out the token rotated.
|
| 94 |
+
*
|
| 95 |
+
* A fetch that is already in flight was started against the values this call
|
| 96 |
+
* discards, so it is left to resolve its own callers without repopulating the
|
| 97 |
+
* cache behind them.
|
| 98 |
+
*/
|
| 99 |
+
export function invalidateConfig(): void {
|
| 100 |
+
cachedConfig = null;
|
| 101 |
+
pendingFetch = null;
|
| 102 |
+
configGeneration += 1;
|
| 103 |
+
}
|
client/modules/search.test.ts
CHANGED
|
@@ -1,11 +1,17 @@
|
|
| 1 |
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
|
|
| 2 |
import { addLogEntry } from "./logEntries";
|
| 3 |
import * as searchModule from "./search";
|
|
|
|
| 4 |
|
| 5 |
vi.mock("./logEntries", () => ({
|
| 6 |
addLogEntry: vi.fn(),
|
| 7 |
}));
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
vi.mock("./searchTokenHash", () => ({
|
| 10 |
getSearchTokenHash: vi.fn().mockResolvedValue("mock-token-hash"),
|
| 11 |
}));
|
|
@@ -261,6 +267,83 @@ describe("Search Module", () => {
|
|
| 261 |
});
|
| 262 |
});
|
| 263 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 264 |
describe("Update Cache Config Function", () => {
|
| 265 |
it.each([
|
| 266 |
["ttl", 60000],
|
|
|
|
| 1 |
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
| 2 |
+
import { invalidateConfig } from "./config";
|
| 3 |
import { addLogEntry } from "./logEntries";
|
| 4 |
import * as searchModule from "./search";
|
| 5 |
+
import { getSearchTokenHash } from "./searchTokenHash";
|
| 6 |
|
| 7 |
vi.mock("./logEntries", () => ({
|
| 8 |
addLogEntry: vi.fn(),
|
| 9 |
}));
|
| 10 |
|
| 11 |
+
vi.mock("./config", () => ({
|
| 12 |
+
invalidateConfig: vi.fn(),
|
| 13 |
+
}));
|
| 14 |
+
|
| 15 |
vi.mock("./searchTokenHash", () => ({
|
| 16 |
getSearchTokenHash: vi.fn().mockResolvedValue("mock-token-hash"),
|
| 17 |
}));
|
|
|
|
| 267 |
});
|
| 268 |
});
|
| 269 |
|
| 270 |
+
// A restart, a second instance or a dev-server reload mints a new search
|
| 271 |
+
// token, and a tab opened before that keeps hashing the old one. These pin
|
| 272 |
+
// the recovery: the page takes the token the server is handing out now and
|
| 273 |
+
// tries again, once.
|
| 274 |
+
describe("Search Token Rotation", () => {
|
| 275 |
+
beforeEach(() => {
|
| 276 |
+
vi.clearAllMocks();
|
| 277 |
+
mockFetch.mockReset();
|
| 278 |
+
});
|
| 279 |
+
|
| 280 |
+
afterEach(() => {
|
| 281 |
+
// The token mock is module-level, so the queued one-shot values have to
|
| 282 |
+
// go back to the shared default for the describe blocks that follow.
|
| 283 |
+
vi.mocked(getSearchTokenHash).mockReset();
|
| 284 |
+
vi.mocked(getSearchTokenHash).mockResolvedValue("mock-token-hash");
|
| 285 |
+
});
|
| 286 |
+
|
| 287 |
+
it("refetches the config and retries once when the token is rejected", async () => {
|
| 288 |
+
const mockResults: string[][] = [
|
| 289 |
+
["Title", "Snippet", "https://example.com"],
|
| 290 |
+
];
|
| 291 |
+
vi.mocked(getSearchTokenHash)
|
| 292 |
+
.mockResolvedValueOnce("stale-token-hash")
|
| 293 |
+
.mockResolvedValueOnce("fresh-token-hash");
|
| 294 |
+
mockFetch
|
| 295 |
+
.mockResolvedValueOnce({ ok: false, status: 401 })
|
| 296 |
+
.mockResolvedValueOnce({
|
| 297 |
+
ok: true,
|
| 298 |
+
json: vi.fn().mockResolvedValue(mockResults),
|
| 299 |
+
});
|
| 300 |
+
|
| 301 |
+
await expect(
|
| 302 |
+
searchModule.searchServiceInstance.performSearch<string[][]>(
|
| 303 |
+
"text",
|
| 304 |
+
"test query",
|
| 305 |
+
),
|
| 306 |
+
).resolves.toEqual(mockResults);
|
| 307 |
+
|
| 308 |
+
expect(invalidateConfig).toHaveBeenCalledTimes(1);
|
| 309 |
+
expect(mockFetch).toHaveBeenCalledTimes(2);
|
| 310 |
+
expect(mockFetch.mock.calls[0][0] as string).toContain(
|
| 311 |
+
"token=stale-token-hash",
|
| 312 |
+
);
|
| 313 |
+
expect(mockFetch.mock.calls[1][0] as string).toContain(
|
| 314 |
+
"token=fresh-token-hash",
|
| 315 |
+
);
|
| 316 |
+
});
|
| 317 |
+
|
| 318 |
+
it("reports a second rejection instead of retrying again", async () => {
|
| 319 |
+
mockFetch.mockResolvedValue({ ok: false, status: 401 });
|
| 320 |
+
|
| 321 |
+
await expect(
|
| 322 |
+
searchModule.searchServiceInstance.performSearch<string[][]>(
|
| 323 |
+
"text",
|
| 324 |
+
"test query",
|
| 325 |
+
),
|
| 326 |
+
).rejects.toThrow("HTTP error! status: 401");
|
| 327 |
+
|
| 328 |
+
expect(mockFetch).toHaveBeenCalledTimes(2);
|
| 329 |
+
expect(invalidateConfig).toHaveBeenCalledTimes(1);
|
| 330 |
+
});
|
| 331 |
+
|
| 332 |
+
it("leaves the token alone when the request fails for another reason", async () => {
|
| 333 |
+
mockFetch.mockResolvedValue({ ok: false, status: 500 });
|
| 334 |
+
|
| 335 |
+
await expect(
|
| 336 |
+
searchModule.searchServiceInstance.performSearch<string[][]>(
|
| 337 |
+
"text",
|
| 338 |
+
"test query",
|
| 339 |
+
),
|
| 340 |
+
).rejects.toThrow("HTTP error! status: 500");
|
| 341 |
+
|
| 342 |
+
expect(mockFetch).toHaveBeenCalledTimes(1);
|
| 343 |
+
expect(invalidateConfig).not.toHaveBeenCalled();
|
| 344 |
+
});
|
| 345 |
+
});
|
| 346 |
+
|
| 347 |
describe("Update Cache Config Function", () => {
|
| 348 |
it.each([
|
| 349 |
["ttl", 60000],
|
client/modules/search.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
| 1 |
import Dexie, { type Table } from "dexie";
|
| 2 |
import { sha256 } from "hash-wasm";
|
|
|
|
| 3 |
import { addLogEntry } from "./logEntries";
|
| 4 |
import { getSearchTokenHash } from "./searchTokenHash";
|
| 5 |
import type { ImageSearchResults, TextSearchResults } from "./types";
|
|
@@ -188,6 +189,57 @@ async function executeCachedSearch<T extends SearchResults>(
|
|
| 188 |
}
|
| 189 |
}
|
| 190 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
interface SearchCacheEntry {
|
| 192 |
key: string;
|
| 193 |
timestamp: number;
|
|
@@ -402,33 +454,31 @@ const searchService = {
|
|
| 402 |
);
|
| 403 |
}
|
| 404 |
|
| 405 |
-
const searchUrl = new URL(`/search/${endpoint}`, self.location.origin);
|
| 406 |
-
searchUrl.searchParams.set("q", query);
|
| 407 |
-
searchUrl.searchParams.set("token", await getSearchTokenHash());
|
| 408 |
-
if (limit) searchUrl.searchParams.set("limit", limit.toString());
|
| 409 |
-
|
| 410 |
-
const controller = new AbortController();
|
| 411 |
-
const timeoutId = setTimeout(
|
| 412 |
-
() => controller.abort(),
|
| 413 |
-
CACHE_CONFIG.REQUEST_TIMEOUT,
|
| 414 |
-
);
|
| 415 |
-
|
| 416 |
try {
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
|
| 421 |
-
|
| 422 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 423 |
}
|
| 424 |
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
} catch (parseError) {
|
| 428 |
-
throw new Error(
|
| 429 |
-
`JSON parsing error: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
|
| 430 |
-
);
|
| 431 |
}
|
|
|
|
|
|
|
| 432 |
} catch (error) {
|
| 433 |
if (error instanceof Error) {
|
| 434 |
if (error.name === "AbortError") {
|
|
@@ -441,8 +491,6 @@ const searchService = {
|
|
| 441 |
}
|
| 442 |
}
|
| 443 |
throw error;
|
| 444 |
-
} finally {
|
| 445 |
-
clearTimeout(timeoutId);
|
| 446 |
}
|
| 447 |
},
|
| 448 |
|
|
|
|
| 1 |
import Dexie, { type Table } from "dexie";
|
| 2 |
import { sha256 } from "hash-wasm";
|
| 3 |
+
import { invalidateConfig } from "./config";
|
| 4 |
import { addLogEntry } from "./logEntries";
|
| 5 |
import { getSearchTokenHash } from "./searchTokenHash";
|
| 6 |
import type { ImageSearchResults, TextSearchResults } from "./types";
|
|
|
|
| 189 |
}
|
| 190 |
}
|
| 191 |
|
| 192 |
+
/** The status the server answers a search with when the token is not its own. */
|
| 193 |
+
const UNAUTHORIZED_STATUS = 401;
|
| 194 |
+
|
| 195 |
+
type SearchAttempt<T> =
|
| 196 |
+
| { ok: true; results: T }
|
| 197 |
+
| { ok: false; status: number };
|
| 198 |
+
|
| 199 |
+
async function buildSearchUrl(
|
| 200 |
+
endpoint: "text" | "images",
|
| 201 |
+
query: string,
|
| 202 |
+
limit?: number,
|
| 203 |
+
): Promise<URL> {
|
| 204 |
+
const searchUrl = new URL(`/search/${endpoint}`, self.location.origin);
|
| 205 |
+
searchUrl.searchParams.set("q", query);
|
| 206 |
+
searchUrl.searchParams.set("token", await getSearchTokenHash());
|
| 207 |
+
if (limit) searchUrl.searchParams.set("limit", limit.toString());
|
| 208 |
+
return searchUrl;
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
/**
|
| 212 |
+
* One request, with the timeout covering the body as well as the response, so
|
| 213 |
+
* a server that answers and then stalls mid-stream still aborts.
|
| 214 |
+
*/
|
| 215 |
+
async function attemptSearch<T>(searchUrl: URL): Promise<SearchAttempt<T>> {
|
| 216 |
+
const controller = new AbortController();
|
| 217 |
+
const timeoutId = setTimeout(
|
| 218 |
+
() => controller.abort(),
|
| 219 |
+
CACHE_CONFIG.REQUEST_TIMEOUT,
|
| 220 |
+
);
|
| 221 |
+
|
| 222 |
+
try {
|
| 223 |
+
const response = await fetch(searchUrl.toString(), {
|
| 224 |
+
signal: controller.signal,
|
| 225 |
+
});
|
| 226 |
+
|
| 227 |
+
if (!response.ok) {
|
| 228 |
+
return { ok: false, status: response.status };
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
try {
|
| 232 |
+
return { ok: true, results: (await response.json()) as T };
|
| 233 |
+
} catch (parseError) {
|
| 234 |
+
throw new Error(
|
| 235 |
+
`JSON parsing error: ${parseError instanceof Error ? parseError.message : String(parseError)}`,
|
| 236 |
+
);
|
| 237 |
+
}
|
| 238 |
+
} finally {
|
| 239 |
+
clearTimeout(timeoutId);
|
| 240 |
+
}
|
| 241 |
+
}
|
| 242 |
+
|
| 243 |
interface SearchCacheEntry {
|
| 244 |
key: string;
|
| 245 |
timestamp: number;
|
|
|
|
| 454 |
);
|
| 455 |
}
|
| 456 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 457 |
try {
|
| 458 |
+
let attempt = await attemptSearch<T>(
|
| 459 |
+
await buildSearchUrl(endpoint, query, limit),
|
| 460 |
+
);
|
| 461 |
|
| 462 |
+
// Every server process mints its own search token, so a restart leaves
|
| 463 |
+
// this page hashing one the server never issued. Drop the cached config,
|
| 464 |
+
// take the token the server is handing out now, and try once more. The
|
| 465 |
+
// retry runs once per request: a second rejection is a real one, and is
|
| 466 |
+
// reported as the failure it is.
|
| 467 |
+
if (!attempt.ok && attempt.status === UNAUTHORIZED_STATUS) {
|
| 468 |
+
addLogEntry(
|
| 469 |
+
"Search token was rejected; refreshing it and retrying the search",
|
| 470 |
+
);
|
| 471 |
+
invalidateConfig();
|
| 472 |
+
attempt = await attemptSearch<T>(
|
| 473 |
+
await buildSearchUrl(endpoint, query, limit),
|
| 474 |
+
);
|
| 475 |
}
|
| 476 |
|
| 477 |
+
if (!attempt.ok) {
|
| 478 |
+
throw new Error(`HTTP error! status: ${attempt.status}`);
|
|
|
|
|
|
|
|
|
|
|
|
|
| 479 |
}
|
| 480 |
+
|
| 481 |
+
return attempt.results;
|
| 482 |
} catch (error) {
|
| 483 |
if (error instanceof Error) {
|
| 484 |
if (error.name === "AbortError") {
|
|
|
|
| 491 |
}
|
| 492 |
}
|
| 493 |
throw error;
|
|
|
|
|
|
|
| 494 |
}
|
| 495 |
},
|
| 496 |
|
docs/configuration.md
CHANGED
|
@@ -304,7 +304,6 @@ engines:
|
|
| 304 |
```bash
|
| 305 |
docker run -p 7860:7860 \
|
| 306 |
-e SEARXNG_SETTINGS_PATH=/etc/searxng-custom/settings.yml \
|
| 307 |
-
-e TMPDIR=/home/node \
|
| 308 |
-v "$(pwd)/my-searxng-settings.yml:/etc/searxng-custom/settings.yml:ro" \
|
| 309 |
ghcr.io/felladrin/minisearch
|
| 310 |
```
|
|
@@ -319,18 +318,16 @@ services:
|
|
| 319 |
- "7860:7860"
|
| 320 |
environment:
|
| 321 |
- SEARXNG_SETTINGS_PATH=/etc/searxng-custom/settings.yml
|
| 322 |
-
- TMPDIR=/home/node
|
| 323 |
volumes:
|
| 324 |
- ./my-searxng-settings.yml:/etc/searxng-custom/settings.yml:ro
|
| 325 |
```
|
| 326 |
|
| 327 |
-
|
| 328 |
|
| 329 |
- **The file must be readable by uid 1000.** The container runs as `node`, and a file readable only by its owner on the host is invisible to it.
|
| 330 |
- **`secret_key` must be set to your own random value.** SearXNG refuses to start on the `ultrasecretkey` placeholder and logs `server.secret_key is not changed. Please use something else instead of ultrasecretkey.` The image's own file never carries the placeholder, because the Dockerfile replaces it at build time, but a file you write has to carry a real key.
|
| 331 |
-
- **`TMPDIR` must point somewhere `node` can write.** SearXNG keeps its SQLite caches in its temp directory, and the published image ships those files owned by root. Changing `secret_key` makes SearXNG wipe and rebuild them at startup, which fails against a root-owned file: `sqlite3.OperationalError: attempt to write a readonly database`. The app server keeps answering through that failure, so it reads as a broken MiniSearch rather than as a settings problem, and every search returns 502. The wipe follows from your key differing from the one baked into the image, so copying that value out of `/etc/searxng/settings.yml` avoids it, but that means carrying a build-time secret in your own file and redoing it on every image update. This is a property of the current image, tracked in #2732; once the image stops shipping root-owned caches the requirement goes away.
|
| 332 |
|
| 333 |
-
|
| 334 |
|
| 335 |
```bash
|
| 336 |
docker exec <container> sh -c 'cd /usr/local/searxng/searxng-src && /usr/local/searxng/searxng-venv/bin/python -m searx.webapp'
|
|
|
|
| 304 |
```bash
|
| 305 |
docker run -p 7860:7860 \
|
| 306 |
-e SEARXNG_SETTINGS_PATH=/etc/searxng-custom/settings.yml \
|
|
|
|
| 307 |
-v "$(pwd)/my-searxng-settings.yml:/etc/searxng-custom/settings.yml:ro" \
|
| 308 |
ghcr.io/felladrin/minisearch
|
| 309 |
```
|
|
|
|
| 318 |
- "7860:7860"
|
| 319 |
environment:
|
| 320 |
- SEARXNG_SETTINGS_PATH=/etc/searxng-custom/settings.yml
|
|
|
|
| 321 |
volumes:
|
| 322 |
- ./my-searxng-settings.yml:/etc/searxng-custom/settings.yml:ro
|
| 323 |
```
|
| 324 |
|
| 325 |
+
Two things about that command are required, not optional:
|
| 326 |
|
| 327 |
- **The file must be readable by uid 1000.** The container runs as `node`, and a file readable only by its owner on the host is invisible to it.
|
| 328 |
- **`secret_key` must be set to your own random value.** SearXNG refuses to start on the `ultrasecretkey` placeholder and logs `server.secret_key is not changed. Please use something else instead of ultrasecretkey.` The image's own file never carries the placeholder, because the Dockerfile replaces it at build time, but a file you write has to carry a real key.
|
|
|
|
| 329 |
|
| 330 |
+
A SearXNG startup failure will not show in `docker logs`. The container's CMD sends SearXNG's output to `/dev/null`, deliberately, so that the app server's log stays clean. To see it, run SearXNG in the foreground inside the container, which is only possible once the original has failed: on a running instance the second process collides with the live one on `127.0.0.1:8888` and reports the address as already in use.
|
| 331 |
|
| 332 |
```bash
|
| 333 |
docker exec <container> sh -c 'cd /usr/local/searxng/searxng-src && /usr/local/searxng/searxng-venv/bin/python -m searx.webapp'
|
docs/page-content.md
CHANGED
|
@@ -100,6 +100,24 @@ they are not truncated to a shortlist. An unloaded model (empty dense scores)
|
|
| 100 |
or a scoring error also keeps lexical ranking. Errors log a fixed diagnostic
|
| 101 |
without query, URL, passage text or model error details.
|
| 102 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 103 |
The limit bounds inference work, not elapsed time. On an Apple M2 Pro CPU,
|
| 104 |
ONNX Runtime 1.30.0 took 7.42β7.48 seconds for 256 passages, 15.07β15.20 seconds
|
| 105 |
for 512, and 22.37β22.94 seconds for 768, with two warm runs per pool using
|
|
|
|
| 100 |
or a scoring error also keeps lexical ranking. Errors log a fixed diagnostic
|
| 101 |
without query, URL, passage text or model error details.
|
| 102 |
|
| 103 |
+
Dense scoring runs in a worker thread (`server/biEncoderWorker.ts`), not on
|
| 104 |
+
the server's main thread. `onnxruntime-node` runs inference synchronously on
|
| 105 |
+
whichever JS thread calls it, so the pass used to stop the event loop for its
|
| 106 |
+
whole duration and no other request could be served while one page-content read
|
| 107 |
+
ranked its passages. `biEncoderService.ts` keeps its four functions and now
|
| 108 |
+
only ships the query and the passages across, and takes `number[]` scores back;
|
| 109 |
+
the model and the embeddings stay in the worker. On this host (x86_64, 32
|
| 110 |
+
logical cores) a 200-passage pass blocked the main thread for 815 ms before the
|
| 111 |
+
move and 1.5 ms after, for identical scores.
|
| 112 |
+
|
| 113 |
+
The worker caps ONNX Runtime's intra-op threads at half the reported
|
| 114 |
+
parallelism. The default sizes that pool from the logical core count and its
|
| 115 |
+
threads spin before yielding, which oversubscribes every core: the same pass
|
| 116 |
+
measured 31-38 ms of main-thread delay and 7.1 s of wall time at the default
|
| 117 |
+
against 1-4 ms and 2.6 s when capped. The cross-encoder is untouched and still
|
| 118 |
+
runs on the main thread, where it keeps its block to about 13 ms by refusing to
|
| 119 |
+
batch.
|
| 120 |
+
|
| 121 |
The limit bounds inference work, not elapsed time. On an Apple M2 Pro CPU,
|
| 122 |
ONNX Runtime 1.30.0 took 7.42β7.48 seconds for 256 passages, 15.07β15.20 seconds
|
| 123 |
for 512, and 22.37β22.94 seconds for 768, with two warm runs per pool using
|