// src/text-index/tokenize.ts
//
// Tokenizer of the full-text index: Latin/number words + CJK unigrams &
// bigrams (no dictionary, zero dependencies, works for Chinese without a
// segmenter), plus the document text extraction shared with the generation
// build's worker. This file is part of the worker's import closure, so every
// relative import carries an explicit `.ts` specifier (see the IMPORT NOTE in
// worker/text-build-core.ts).

import { getPath } from '../query.ts';

const LATIN = /[a-z0-9]+/g;
const CJK = /[\u3400-\u9fff\u3040-\u30ff\uff00-\uffef]+/g;
// Postings records store the term length in a uint16. A single document with a
// longer token previously made every postings rebuild throw after the index had
// already been cleared, permanently poisoning the index (and compaction). Such
// tokens can never be real query terms — drop them at tokenization so one
// pathological document cannot destroy the index.
const MAX_TERM_CHARS = 0xffff;
// The same postings uint16 limit in UTF-8 BYTES (encodeRecord's unit). A
// custom tokenizer's output is validated against it at every write boundary
// (tokensFor): an overlong term rejects the write loudly instead of poisoning
// the next postings rebuild (review #27).
export const MAX_TERM_BYTES = 0xffff;

export const yieldToLoop = (): Promise<void> => new Promise((r) => setImmediate(r));

/** Tokenize text into terms (lowercased latin words + CJK uni/bigrams). */
export function tokenize(str: unknown): string[] {
  const s = String(str).toLowerCase();
  const terms: string[] = [];
  const latin = s.match(LATIN);
  // Loop-push instead of `terms.push(...latin)`: spreading a large match array
  // (hundreds of thousands of tokens from a big doc) overflows the call stack.
  // Latin matches are ASCII, so chars == utf8 bytes for the length guard.
  if (latin) for (const t of latin) if (t.length <= MAX_TERM_CHARS) terms.push(t);
  const runs = s.match(CJK) ?? [];
  for (const r of runs) {
    for (let i = 0; i < r.length; i++) {
      terms.push(r[i]!);
      if (i + 1 < r.length) terms.push(r[i]! + r[i + 1]!);
    }
  }
  return terms;
}

function stringLeaves(obj: unknown, acc: string[] = []): string[] {
  if (obj === null || obj === undefined) return acc;
  if (typeof obj === 'string') {
    acc.push(obj);
    return acc;
  }
  if (typeof obj !== 'object') return acc;
  for (const v of Object.values(obj as Record<string, unknown>)) stringLeaves(v, acc);
  return acc;
}

/** Extract the indexable text of a document (shared with the generation
 *  build's worker: fields mode joins the configured paths, otherwise every
 *  string leaf). Internal to the package. */
export function extractText(fields: readonly string[] | null, doc: unknown): string {
  if (fields && fields.length) {
    return fields
      .map((f) => getPath(doc, f))
      .filter((v): v is string => typeof v === 'string')
      .join(' ');
  }
  return stringLeaves(doc).join(' ');
}