src/tokenizer/protocol.ts

import type {
  LoadProgress,
  TokenizationResult,
  TokenizerMetadata,
} from "./types";

export type WorkerRequest =
  | { id: number; type: "load"; reference: string }
  | {
      id: number;
      type: "tokenize";
      text: string;
      addSpecialTokens: boolean;
    }
  | { id: number; type: "cache-size" }
  | { id: number; type: "clear-cache" };

export type WorkerRequestPayload = WorkerRequest extends infer Request
  ? Request extends { id: number }
    ? Omit<Request, "id">
    : never
  : never;

export type WorkerSuccessPayload =
  | TokenizerMetadata
  | TokenizationResult
  | number
  | null;

export type WorkerResponse =
  | {
      id: number;
      type: "progress";
      progress: LoadProgress;
    }
  | {
      id: number;
      type: "success";
      payload: WorkerSuccessPayload;
    }
  | {
      id: number;
      type: "error";
      message: string;
    };