Spaces:
Sleeping
Sleeping
| export type TokenizerFormat = | |
| | "tokenizers" | |
| | "sentencepiece" | |
| | "tiktoken" | |
| | "tekken"; | |
| export interface ModelReference { | |
| repoId: string; | |
| revision: string; | |
| } | |
| export interface HubSibling { | |
| rfilename: string; | |
| blobId?: string; | |
| size?: number; | |
| lfs?: { | |
| sha256: string; | |
| size: number; | |
| pointerSize: number; | |
| }; | |
| } | |
| export interface FileHash { | |
| algorithm: "git-sha1" | "sha256"; | |
| value: string; | |
| } | |
| export interface HubFile { | |
| filename: string; | |
| hash: FileHash; | |
| } | |
| export interface RemoteFile extends HubFile { | |
| url: string; | |
| } | |
| export interface HubModelInfo { | |
| id?: string; | |
| modelId?: string; | |
| sha: string; | |
| private?: boolean; | |
| gated?: boolean | "auto" | "manual"; | |
| siblings: HubSibling[]; | |
| config?: { | |
| model_type?: string; | |
| tokenizer_config?: Record<string, unknown>; | |
| }; | |
| } | |
| export interface ResolvedTokenizer { | |
| repoId: string; | |
| requestedRevision: string; | |
| commitSha: string; | |
| format: TokenizerFormat; | |
| file: HubFile; | |
| availableFiles: HubFile[]; | |
| tokenizerConfig?: Record<string, unknown>; | |
| } | |
| export interface LoadProgress { | |
| phase: "resolving" | "downloading" | "initializing"; | |
| loaded: number; | |
| total: number | null; | |
| cached?: boolean; | |
| message: string; | |
| } | |
| export type SpecialTokenMap = Record<string, string | string[]>; | |
| export interface TokenizerMetadata { | |
| repoId: string; | |
| requestedRevision: string; | |
| commitSha: string; | |
| format: TokenizerFormat; | |
| filename: string; | |
| tokenizerType: string; | |
| vocabSize: number | null; | |
| specialTokens: SpecialTokenMap; | |
| fileSize: number; | |
| loadTimeMs: number; | |
| sourceUrl: string; | |
| } | |
| export interface TokenPiece { | |
| index: number; | |
| id: number; | |
| text: string; | |
| displayText: string; | |
| special: boolean; | |
| } | |
| export interface TokenizationResult { | |
| tokens: TokenPiece[]; | |
| tokenCount: number; | |
| characterCount: number; | |
| elapsedMs: number; | |
| } | |