Token Classification
ONNX
Safetensors
Transformers.js
Nepali
English
Hindi
bert
named-entity-recognition
pii
pii-detection
privacy
redaction
anonymization
nepali
romanized-nepali
devanagari
english
wasm
browser
rakshya
gopya
fiste-ai
Instructions to use fiste-ai/rakshya with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers.js
How to use fiste-ai/rakshya with Transformers.js:
// npm i @huggingface/transformers import { pipeline } from '@huggingface/transformers'; // Allocate pipeline const pipe = await pipeline('token-classification', 'fiste-ai/rakshya');
Download web/pkg/rakshya.js from fiste-ai/rakshya: direct link, hf CLI and curl.
- Browser
- Download file 15 kB
-
https://huggingface.co/fiste-ai/rakshya/resolve/main/web/pkg/rakshya.js
- Command line
-
hf download hf://fiste-ai/rakshya/web/pkg/rakshya.js
-
curl -L -o rakshya.js https://huggingface.co/fiste-ai/rakshya/resolve/main/web/pkg/rakshya.js
15 kB
| /* @ts-self-types="./rakshya.d.ts" */ | |
| /** | |
| * The tokenizer and labels of one model. | |
| */ | |
| export class Rakshya { | |
| __destroy_into_raw() { | |
| const ptr = this.__wbg_ptr; | |
| this.__wbg_ptr = 0; | |
| RakshyaFinalization.unregister(this); | |
| return ptr; | |
| } | |
| free() { | |
| const ptr = this.__destroy_into_raw(); | |
| wasm.__wbg_rakshya_free(ptr, 0); | |
| } | |
| /** | |
| * `tokenizer_json` and `tagger_json` are the text of those two files in the model folder. | |
| * @param {string} tokenizer_json | |
| * @param {string} tagger_json | |
| */ | |
| constructor(tokenizer_json, tagger_json) { | |
| let ptr0, len0, ptr1, len1; | |
| try { | |
| ptr0 = passStringToWasm0(tokenizer_json, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); | |
| len0 = WASM_VECTOR_LEN; | |
| ptr1 = passStringToWasm0(tagger_json, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); | |
| len1 = WASM_VECTOR_LEN; | |
| } catch (e) { | |
| if (ptr1) { wasm.__wbindgen_free(ptr1, len1 * 1, 1); } | |
| if (ptr0) { wasm.__wbindgen_free(ptr0, len0 * 1, 1); } | |
| throw e; | |
| } | |
| const ret = wasm.rakshya_new(ptr0, len0, ptr1, len1); | |
| if (ret[2]) { | |
| throw takeFromExternrefTable0(ret[1]); | |
| } | |
| this.__wbg_ptr = ret[0]; | |
| RakshyaFinalization.register(this, this.__wbg_ptr, this); | |
| return this; | |
| } | |
| /** | |
| * Begin work on one text: run the rules and cut the text into pieces for the model. | |
| * @param {string} text | |
| * @returns {Run} | |
| */ | |
| start(text) { | |
| const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); | |
| const len0 = WASM_VECTOR_LEN; | |
| const ret = wasm.rakshya_start(this.__wbg_ptr, ptr0, len0); | |
| return Run.__wrap(ret); | |
| } | |
| } | |
| if (Symbol.dispose) Rakshya.prototype[Symbol.dispose] = Rakshya.prototype.free; | |
| /** | |
| * One text on its way through the model, piece by piece. | |
| */ | |
| export class Run { | |
| static __wrap(ptr) { | |
| const obj = Object.create(Run.prototype); | |
| obj.__wbg_ptr = ptr; | |
| RunFinalization.register(obj, obj.__wbg_ptr, obj); | |
| return obj; | |
| } | |
| __destroy_into_raw() { | |
| const ptr = this.__wbg_ptr; | |
| this.__wbg_ptr = 0; | |
| RunFinalization.unregister(this); | |
| return ptr; | |
| } | |
| free() { | |
| const ptr = this.__destroy_into_raw(); | |
| wasm.__wbg_run_free(ptr, 0); | |
| } | |
| /** | |
| * How many pieces the model has to read. | |
| * @returns {number} | |
| */ | |
| get count() { | |
| const ret = wasm.run_count(this.__wbg_ptr); | |
| return ret >>> 0; | |
| } | |
| /** | |
| * Token ids of piece `i`, the model's `input_ids`. The attention mask is all ones. | |
| * @param {number} i | |
| * @returns {BigInt64Array} | |
| */ | |
| encode(i) { | |
| const ret = wasm.run_encode(this.__wbg_ptr, i); | |
| if (ret[3]) { | |
| throw takeFromExternrefTable0(ret[2]); | |
| } | |
| var v1 = getArrayI64FromWasm0(ret[0], ret[1]).slice(); | |
| wasm.__wbindgen_free(ret[0], ret[1] * 8, 8); | |
| return v1; | |
| } | |
| /** | |
| * Everything found, as JSON `[[start, end, label, source], ...]`; source is "rule" or "tagger". | |
| * @returns {string} | |
| */ | |
| finish() { | |
| let deferred1_0; | |
| let deferred1_1; | |
| try { | |
| const ret = wasm.run_finish(this.__wbg_ptr); | |
| deferred1_0 = ret[0]; | |
| deferred1_1 = ret[1]; | |
| return getStringFromWasm0(ret[0], ret[1]); | |
| } finally { | |
| wasm.__wbindgen_free(deferred1_0, deferred1_1, 1); | |
| } | |
| } | |
| /** | |
| * Give back the model's two outputs for piece `i`: `logits` and `trim_logits`. | |
| * @param {number} i | |
| * @param {Float32Array | number[]} logits | |
| * @param {Float32Array | number[]} trim_logits | |
| */ | |
| push(i, logits, trim_logits) { | |
| const val3 = +i; | |
| let ptr0, len0, ptr1, len1; | |
| try { | |
| ptr0 = passArrayF32ToWasm0(logits, wasm.__wbindgen_malloc); | |
| len0 = WASM_VECTOR_LEN; | |
| ptr1 = passArrayF32ToWasm0(trim_logits, wasm.__wbindgen_malloc); | |
| len1 = WASM_VECTOR_LEN; | |
| } catch (e) { | |
| if (ptr1) { wasm.__wbindgen_free(ptr1, len1 * 4, 4); } | |
| if (ptr0) { wasm.__wbindgen_free(ptr0, len0 * 4, 4); } | |
| throw e; | |
| } | |
| wasm.run_push(this.__wbg_ptr, val3, ptr0, len0, ptr1, len1); | |
| } | |
| /** | |
| * The text with every found item replaced by a placeholder such as `[PERSON_1]`. | |
| * @returns {string} | |
| */ | |
| redacted() { | |
| let deferred1_0; | |
| let deferred1_1; | |
| try { | |
| const ret = wasm.run_redacted(this.__wbg_ptr); | |
| deferred1_0 = ret[0]; | |
| deferred1_1 = ret[1]; | |
| return getStringFromWasm0(ret[0], ret[1]); | |
| } finally { | |
| wasm.__wbindgen_free(deferred1_0, deferred1_1, 1); | |
| } | |
| } | |
| } | |
| if (Symbol.dispose) Run.prototype[Symbol.dispose] = Run.prototype.free; | |
| /** | |
| * What went wrong the last time a call into this module failed; empty if nothing did. | |
| * @returns {string} | |
| */ | |
| export function failure() { | |
| let deferred1_0; | |
| let deferred1_1; | |
| try { | |
| const ret = wasm.failure(); | |
| deferred1_0 = ret[0]; | |
| deferred1_1 = ret[1]; | |
| return getStringFromWasm0(ret[0], ret[1]); | |
| } finally { | |
| wasm.__wbindgen_free(deferred1_0, deferred1_1, 1); | |
| } | |
| } | |
| /** | |
| * The identifiers the rules find, as JSON `[[start, end, label], ...]`. No model is needed. | |
| * @param {string} text | |
| * @returns {string} | |
| */ | |
| export function find(text) { | |
| let deferred2_0; | |
| let deferred2_1; | |
| try { | |
| const ptr0 = passStringToWasm0(text, wasm.__wbindgen_malloc, wasm.__wbindgen_realloc); | |
| const len0 = WASM_VECTOR_LEN; | |
| const ret = wasm.find(ptr0, len0); | |
| deferred2_0 = ret[0]; | |
| deferred2_1 = ret[1]; | |
| return getStringFromWasm0(ret[0], ret[1]); | |
| } finally { | |
| wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); | |
| } | |
| } | |
| /** | |
| * Keep the message of a failure in this module, so the page can report more than "unreachable". | |
| */ | |
| export function start() { | |
| wasm.start(); | |
| } | |
| function __wbg_get_imports() { | |
| const import0 = { | |
| __proto__: null, | |
| __wbg_Error_208446807d0d1e30: function(arg0, arg1) { | |
| const ret = Error(getStringFromWasm0(arg0, arg1)); | |
| return ret; | |
| }, | |
| __wbg___wbindgen_throw_4c88d8bf264fd984: function(arg0, arg1) { | |
| throw new Error(getStringFromWasm0(arg0, arg1)); | |
| }, | |
| __wbg_getRandomValues_5d8c030b527203a6: function() { return handleError(function (arg0, arg1) { | |
| globalThis.crypto.getRandomValues(getArrayU8FromWasm0(arg0, arg1)); | |
| }, arguments); }, | |
| __wbindgen_init_externref_table: function() { | |
| const table = wasm.__wbindgen_externrefs; | |
| table.set(0, undefined); | |
| table.set(1024, undefined); | |
| table.set(1025, null); | |
| table.set(1026, true); | |
| table.set(1027, false); | |
| }, | |
| }; | |
| return { | |
| __proto__: null, | |
| "./rakshya_bg.js": import0, | |
| }; | |
| } | |
| const RakshyaFinalization = (typeof FinalizationRegistry === 'undefined') | |
| ? { register: () => {}, unregister: () => {} } | |
| : new FinalizationRegistry(ptr => wasm.__wbg_rakshya_free(ptr, 1)); | |
| const RunFinalization = (typeof FinalizationRegistry === 'undefined') | |
| ? { register: () => {}, unregister: () => {} } | |
| : new FinalizationRegistry(ptr => wasm.__wbg_run_free(ptr, 1)); | |
| function addToExternrefTable0(obj) { | |
| const idx = wasm.__externref_table_alloc(); | |
| wasm.__wbindgen_externrefs.set(idx, obj); | |
| return idx; | |
| } | |
| function getArrayI64FromWasm0(ptr, len) { | |
| ptr = ptr >>> 0; | |
| return getBigInt64ArrayMemory0().subarray(ptr / 8, ptr / 8 + len); | |
| } | |
| function getArrayU8FromWasm0(ptr, len) { | |
| ptr = ptr >>> 0; | |
| return getUint8ArrayMemory0().subarray(ptr / 1, ptr / 1 + len); | |
| } | |
| let cachedBigInt64ArrayMemory0 = null; | |
| function getBigInt64ArrayMemory0() { | |
| if (cachedBigInt64ArrayMemory0 === null || cachedBigInt64ArrayMemory0.byteLength === 0) { | |
| cachedBigInt64ArrayMemory0 = new BigInt64Array(wasm.memory.buffer); | |
| } | |
| return cachedBigInt64ArrayMemory0; | |
| } | |
| let cachedFloat32ArrayMemory0 = null; | |
| function getFloat32ArrayMemory0() { | |
| if (cachedFloat32ArrayMemory0 === null || cachedFloat32ArrayMemory0.byteLength === 0) { | |
| cachedFloat32ArrayMemory0 = new Float32Array(wasm.memory.buffer); | |
| } | |
| return cachedFloat32ArrayMemory0; | |
| } | |
| function getStringFromWasm0(ptr, len) { | |
| ptr = ptr >>> 0; | |
| return decodeText(getUint8ArrayMemory0().subarray(ptr, ptr + len)); | |
| } | |
| let cachedUint8ArrayMemory0 = null; | |
| function getUint8ArrayMemory0() { | |
| if (cachedUint8ArrayMemory0 === null || cachedUint8ArrayMemory0.byteLength === 0) { | |
| cachedUint8ArrayMemory0 = new Uint8Array(wasm.memory.buffer); | |
| } | |
| return cachedUint8ArrayMemory0; | |
| } | |
| function handleError(f, args) { | |
| try { | |
| return f.apply(this, args); | |
| } catch (e) { | |
| const idx = addToExternrefTable0(e); | |
| wasm.__wbindgen_exn_store(idx); | |
| } | |
| } | |
| function passArrayF32ToWasm0(arg, malloc) { | |
| const ptr = malloc(arg.length * 4, 4) >>> 0; | |
| getFloat32ArrayMemory0().set(arg, ptr / 4); | |
| WASM_VECTOR_LEN = arg.length; | |
| return ptr; | |
| } | |
| function passStringToWasm0(arg, malloc, realloc) { | |
| if (realloc === undefined) { | |
| const buf = cachedTextEncoder.encode(arg); | |
| const ptr = malloc(buf.length, 1) >>> 0; | |
| getUint8ArrayMemory0().subarray(ptr, ptr + buf.length).set(buf); | |
| WASM_VECTOR_LEN = buf.length; | |
| return ptr; | |
| } | |
| let len = arg.length; | |
| let ptr = malloc(len, 1) >>> 0; | |
| const mem = getUint8ArrayMemory0(); | |
| let offset = 0; | |
| for (; offset < len; offset++) { | |
| const code = arg.charCodeAt(offset); | |
| if (code > 0x7F) break; | |
| mem[ptr + offset] = code; | |
| } | |
| if (offset !== len) { | |
| if (offset !== 0) { | |
| arg = arg.slice(offset); | |
| } | |
| ptr = realloc(ptr, len, len = offset + arg.length * 3, 1) >>> 0; | |
| const view = getUint8ArrayMemory0().subarray(ptr + offset, ptr + len); | |
| const ret = cachedTextEncoder.encodeInto(arg, view); | |
| offset += ret.written; | |
| ptr = realloc(ptr, len, offset, 1) >>> 0; | |
| } | |
| WASM_VECTOR_LEN = offset; | |
| return ptr; | |
| } | |
| function takeFromExternrefTable0(idx) { | |
| const value = wasm.__wbindgen_externrefs.get(idx); | |
| wasm.__externref_table_dealloc(idx); | |
| return value; | |
| } | |
| let cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true }); | |
| cachedTextDecoder.decode(); | |
| const MAX_SAFARI_DECODE_BYTES = 2146435072; | |
| let numBytesDecoded = 0; | |
| function decodeText(bytes) { | |
| numBytesDecoded += bytes.length; | |
| if (numBytesDecoded >= MAX_SAFARI_DECODE_BYTES) { | |
| cachedTextDecoder = new TextDecoder('utf-8', { ignoreBOM: true }); | |
| cachedTextDecoder.decode(); | |
| numBytesDecoded = bytes.length; | |
| } | |
| return cachedTextDecoder.decode(bytes); | |
| } | |
| const cachedTextEncoder = new TextEncoder(); | |
| if (!('encodeInto' in cachedTextEncoder)) { | |
| cachedTextEncoder.encodeInto = function (arg, view) { | |
| const buf = cachedTextEncoder.encode(arg); | |
| view.set(buf); | |
| return { | |
| read: arg.length, | |
| written: buf.length | |
| }; | |
| }; | |
| } | |
| let WASM_VECTOR_LEN = 0; | |
| let wasmModule, wasmInstance, wasm; | |
| function __wbg_finalize_init(instance, module) { | |
| wasmInstance = instance; | |
| wasm = instance.exports; | |
| wasmModule = module; | |
| cachedBigInt64ArrayMemory0 = null; | |
| cachedFloat32ArrayMemory0 = null; | |
| cachedUint8ArrayMemory0 = null; | |
| wasm.__wbindgen_start(); | |
| return wasm; | |
| } | |
| async function __wbg_load(module, imports) { | |
| if (typeof Response === 'function' && module instanceof Response) { | |
| if (!module.ok) { | |
| throw new Error(`failed to fetch Wasm: ${module.status} ${module.statusText} fetching '${module.url}'`); | |
| } | |
| if (typeof WebAssembly.instantiateStreaming === 'function') { | |
| try { | |
| return await WebAssembly.instantiateStreaming(module, imports); | |
| } catch (e) { | |
| const validResponse = expectedResponseType(module.type); | |
| if (validResponse && module.headers.get('Content-Type') !== 'application/wasm') { | |
| console.warn("`WebAssembly.instantiateStreaming` failed because your server does not serve Wasm with `application/wasm` MIME type. Falling back to `WebAssembly.instantiate` which is slower. Original error:\n", e); | |
| } else { throw e; } | |
| } | |
| } | |
| const bytes = await module.arrayBuffer(); | |
| return await WebAssembly.instantiate(bytes, imports); | |
| } else { | |
| const instance = await WebAssembly.instantiate(module, imports); | |
| if (instance instanceof WebAssembly.Instance) { | |
| return { instance, module }; | |
| } else { | |
| return instance; | |
| } | |
| } | |
| function expectedResponseType(type) { | |
| switch (type) { | |
| case 'basic': case 'cors': case 'default': return true; | |
| } | |
| return false; | |
| } | |
| } | |
| function initSync(module) { | |
| if (wasm !== undefined) return wasm; | |
| if (module !== undefined) { | |
| if (Object.getPrototypeOf(module) === Object.prototype) { | |
| ({module} = module) | |
| } else { | |
| console.warn('using deprecated parameters for `initSync()`; pass a single object instead') | |
| } | |
| } | |
| const imports = __wbg_get_imports(); | |
| if (!(module instanceof WebAssembly.Module)) { | |
| module = new WebAssembly.Module(module); | |
| } | |
| const instance = new WebAssembly.Instance(module, imports); | |
| return __wbg_finalize_init(instance, module); | |
| } | |
| let __wbg_init_promise; | |
| async function __wbg_init(module_or_path) { | |
| if (wasm !== undefined) return wasm; | |
| if (__wbg_init_promise === undefined) { | |
| __wbg_init_promise = __wbg_init_once(module_or_path); | |
| __wbg_init_promise.catch(() => { __wbg_init_promise = undefined; }); | |
| } | |
| return __wbg_init_promise; | |
| } | |
| async function __wbg_init_once(module_or_path) { | |
| if (module_or_path !== undefined) { | |
| if (Object.getPrototypeOf(module_or_path) === Object.prototype) { | |
| ({module_or_path} = module_or_path) | |
| } else { | |
| console.warn('using deprecated parameters for the initialization function; pass a single object instead') | |
| } | |
| } | |
| if (module_or_path === undefined) { | |
| module_or_path = new URL('rakshya_bg.wasm', import.meta.url); | |
| } | |
| const imports = __wbg_get_imports(); | |
| if (typeof module_or_path === 'string' || (typeof Request === 'function' && module_or_path instanceof Request) || (typeof URL === 'function' && module_or_path instanceof URL)) { | |
| module_or_path = fetch(module_or_path); | |
| } | |
| const { instance, module } = await __wbg_load(await module_or_path, imports); | |
| return __wbg_finalize_init(instance, module); | |
| } | |
| export { initSync, __wbg_init as default }; | |