OpceanAI commited on
Commit
80d6ea1
·
verified ·
1 Parent(s): 18fbe50

Upload pasita.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. pasita.py +433 -1122
pasita.py CHANGED
@@ -1,1160 +1,471 @@
1
- """Self-contained inference harness for OpceanAI/PASITA text-to-Markdown.
2
-
3
- No gradio/spaces imports. Works on CPU and bf16 GPU. CLI:
4
- python3 harness.py --text-file doc.txt
5
- """
6
-
7
- from __future__ import annotations
8
-
9
- import argparse
10
- import math
11
- import re
12
  import time
13
- from dataclasses import dataclass, field
14
- from html.parser import HTMLParser
15
- from typing import Any
16
 
 
17
  import torch
 
 
18
 
19
- MODEL_ID = "OpceanAI/PASITA"
20
- CONTEXT_LEN = 2048
21
- DEFAULT_HARD_CAP = 512
22
-
23
 
24
- def build_prompt(text: str, mode: str = "canonical", instruction: str | None = None) -> str:
25
- if instruction is not None:
26
- return instruction + text + "\n\n### Markdown:\n"
27
- if mode == "canonical":
28
- return "CONVIERTE A MARKDOWN:\n" + text + "\n\n### Markdown:\n"
29
- if mode == "english":
30
- return "CONVERT TO MARKDOWN:\n" + text + "\n\n### Markdown:\n"
31
- if mode == "spanish_instr":
32
- return (
33
- "Transforma el siguiente texto en Markdown, conservando toda la información:\n\n"
34
- + text
35
- + "\n\n### Markdown:\n"
36
- )
37
- if mode == "fewshot":
38
- return (
39
- "CONVIERTE A MARKDOWN:\n\n"
40
- "### Ejemplo 1\n"
41
- "Entrada: <b>Aviso</b><p>Entrega en 24 h.</p>\n"
42
- "Salida:\n"
43
- "## Aviso\n\n"
44
- "Entrega en 24 h.\n\n"
45
- "### Ejemplo 2\n"
46
- "Entrada: producto: teclado; precio: 49,90 €\n"
47
- "Salida:\n"
48
- "- **producto:** teclado\n"
49
- "- **precio:** 49,90 €\n\n"
50
- "### Documento\n"
51
- + text
52
- + "\n\n### Markdown:\n"
53
- )
54
- if mode == "plain":
55
- return text
56
- if mode == "md_label":
57
- return "CONVIERTE A MARKDOWN:\n" + text + "\n\nMarkdown:\n"
58
- if mode == "sep_only":
59
- return text + "\n\n### Markdown:\n"
60
- if mode == "doc_instr_es":
61
- return "Convierte este documento a Markdown:\n" + text + "\n\n### Markdown:\n"
62
- if mode == "doc_instr_en":
63
- return "Convert this document to Markdown:\n" + text + "\n\n### Markdown:\n"
64
- if mode == "csv_instr":
65
- return "Convierte este CSV a tabla markdown:\n" + text + "\n\n### Markdown:\n"
66
- if mode == "html_instr":
67
- return "Convierte este HTML a Markdown:\n" + text + "\n\n### Markdown:\n"
68
- if mode == "html_instr_en":
69
- return "Convert this HTML to Markdown:\n" + text + "\n\n### Markdown:\n"
70
- if mode == "fewshot2":
71
- return (
72
- "CONVIERTE A MARKDOWN:\n\n"
73
- "Ejemplo:\n"
74
- "Entrada: Puntos\n"
75
- "1. Primero.\n"
76
- "2. Segundo.\n"
77
- "Salida:\n"
78
- "## Puntos\n\n"
79
- "1. Primero.\n"
80
- "2. Segundo.\n\n"
81
- "Documento:\n"
82
- + text
83
- + "\n\n### Markdown:\n"
84
- )
85
- if mode == "respuesta":
86
- return "CONVIERTE A MARKDOWN:\n" + text + "\n\n### Respuesta:\n"
87
- if mode == "output":
88
- return "CONVIERTE A MARKDOWN:\n" + text + "\n\n### Output:\n"
89
- raise ValueError(f"unknown prompt mode: {mode}")
90
-
91
-
92
- @dataclass
93
- class GenerationResult:
94
- candidates: list[str]
95
- raw_candidates: list[str]
96
- prompt: str
97
- prompt_tokens: int
98
- max_new_tokens: int
99
- seconds: float
100
- strategy: dict[str, Any] = field(default_factory=dict)
101
-
102
- @property
103
- def best_index(self) -> int:
104
- return self.strategy.get("best_index", 0)
105
-
106
- @property
107
- def best(self) -> str:
108
- return self.candidates[self.best_index]
109
-
110
-
111
- def _model_device(model) -> torch.device:
112
- try:
113
- return next(model.parameters()).device
114
- except StopIteration:
115
- return torch.device("cpu")
116
-
117
-
118
- def compute_max_new_tokens(prompt_tokens: int, ratio: float = 1.5, hard_cap: int = DEFAULT_HARD_CAP) -> int:
119
- budget = CONTEXT_LEN - prompt_tokens - 1
120
- if budget <= 16:
121
- return 16
122
- by_ratio = int(math.ceil(prompt_tokens * ratio))
123
- return max(16, min(hard_cap, budget, by_ratio))
124
-
125
-
126
- def _truncate_text_to_tokens(tokenizer, text: str, max_tokens: int) -> str:
127
- ids = tokenizer.encode(text, add_special_tokens=False)
128
- if len(ids) <= max_tokens:
129
- return text
130
- head = int(max_tokens * 0.6)
131
- tail = max_tokens - head
132
- head_text = tokenizer.decode(ids[:head])
133
- tail_text = tokenizer.decode(ids[-tail:]) if tail > 0 else ""
134
- return head_text + "\n...\n" + tail_text
135
 
 
 
 
 
136
 
137
- def generate_candidates(
138
- model,
139
- tokenizer,
140
- text: str,
141
- *,
142
- mode: str = "canonical",
143
- instruction: str | None = None,
144
- num_candidates: int = 1,
145
- num_beams: int = 1,
146
- do_sample: bool = False,
147
- temperature: float = 1.0,
148
- top_p: float = 0.95,
149
- no_repeat_ngram_size: int = 0,
150
- repetition_penalty: float = 1.0,
151
- max_new_tokens: int | str | None = None,
152
- max_new_ratio: float = 1.5,
153
- hard_cap: int = DEFAULT_HARD_CAP,
154
- use_cache: bool = True,
155
- early_stopping: bool | None = None,
156
- seed: int = 0,
157
- device: torch.device | str | None = None,
158
- **gen_kwargs: Any,
159
- ) -> GenerationResult:
160
- tokenizer.padding_side = "left"
161
- prompt = build_prompt(text, mode, instruction)
162
- max_new = max_new_tokens
163
- if max_new is None or max_new == "auto":
164
- prompt_ids = tokenizer(prompt, add_special_tokens=True)["input_ids"]
165
- if len(prompt_ids) > CONTEXT_LEN - 64:
166
- text = _truncate_text_to_tokens(tokenizer, text, CONTEXT_LEN - 96)
167
- prompt = build_prompt(text, mode, instruction)
168
- prompt_ids = tokenizer(prompt, add_special_tokens=True)["input_ids"]
169
- max_new = compute_max_new_tokens(len(prompt_ids), max_new_ratio, hard_cap)
170
- else:
171
- max_new = int(max_new)
172
 
173
- inputs = tokenizer(
174
- prompt,
175
- return_tensors="pt",
176
- add_special_tokens=True,
177
- truncation=True,
178
- max_length=CONTEXT_LEN - max_new,
179
- )
180
- if device is None:
181
- device = _model_device(model)
182
- inputs = {k: v.to(device) for k, v in inputs.items()}
183
- prompt_tokens = int(inputs["input_ids"].shape[1])
184
 
185
- if do_sample:
186
- torch.manual_seed(seed)
187
- if early_stopping is None:
188
- early_stopping = num_beams > 1
189
 
190
- generate_kwargs: dict[str, Any] = dict(
191
- max_new_tokens=max_new,
192
- do_sample=do_sample,
193
- num_beams=max(1, num_beams),
194
- num_return_sequences=max(1, num_candidates),
195
- use_cache=use_cache,
196
- pad_token_id=tokenizer.pad_token_id,
197
- eos_token_id=tokenizer.eos_token_id,
198
- early_stopping=early_stopping,
199
  )
200
- if no_repeat_ngram_size:
201
- generate_kwargs["no_repeat_ngram_size"] = no_repeat_ngram_size
202
- if repetition_penalty and repetition_penalty != 1.0:
203
- generate_kwargs["repetition_penalty"] = repetition_penalty
204
- if do_sample:
205
- generate_kwargs["temperature"] = temperature
206
- generate_kwargs["top_p"] = top_p
207
- generate_kwargs.update(gen_kwargs)
208
 
209
- t0 = time.perf_counter()
210
- with torch.inference_mode():
211
- out = model.generate(**inputs, **generate_kwargs)
212
- seconds = time.perf_counter() - t0
213
 
214
- new_tokens = out[:, prompt_tokens:]
215
- raw_candidates = tokenizer.batch_decode(new_tokens, skip_special_tokens=True)
216
- strategy = {
217
- "mode": mode,
218
- "num_candidates": num_candidates,
219
- "num_beams": max(1, num_beams),
220
- "do_sample": do_sample,
221
- "temperature": temperature if do_sample else 0.0,
222
- "no_repeat_ngram_size": no_repeat_ngram_size,
223
- "repetition_penalty": repetition_penalty,
224
- "max_new_tokens": max_new,
225
- "max_new_ratio": max_new_ratio,
226
- }
227
- return GenerationResult(
228
- candidates=[postprocess(c, text) for c in raw_candidates],
229
- raw_candidates=raw_candidates,
230
- prompt=prompt,
231
- prompt_tokens=prompt_tokens,
232
- max_new_tokens=max_new,
233
- seconds=seconds,
234
- strategy=strategy,
235
  )
236
 
237
 
238
- _MARKER_RE = re.compile(
239
- r"^\s*(?:#{0,6}\s*)?(?:###\s*markdown\s*:?\s*|##\s*answer\s*:?\s*|"
240
- r"conviert[ea] a markdown\s*:?\s*|convert to markdown\s*:?\s*|"
241
- r"transforma el siguiente texto[^\n]*|aqu[íi] tienes[^\n]*|"
242
- r"entrada\s*:|salida\s*:|\[your answer\])\s*$",
243
- re.IGNORECASE,
244
- )
245
-
246
-
247
- def _norm_ws(s: str) -> str:
248
- return re.sub(r"\s+", " ", s).strip()
249
-
250
-
251
- def _norm_cmp(s: str) -> str:
252
- s = s.lower()
253
- s = re.sub(r"[\u2010-\u2015\u2212]", "-", s)
254
- s = s.replace("\u00a0", " ")
255
- return _norm_ws(s)
256
-
257
-
258
- def _is_marker(line: str) -> bool:
259
- return bool(_MARKER_RE.match(line))
260
-
261
-
262
- def _strip_prompt_echo(text: str) -> str:
263
- lines = text.splitlines()
264
- while lines and (not lines[0].strip() or _is_marker(lines[0])):
265
- lines.pop(0)
266
- while lines and (not lines[-1].strip() or _is_marker(lines[-1])):
267
- lines.pop()
268
- for i, line in enumerate(lines):
269
- if i >= 1 and _is_marker(line) and len(_norm_ws(" ".join(lines[:i]))) >= 30:
270
- lines = lines[:i]
271
- break
272
- return "\n".join(lines).strip()
273
-
274
 
275
- def _unwrap_code_fence(text: str) -> str:
276
- lines = text.strip().splitlines()
277
- if len(lines) < 3:
278
- return text
279
- if not lines[0].strip().startswith("```") or lines[-1].strip() != "```":
280
- return text
281
- inner = lines[1:-1]
282
- if any(l.strip().startswith("```") for l in inner):
283
- return text
284
- return "\n".join(inner).strip()
285
 
 
 
 
286
 
287
- def _dedup_leading_h1(text: str) -> str:
288
- lines = text.strip().splitlines()
289
- if not lines or not re.match(r"^#\s+\S", lines[0].strip()):
290
- return text
291
- head = _norm_cmp(lines[0].lstrip("# "))
292
- rest = "\n".join(lines[1:])
293
- if head and head in _norm_cmp(rest):
294
- return rest.strip()
295
- return text
296
 
297
-
298
- def _fix_double_heading(text: str) -> str:
299
- out = []
300
- for line in text.splitlines():
301
- m = re.match(r"^\s*#{1,6}\s+(#{1,6}\s+.+?)\s*$", line)
302
- if m:
303
- line = m.group(1)
304
- out.append(line)
305
- return "\n".join(out)
306
-
307
-
308
- def _space_heading_markers(text: str) -> str:
309
- return re.sub(r"(?m)^(\s*#{1,6})(?=[^\s#])", r"\1 ", text)
310
-
311
-
312
- def _cut_repeated_tail_paragraphs(text: str, input_text: str) -> str:
313
- paras = re.split(r"\n\s*\n", text.strip())
314
- if len(paras) < 2:
315
- return text
316
- in_norm = _norm_cmp(input_text) if input_text else ""
317
- seen: dict[str, int] = {}
318
- for i, p in enumerate(paras):
319
- n = _norm_cmp(p)
320
- if len(n) < 40 or (in_norm and n in in_norm):
321
- continue
322
- key = n[:60]
323
- if key in seen:
324
- return "\n\n".join(paras[: seen[key]]).strip()
325
- seen[key] = i
326
- for i in range(max(1, len(paras) - 3), len(paras)):
327
- p = paras[i].strip()
328
- if p.startswith("#") and "\n" not in p:
329
- body = _norm_cmp(p.lstrip("# "))
330
- if len(body) >= 6:
331
- earlier = _norm_cmp("\n\n".join(paras[:i]))
332
- if body in earlier:
333
- return "\n\n".join(paras[:i]).strip()
334
- changed = True
335
- while changed and len(paras) >= 2:
336
- changed = False
337
- last = _norm_cmp(paras[-1])
338
- if len(last) < 40:
339
- break
340
- earlier = [_norm_cmp(p) for p in paras[:-1]]
341
- if last in earlier or any(last[:60] == e[:60] for e in earlier if len(e) >= 60):
342
- paras.pop()
343
- changed = True
344
- return "\n\n".join(paras).strip()
345
-
346
-
347
- def _cut_repeated_tail_ngram(text: str, input_text: str, min_n: int = 6) -> str:
348
- paras = re.split(r"\n\s*\n", text.strip())
349
- if len(paras) < 3:
350
- return text
351
- flat: list[str] = []
352
- owner: list[int] = []
353
- for pi, p in enumerate(paras):
354
- for w in _norm_cmp(p).split():
355
- flat.append(w)
356
- owner.append(pi)
357
- n = len(flat)
358
- best: tuple[int, int, int] | None = None
359
- for size in range(min(16, n // 2), min_n - 1, -1):
360
- seen: dict[tuple[str, ...], int] = {}
361
- for i in range(n - size + 1):
362
- gram = tuple(flat[i : i + size])
363
- if gram in seen:
364
- j = seen[gram]
365
- if owner[j] < owner[i] and owner[i] >= len(paras) - 3:
366
- if best is None or owner[i] < best[0] or (owner[i] == best[0] and owner[j] < best[1]):
367
- best = (owner[i], owner[j], size)
368
- else:
369
- seen[gram] = i
370
- if best is not None:
371
- break
372
- if best is None:
373
- return text
374
- s2, f, size = best
375
- in_norm = _norm_cmp(input_text) if input_text else ""
376
- for k in range(f + 1, s2):
377
- nk = _norm_cmp(paras[k])
378
- if len(nk) >= 30 and (not in_norm or nk not in in_norm):
379
- return "\n\n".join(paras[:k]).strip()
380
- if size >= 12:
381
- return "\n\n".join(paras[:s2]).strip()
382
- return text
383
-
384
-
385
- def _norm_line(line: str) -> str:
386
- return _norm_cmp(re.sub(r"^\s*#{1,6}\s*", "", line))
387
-
388
-
389
- def _cut_repeated_line_blocks(text: str) -> str:
390
- lines = text.rstrip().splitlines()
391
- changed = True
392
- while changed and len(lines) >= 2:
393
- changed = False
394
- n = len(lines)
395
- for k in range(n // 2, 0, -1):
396
- if lines[n - k :] != lines[n - 2 * k : n - k]:
397
- if [_norm_line(l) for l in lines[n - k :]] != [
398
- _norm_line(l) for l in lines[n - 2 * k : n - k]
399
- ]:
400
- continue
401
- block = [l for l in lines[n - k :] if l.strip()]
402
- if not block:
403
- continue
404
- if k == 1:
405
- line = lines[n - 1]
406
- long_enough = len(line.split()) >= 6
407
- if not long_enough or text.count(line.strip()) < 3:
408
- continue
409
- lines = lines[: n - k]
410
- changed = True
411
- break
412
- return "\n".join(lines).rstrip()
413
-
414
-
415
- def _content_coverage(text: str, input_words: set[str]) -> float:
416
- if not input_words:
417
- return 1.0
418
- words = set(_content_words(text))
419
- return len(input_words & words) / len(input_words)
420
-
421
-
422
- def _cut_after_completion(text: str, input_text: str, min_cov: float = 0.85) -> str:
423
- if not input_text:
424
- return text
425
- in_nums = extract_number_keys(input_text)
426
- if not in_nums:
427
- return text
428
- in_words = set(_content_words(input_text))
429
- lines = text.splitlines()
430
- seen: set[str] = set()
431
- for i, line in enumerate(lines):
432
- seen |= extract_number_keys(line)
433
- if in_nums <= seen:
434
- candidate = "\n".join(lines[: i + 1]).rstrip()
435
- if candidate and _content_coverage(candidate, in_words) >= min_cov:
436
- return candidate
437
- break
438
- return text
439
-
440
-
441
- def _cut_repeated_char_tail(text: str) -> str:
442
- t = text.rstrip()
443
- n = len(t)
444
- for p in range(24, min(800, n // 2) + 1):
445
- block = t[n - p :]
446
- if not block.strip():
447
- continue
448
- reps = 1
449
- while (reps + 1) * p <= n and t[n - (reps + 1) * p : n - reps * p] == block:
450
- reps += 1
451
- if reps >= 3:
452
- return t[: n - reps * p].rstrip()
453
- if reps >= 2 and p >= 120:
454
- return t[: n - reps * p].rstrip()
455
- return t
456
-
457
-
458
- def _cut_input_echo(text: str, input_text: str) -> str:
459
- if not input_text:
460
- return text
461
- in_lines = [l.strip() for l in input_text.strip().splitlines() if l.strip()]
462
- out_lines = text.rstrip().splitlines()
463
- for k in range(min(8, len(out_lines), len(in_lines)), 1, -1):
464
- tail = [l.strip() for l in out_lines[-k:]]
465
- if any(not l for l in tail):
466
- continue
467
- joined = _norm_ws(" ".join(tail))
468
- if len(joined.split()) < 8 and k < 2:
469
- continue
470
- for i in range(len(in_lines) - k + 1):
471
- if in_lines[i : i + k] == tail:
472
- return "\n".join(out_lines[: len(out_lines) - k]).rstrip()
473
- if out_lines:
474
- tail = _norm_ws(out_lines[-1])
475
- earlier = _norm_cmp("\n".join(out_lines[:-1]))
476
- if len(tail.split()) >= 8 and tail in _norm_cmp(input_text) and _norm_cmp(tail) in earlier:
477
- return "\n".join(out_lines[:-1]).rstrip()
478
- return text
479
-
480
-
481
- def _repair_headings(text: str, input_text: str) -> str:
482
- if not input_text:
483
- return text
484
- import difflib
485
-
486
- source_lines = [l.strip() for l in input_text.splitlines() if len(l.strip()) >= 8]
487
- if not source_lines:
488
- return text
489
- out_lines = text.splitlines()
490
- for i, line in enumerate(out_lines):
491
- m = re.match(r"^(\s*#{1,6}\s+)(.+?)\s*$", line)
492
- if not m:
493
- continue
494
- body = m.group(2).strip()
495
- if len(body) < 8:
496
- continue
497
- best_ratio, best_src = 0.0, None
498
- for src in source_lines:
499
- ratio = difflib.SequenceMatcher(None, body.lower(), src.lower()).ratio()
500
- if ratio > best_ratio:
501
- best_ratio, best_src = ratio, src
502
- if best_src and 0.88 <= best_ratio < 1.0:
503
- out_lines[i] = m.group(1) + best_src
504
- return "\n".join(out_lines)
505
-
506
-
507
- def postprocess(raw: str, input_text: str | None = None) -> str:
508
- text = raw.strip()
509
  if not text:
510
- return raw.strip()
511
- original = text
512
- text = _unwrap_code_fence(text)
513
- text = _strip_prompt_echo(text)
514
- text = _cut_input_echo(text, input_text or "")
515
- text = _dedup_leading_h1(text)
516
- text = _repair_headings(text, input_text or "")
517
- text = _fix_double_heading(text)
518
- text = _space_heading_markers(text)
519
- text = _cut_after_completion(text, input_text or "")
520
- text = _cut_repeated_tail_paragraphs(text, input_text or "")
521
- text = _cut_repeated_line_blocks(text)
522
- text = _cut_repeated_tail_ngram(text, input_text or "")
523
- text = _cut_repeated_char_tail(text)
524
- text = _strip_prompt_echo(text)
525
- text = "\n".join(l.rstrip() for l in text.splitlines()).strip()
526
- return text if text else original
527
-
528
-
529
- NUM_RE = re.compile(r"\d+(?:[.,]\d+)?%?")
530
- WORD_RE = re.compile(r"[a-zA-ZáéíóúüñçÁÉÍÓÚÜÑÇ]{3,}")
531
- LIST_RE = re.compile(r"^\s*(?:[-*•]|\d+[.)])\s+\S", re.M)
532
- TABLE_ROW_RE = re.compile(r"^\s*\|.*\|\s*$", re.M)
533
- FENCE_RE = re.compile(r"^\s*(```|~~~)", re.M)
534
-
535
- STOPWORDS = {
536
- "the", "and", "for", "are", "but", "not", "you", "your", "with", "that", "this",
537
- "from", "have", "has", "was", "were", "will", "would", "can", "could", "should",
538
- "into", "than", "then", "them", "they", "their", "there", "these", "those", "our",
539
- "its", "it's", "also", "some", "any", "all", "each", "when", "where", "which",
540
- "who", "what", "how", "why", "does", "did", "been", "being", "about", "over",
541
- "under", "after", "before", "between", "same", "more", "most", "other", "only",
542
- "los", "las", "una", "uno", "unos", "unas", "del", "para", "por", "con", "sin",
543
- "que", "como", "más", "mas", "pero", "sus", "este", "esta", "estos", "estas",
544
- "eso", "esa", "esos", "esas", "son", "sea", "fue", "han", "hay", "muy", "sin",
545
- "sobre", "entre", "cuando", "donde", "porque", "también", "tambien", "todos",
546
- "toda", "todo", "cada", "desde", "hasta", "según", "segun", "solo", "sólo",
547
- "nos", "les", "nuestro", "nuestra", "vuestro", "vuestra", "del", "al", "se",
548
- }
549
-
550
-
551
- def _num_key(token: str) -> str | None:
552
- t = token.rstrip("%").strip()
553
- if not t:
554
- return None
555
- has_c, has_d = "," in t, "." in t
556
- if has_c and has_d:
557
- if t.rfind(",") > t.rfind("."):
558
- t = t.replace(".", "").replace(",", ".")
559
- else:
560
- t = t.replace(",", "")
561
- elif has_c:
562
- parts = t.split(",")
563
- if len(parts[-1]) == 3 and len(parts) == 2 and len(parts[0]) <= 3:
564
- t = t.replace(",", "")
565
- else:
566
- t = t.replace(",", ".")
567
- elif has_d:
568
- parts = t.split(".")
569
- if len(parts) == 2 and len(parts[-1]) == 3 and len(parts[0]) <= 3:
570
- t = t.replace(".", "")
571
- try:
572
- return format(float(t), ".6g")
573
- except ValueError:
574
- return t
575
-
576
-
577
- def extract_number_keys(text: str) -> set[str]:
578
- keys = set()
579
- for m in NUM_RE.finditer(text):
580
- k = _num_key(m.group(0))
581
- if k is not None and k != "0":
582
- keys.add(k)
583
- return keys
584
-
585
-
586
- def _content_words(text: str) -> list[str]:
587
- words = [w.lower() for w in WORD_RE.findall(text)]
588
- return [w for w in words if w not in STOPWORDS]
589
-
590
-
591
- def _mk():
592
- from markdown_it import MarkdownIt
593
 
594
- return MarkdownIt("commonmark").enable("table").enable("strikethrough")
595
-
596
-
597
- _MD = None
598
-
599
-
600
- def _count_html_tokens(tokens) -> int:
601
- total = 0
602
- for t in tokens:
603
- if t.type in ("html_block", "html_inline"):
604
- total += 1
605
- if getattr(t, "children", None):
606
- total += _count_html_tokens(t.children)
607
- return total
608
-
609
-
610
- def gfm_stats(text: str) -> dict[str, Any]:
611
- global _MD
612
- if _MD is None:
613
- try:
614
- _MD = _mk()
615
- except Exception:
616
- return {"html_tokens": 0, "malformed_tables": 0, "malformed_links": 0, "fences_even": True, "gfm_clean": None}
617
  try:
618
- tokens = _MD.parse(text)
619
- except Exception:
620
- return {"html_tokens": 0, "malformed_tables": 1, "malformed_links": 0, "fences_even": True, "gfm_clean": 0.0}
621
- html_tokens = _count_html_tokens(tokens)
622
- tr_open = sum(1 for t in tokens if t.type == "tr_open")
623
- pipe_lines = TABLE_ROW_RE.findall(text)
624
- malformed = 0
625
- if pipe_lines and tr_open == 0:
626
- malformed = 1
627
- link_opens = len(re.findall(r"\]\(", text))
628
- valid_links = 0
629
- for link in re.finditer(r"\[[^\]]*\]\(([^)]*)\)", text):
630
- dest = link.group(1)
631
- if " " not in dest and "\n" not in dest:
632
- valid_links += 1
633
- malformed_links = max(0, link_opens - valid_links)
634
- if text.count("](") > valid_links and re.search(r"\]\([^)]*$", text, re.M):
635
- malformed_links = max(malformed_links, 1)
636
- fences = FENCE_RE.findall(text)
637
- fences_even = len(fences) % 2 == 0
638
- clean = 1.0
639
- if html_tokens:
640
- clean -= 0.25 * min(4, html_tokens)
641
- if malformed:
642
- clean -= 0.5
643
- if malformed_links:
644
- clean -= 0.25 * min(4, malformed_links)
645
- if not fences_even:
646
- clean -= 0.25
647
- return {
648
- "html_tokens": html_tokens,
649
- "malformed_tables": malformed,
650
- "malformed_links": malformed_links,
651
- "fences_even": fences_even,
652
- "gfm_clean": max(0.0, clean),
653
- }
654
-
655
-
656
- def _longest_repeated_ngram(text: str, max_n: int = 64) -> int:
657
- words = [_norm_cmp(w) for w in text.split()]
658
- n = len(words)
659
- for size in range(min(max_n, n // 2), 4, -1):
660
- seen = set()
661
- for i in range(n - size + 1):
662
- gram = tuple(words[i : i + size])
663
- if gram in seen:
664
- return size
665
- seen.add(gram)
666
- return 0
667
-
668
-
669
- def degeneracy_stats(text: str) -> dict[str, Any]:
670
- paras = [_norm_cmp(p) for p in re.split(r"\n\s*\n", text) if len(_norm_cmp(p)) > 80]
671
- dup_para = len(paras) > len(set(paras))
672
- longest = _longest_repeated_ngram(text)
673
- return {"longest_repeated_ngram": longest, "duplicated_paragraph": dup_para}
674
-
675
-
676
- def list_line_counts(text: str, kind: str = "generic") -> tuple[int, int]:
677
- in_list = len(LIST_RE.findall(text))
678
- if kind == "data":
679
- rows = [l for l in text.splitlines() if "," in l and l.strip()]
680
- if rows and "year" in rows[0].lower():
681
- rows = rows[1:]
682
- in_list += len(rows)
683
- return in_list, 0
684
-
685
-
686
- def _preface_ratio(input_text: str, output: str) -> float:
687
- in_words = set(_content_words(input_text))
688
- if not in_words:
689
- return 0.0
690
- head_words = _content_words("\n".join(output.strip().splitlines()[:3]))[:40]
691
- if len(head_words) < 8:
692
- return 0.0
693
- novel = sum(1 for w in head_words if w not in in_words)
694
- return novel / len(head_words)
695
-
696
-
697
- def score_text(input_text: str, output: str, must: list[str] | None = None, kind: str = "generic") -> dict[str, Any]:
698
- out_norm = _norm_cmp(output)
699
- in_numbers = extract_number_keys(input_text)
700
- out_numbers = extract_number_keys(output)
701
- matched = in_numbers & out_numbers
702
- number_fidelity = len(matched) / len(in_numbers) if in_numbers else 1.0
703
-
704
- in_words = _content_words(input_text)
705
- out_word_set = set(_content_words(output))
706
- if in_words:
707
- overlap = sum(1 for w in set(in_words) if w in out_word_set) / len(set(in_words))
708
- else:
709
- overlap = 1.0
710
-
711
- must = must or []
712
- must_hits = sum(1 for m in must if _norm_cmp(m) in out_norm)
713
- must_fidelity = must_hits / len(must) if must else None
714
-
715
- list_in, _ = list_line_counts(input_text, kind)
716
- list_out = len(LIST_RE.findall(output)) + len(TABLE_ROW_RE.findall(output))
717
- list_ratio = min(1.0, list_out / list_in) if list_in else 1.0
718
 
719
- gfm = gfm_stats(output)
720
- deg = degeneracy_stats(output)
721
- metrics = {
722
- "number_fidelity": round(number_fidelity, 4),
723
- "numbers_in": len(in_numbers),
724
- "numbers_matched": len(matched),
725
- "spurious_numbers": len(out_numbers - in_numbers),
726
- "content_overlap": round(overlap, 4),
727
- "must_fidelity": None if must_fidelity is None else round(must_fidelity, 4),
728
- "must_hits": must_hits,
729
- "must_total": len(must),
730
- "list_in": list_in,
731
- "list_out": list_out,
732
- "list_ratio": round(list_ratio, 4),
733
- "chars_in": len(input_text),
734
- "chars_out": len(output),
735
- "preface_ratio": round(_preface_ratio(input_text, output), 4),
736
- "degenerate": int(deg["duplicated_paragraph"] or deg["longest_repeated_ngram"] >= 12),
737
- **gfm,
738
- **deg,
739
  }
740
- metrics["quality"] = round(quality_score(metrics), 4)
741
- return metrics
742
-
743
-
744
- def quality_score(metrics: dict[str, Any]) -> float:
745
- score = 0.0
746
- score += 0.30 * (metrics.get("must_fidelity") or 0.0)
747
- score += 0.25 * metrics.get("number_fidelity", 0.0)
748
- score += 0.15 * metrics.get("content_overlap", 0.0)
749
- score += 0.10 * metrics.get("list_ratio", 0.0)
750
- clean = metrics.get("gfm_clean")
751
- score += 0.20 * (clean if clean is not None else 1.0)
752
- if metrics.get("degenerate"):
753
- score -= 0.15
754
- if metrics.get("html_tokens", 0):
755
- score -= 0.10
756
- if metrics.get("malformed_tables", 0):
757
- score -= 0.10
758
- if metrics.get("malformed_links", 0):
759
- score -= 0.05 * min(4, metrics["malformed_links"])
760
- if metrics.get("preface_ratio", 0.0) > 0.5:
761
- score -= 0.25
762
- if metrics.get("spurious_numbers", 0):
763
- score -= 0.05
764
- chars_in = metrics.get("chars_in", 0)
765
- chars_out = metrics.get("chars_out", 0)
766
- if chars_in and chars_out < 0.2 * chars_in:
767
- score -= 0.20
768
- return max(0.0, min(1.5, score))
769
-
770
-
771
- def rerank_candidates(input_text: str, candidates: list[str], must: list[str] | None = None, kind: str = "generic") -> tuple[int, list[dict]]:
772
- scored = [(i, score_text(input_text, c, must, kind)) for i, c in enumerate(candidates)]
773
- best = max(scored, key=lambda pair: pair[1]["quality"])[0] if scored else 0
774
- return best, [m for _, m in scored]
775
-
776
-
777
- class _HtmlToMarkdown(HTMLParser):
778
- def __init__(self):
779
- super().__init__(convert_charrefs=True)
780
- self.blocks: list[str] = []
781
- self.buf: list[str] = []
782
- self.heading: int | None = None
783
- self.list_stack: list[str] = []
784
- self.pending_list: list[str] = []
785
- self.pending_items: list[str] = []
786
- self.in_pre = False
787
- self.href: str | None = None
788
- self.link_pos: int | None = None
789
- self.table_rows: list[list[str]] | None = None
790
- self.row: list[str] | None = None
791
- self.cell_buf: list[str] | None = None
792
-
793
- def _text(self) -> str:
794
- t = "".join(self.buf)
795
- self.buf = []
796
- return t
797
-
798
- def _flush_para(self) -> None:
799
- t = _norm_ws(self._text())
800
- if t:
801
- self.blocks.append(t)
802
-
803
- def _flush_line(self) -> None:
804
- t = "".join(self.buf).strip()
805
- self.buf = []
806
- if t:
807
- self.pending_items.append(t)
808
-
809
- def handle_data(self, data: str) -> None:
810
- if self.cell_buf is not None:
811
- self.cell_buf.append(data)
812
- else:
813
- self.buf.append(data)
814
-
815
- def handle_starttag(self, tag: str, attrs) -> None:
816
- a = dict(attrs)
817
- if tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
818
- self._flush_para()
819
- self.heading = int(tag[1])
820
- elif tag in ("p", "div", "section", "article", "header", "footer", "main", "aside"):
821
- self._flush_para()
822
- elif tag == "br":
823
- self.buf.append("\n")
824
- elif tag in ("strong", "b"):
825
- self.buf.append("**")
826
- elif tag in ("em", "i"):
827
- self.buf.append("*")
828
- elif tag == "code" and not self.in_pre:
829
- self.buf.append("`")
830
- elif tag == "pre":
831
- self._flush_para()
832
- self.in_pre = True
833
- self.buf.append("```\n")
834
- elif tag == "a":
835
- self.href = a.get("href")
836
- self.link_pos = len(self.buf)
837
- self.buf.append("")
838
- elif tag in ("ul", "ol"):
839
- self._flush_para()
840
- self.list_stack.append(tag)
841
- elif tag == "li":
842
- self._flush_line()
843
- depth = max(0, len(self.list_stack) - 1)
844
- ordered = self.list_stack and self.list_stack[-1] == "ol"
845
- self.buf.append(" " * depth + ("1. " if ordered else "- "))
846
- elif tag == "table":
847
- self._flush_para()
848
- self.table_rows = []
849
- elif tag == "tr":
850
- self.row = []
851
- elif tag in ("td", "th"):
852
- self.cell_buf = []
853
-
854
- def handle_endtag(self, tag: str) -> None:
855
- if tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
856
- t = _norm_ws(self._text())
857
- if t:
858
- level = self.heading or 1
859
- self.blocks.append("#" * level + " " + t)
860
- self.heading = None
861
- elif tag in ("p", "div", "section", "article", "header", "footer", "main", "aside"):
862
- self._flush_para()
863
- elif tag in ("strong", "b"):
864
- self.buf.append("**")
865
- elif tag in ("em", "i"):
866
- self.buf.append("*")
867
- elif tag == "code" and not self.in_pre:
868
- self.buf.append("`")
869
- elif tag == "pre":
870
- self.in_pre = False
871
- t = "".join(self.buf)
872
- self.buf = []
873
- self.blocks.append("```\n" + t.strip("\n") + "\n```")
874
- elif tag == "a":
875
- if self.link_pos is not None:
876
- text = "".join(self.buf[self.link_pos :])
877
- del self.buf[self.link_pos :]
878
- if self.href:
879
- self.buf.append(f"[{text}]({self.href})")
880
- else:
881
- self.buf.append(text)
882
- self.link_pos = None
883
- self.href = None
884
- elif tag in ("ul", "ol"):
885
- if self.list_stack:
886
- self.list_stack.pop()
887
- if not self.list_stack and self.pending_items:
888
- self.blocks.append("\n".join(self.pending_items))
889
- self.pending_items = []
890
- elif tag == "li":
891
- self._flush_line()
892
- elif tag in ("td", "th"):
893
- if self.row is not None and self.cell_buf is not None:
894
- self.row.append(_norm_ws("".join(self.cell_buf)))
895
- self.cell_buf = None
896
- elif tag == "tr":
897
- if self.table_rows is not None and self.row is not None:
898
- self.table_rows.append(self.row)
899
- self.row = None
900
- elif tag == "table":
901
- self._emit_table()
902
- self.table_rows = None
903
-
904
- def _emit_table(self) -> None:
905
- rows = [r for r in (self.table_rows or []) if any(c for c in r)]
906
- if not rows:
907
- return
908
- width = max(len(r) for r in rows)
909
- header = (rows[0] + [""] * width)[:width]
910
- lines = ["| " + " | ".join(header) + " |", "| " + " | ".join(["---"] * width) + " |"]
911
- for r in rows[1:]:
912
- cells = (r + [""] * width)[:width]
913
- lines.append("| " + " | ".join(cells) + " |")
914
- self.blocks.append("\n".join(lines))
915
-
916
- def finish(self) -> str:
917
- if self.in_pre:
918
- self.handle_endtag("pre")
919
- if self.pending_items:
920
- self.blocks.append("\n".join(self.pending_items))
921
- self.pending_items = []
922
- self._flush_para()
923
- md = "\n\n".join(b for b in self.blocks if b.strip())
924
- return re.sub(r"\n{3,}", "\n\n", md).strip()
925
-
926
-
927
- def looks_like_html(text: str) -> bool:
928
- return bool(
929
- re.search(r"</?(?:p|div|h[1-6]|ul|ol|li|table|tr|td|th|br|strong|em|a|span|img|section|article)\b[^>]*>", text, re.I)
930
- )
931
-
932
-
933
- def html_to_markdown(text: str) -> str:
934
- parser = _HtmlToMarkdown()
935
- parser.feed(text)
936
- parser.close()
937
- return parser.finish()
938
-
939
-
940
- def _split_cells(line: str, delim: str) -> list[str]:
941
- if delim == ",":
942
- import csv as _csv
943
-
944
- return [c.strip() for c in next(_csv.reader([line]))]
945
- if delim == "\t":
946
- return [c.strip() for c in line.split("\t")]
947
- if delim == "|":
948
- return [c.strip() for c in line.strip().strip("|").split("|")]
949
- return [c.strip() for c in re.split(r"\s{2,}", line.strip())]
950
-
951
-
952
- def _find_tabular_block(text: str):
953
- lines = [l for l in text.splitlines() if l.strip()]
954
- if len(lines) < 3:
955
- return None
956
- for delim in (",", "\t", "|", " "):
957
- for start in range(0, min(2, len(lines))):
958
- seg = lines[start:]
959
- if len(seg) < 3:
960
- continue
961
- counts = [len(_split_cells(l, delim)) for l in seg]
962
- if min(counts) >= 2 and max(counts) == min(counts):
963
- return delim, lines[:start], seg
964
- return None
965
-
966
-
967
- def delimited_to_markdown_table(text: str) -> str | None:
968
- found = _find_tabular_block(text)
969
- if not found:
970
- return None
971
- delim, pre, seg = found
972
- rows = [_split_cells(l, delim) for l in seg]
973
- width = len(rows[0])
974
- if width < 2 or any(len(r) != width for r in rows):
975
- return None
976
- def esc(cells: list[str]) -> list[str]:
977
- return [c.replace("|", "\\|") for c in cells]
978
- out: list[str] = []
979
- if pre:
980
- out.append("\n".join(pre).strip())
981
- out.append("| " + " | ".join(esc(rows[0])) + " |")
982
- out.append("| " + " | ".join(["---"] * width) + " |")
983
- for r in rows[1:]:
984
- out.append("| " + " | ".join(esc(r)) + " |")
985
- return "\n".join(out)
986
-
987
-
988
- def infer_kind(text: str) -> str:
989
- if looks_like_html(text):
990
- return "html"
991
- if _find_tabular_block(text):
992
- return "data"
993
- return "generic"
994
 
 
 
 
 
 
 
 
 
 
 
995
 
996
- def structured_candidates(text: str) -> list[str]:
997
- cands: list[str] = []
998
- if looks_like_html(text):
999
- try:
1000
- cands.append(html_to_markdown(text))
1001
- except Exception:
1002
- pass
1003
- table = delimited_to_markdown_table(text)
1004
- if table:
1005
- cands.append(table)
1006
- return cands
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1007
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1008
 
1009
- def convert_to_markdown(
1010
- model,
1011
- tokenizer,
1012
- text: str,
1013
- *,
1014
- must: list[str] | None = None,
1015
- kind: str = "generic",
1016
- structured_fallback: bool = True,
1017
- skip_model_for_structured: bool = False,
1018
- **gen_kwargs: Any,
1019
- ) -> dict[str, Any]:
1020
- if kind == "generic":
1021
- kind = infer_kind(text)
1022
- if skip_model_for_structured and structured_fallback and kind in ("html", "data"):
1023
- candidates = [postprocess(c, text) for c in structured_candidates(text)]
1024
- if not candidates:
1025
- candidates = [postprocess(text, text)]
1026
- best, metrics = rerank_candidates(text, candidates, must, kind)
1027
- return {
1028
- "markdown": candidates[best],
1029
- "candidate_index": best,
1030
- "fallback_used": True,
1031
- "metrics": metrics[best],
1032
- "candidate_metrics": metrics,
1033
- "candidates": candidates,
1034
- "raw_candidates": [],
1035
- "prompt_tokens": 0,
1036
- "max_new_tokens": 0,
1037
- "seconds": 0.0,
1038
- "strategy": {"mode": "structured", "best_index": best, "structured_fallback": True},
1039
- }
1040
- result = generate_candidates(model, tokenizer, text, **gen_kwargs)
1041
- candidates = list(result.candidates)
1042
- fallback_start = len(candidates)
1043
- if structured_fallback:
1044
- for extra in structured_candidates(text):
1045
- candidates.append(postprocess(extra, text))
1046
- best, metrics = rerank_candidates(text, candidates, must, kind)
1047
- chosen = candidates[best]
1048
- if not chosen.strip():
1049
- chosen = text.strip()
1050
- metrics[best]["quality"] = 0.0
1051
- result.strategy["best_index"] = best
1052
- result.strategy["structured_fallback"] = structured_fallback
1053
- return {
1054
- "markdown": chosen,
1055
- "candidate_index": best,
1056
- "fallback_used": best >= fallback_start,
1057
- "metrics": metrics[best],
1058
- "candidate_metrics": metrics,
1059
- "candidates": candidates,
1060
- "raw_candidates": result.raw_candidates,
1061
- "prompt_tokens": result.prompt_tokens,
1062
- "max_new_tokens": result.max_new_tokens,
1063
- "seconds": result.seconds,
1064
- "strategy": result.strategy,
1065
- }
1066
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1067
 
1068
- def load_model(model_id: str = MODEL_ID, device: str | None = None, local_files_only: bool = True):
1069
- from transformers import AutoModelForCausalLM, AutoTokenizer
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1070
 
1071
- tokenizer = AutoTokenizer.from_pretrained(model_id, local_files_only=local_files_only)
1072
- model = AutoModelForCausalLM.from_pretrained(model_id, dtype="auto", local_files_only=local_files_only)
1073
- model.eval()
1074
- if device:
1075
- model.to(device)
1076
- return model, tokenizer
1077
 
 
 
 
 
 
 
1078
 
1079
- def main(argv: list[str] | None = None) -> None:
1080
- parser = argparse.ArgumentParser(description="PASITA text-to-Markdown CLI")
1081
- parser.add_argument("--text-file", required=True)
1082
- parser.add_argument("--model", default=MODEL_ID)
1083
- parser.add_argument(
1084
- "--mode",
1085
- default="canonical",
1086
- choices=[
1087
- "canonical",
1088
- "english",
1089
- "spanish_instr",
1090
- "fewshot",
1091
- "fewshot2",
1092
- "plain",
1093
- "md_label",
1094
- "respuesta",
1095
- "output",
1096
- "sep_only",
1097
- "doc_instr_es",
1098
- "doc_instr_en",
1099
- "csv_instr",
1100
- "html_instr",
1101
- "html_instr_en",
1102
- ],
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1103
  )
1104
- parser.add_argument("--instruction", default=None)
1105
- parser.add_argument("--candidates", type=int, default=3)
1106
- parser.add_argument("--beams", type=int, default=1)
1107
- parser.add_argument("--no-repeat-ngram", type=int, default=0)
1108
- parser.add_argument("--repetition-penalty", type=float, default=1.0)
1109
- parser.add_argument("--temperature", type=float, default=0.3)
1110
- parser.add_argument("--min-new", type=int, default=0)
1111
- parser.add_argument("--max-new", default="auto")
1112
- parser.add_argument("--max-new-ratio", type=float, default=1.5)
1113
- parser.add_argument("--hard-cap", type=int, default=DEFAULT_HARD_CAP)
1114
- parser.add_argument("--seed", type=int, default=0)
1115
- parser.add_argument("--device", default=None)
1116
- parser.add_argument("--no-structured-fallback", action="store_true")
1117
- parser.add_argument("--show-candidates", action="store_true")
1118
- args = parser.parse_args(argv)
1119
-
1120
- with open(args.text_file, encoding="utf-8") as fh:
1121
- text = fh.read()
1122
-
1123
- model, tokenizer = load_model(args.model, args.device)
1124
- max_new: Any = args.max_new
1125
- if args.max_new != "auto":
1126
- max_new = int(args.max_new)
1127
- gen_kwargs: dict[str, Any] = dict(
1128
- mode=args.mode,
1129
- instruction=args.instruction,
1130
- num_candidates=max(1, args.candidates),
1131
- num_beams=max(1, args.beams),
1132
- do_sample=args.temperature > 0,
1133
- temperature=args.temperature if args.temperature > 0 else 1.0,
1134
- no_repeat_ngram_size=args.no_repeat_ngram,
1135
- repetition_penalty=args.repetition_penalty,
1136
- max_new_tokens=max_new,
1137
- max_new_ratio=args.max_new_ratio,
1138
- hard_cap=args.hard_cap,
1139
- seed=args.seed,
1140
  )
1141
- if args.min_new > 0:
1142
- gen_kwargs["min_new_tokens"] = args.min_new
1143
- result = convert_to_markdown(
1144
- model,
1145
- tokenizer,
1146
- text,
1147
- structured_fallback=not args.no_structured_fallback,
1148
- **gen_kwargs,
1149
  )
1150
- if args.show_candidates:
1151
- for i, (cand, met) in enumerate(zip(result["candidates"], result["candidate_metrics"])):
1152
- marker = " *" if i == result["candidate_index"] else ""
1153
- print(f"----- candidate {i}{marker} quality={met['quality']} -----")
1154
- print(cand)
1155
- print()
1156
- print(result["markdown"])
1157
-
1158
 
1159
- if __name__ == "__main__":
1160
- main()
 
1
+ import html
 
 
 
 
 
 
 
 
 
 
2
  import time
 
 
 
3
 
4
+ import spaces
5
  import torch
6
+ import gradio as gr
7
+ from transformers import AutoModelForCausalLM, AutoTokenizer
8
 
9
+ from pasita import infer_kind, convert_to_markdown
 
 
 
10
 
11
+ MODEL_ID = "OpceanAI/PASITA"
12
+ MODEL_CARD = "https://huggingface.co/OpceanAI/PASITA"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
13
 
14
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
15
+ tokenizer.padding_side = "left"
16
+ model = AutoModelForCausalLM.from_pretrained(MODEL_ID, dtype="auto").to("cuda")
17
+ model.eval()
18
 
19
+ EMPTY_MD = "_The converted Markdown appears here. Paste a document on the left and press Convert._"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
20
 
21
+ GLYPH = (
22
+ '<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" '
23
+ 'stroke-linecap="round" aria-hidden="true">'
24
+ '<path d="M9 4 7 20M17 4l-2 16M4 9h16M3 15h16"/></svg>'
25
+ )
 
 
 
 
 
 
26
 
 
 
 
 
27
 
28
+ def _status(text: str, state: str = "ready", detail: str = "") -> str:
29
+ detail_html = f'<span class="p-detail">{html.escape(detail)}</span>' if detail else ""
30
+ return (
31
+ f'<div class="p-status {state}"><span class="p-dot"></span>'
32
+ f'<span class="p-text">{html.escape(text)}</span>{detail_html}</div>'
 
 
 
 
33
  )
 
 
 
 
 
 
 
 
34
 
 
 
 
 
35
 
36
+ def _counter(text: str) -> str:
37
+ text = text or ""
38
+ words = len(text.split())
39
+ return (
40
+ '<span class="p-label">PLAIN TEXT</span>'
41
+ f'<span class="p-count">{len(text):,} CHARS / {words:,} WORDS</span>'
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
  )
43
 
44
 
45
+ def _estimate_duration(text, num_candidates=3, structured_fallback=True, *args, **kwargs):
46
+ if structured_fallback and infer_kind((text or "").strip()) in ("html", "data"):
47
+ return 10
48
+ return min(30, max(12, 8 + len(text or "") // 200 + 4 * (int(num_candidates) - 1)))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
49
 
 
 
 
 
 
 
 
 
 
 
50
 
51
+ @spaces.GPU(duration=_estimate_duration)
52
+ def convert(text: str, num_candidates: int = 3, structured_fallback: bool = True) -> tuple[str, str, str]:
53
+ """Convert a plain-text document into faithful GitHub-Flavored Markdown.
54
 
55
+ PASITA samples the document several times, every candidate is post-processed and
56
+ scored for number fidelity, content coverage and GFM validity, and the best one is
57
+ returned. Inputs detected as HTML or delimited data are converted deterministically.
 
 
 
 
 
 
58
 
59
+ Args:
60
+ text: The document to convert, for example OCR output, pasted HTML, meeting notes or a report.
61
+ num_candidates: How many PASITA samples to generate and rerank, from 1 to 3.
62
+ structured_fallback: Use the deterministic converter for HTML and delimited data.
63
+ Returns:
64
+ A tuple of the rendered Markdown, the raw Markdown and a status line.
65
+ """
66
+ text = (text or "").strip()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
67
  if not text:
68
+ return EMPTY_MD, "", _status("READY / PASTE A DOCUMENT TO CONVERT")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
69
 
70
+ started = time.perf_counter()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
71
  try:
72
+ result = convert_to_markdown(
73
+ model,
74
+ tokenizer,
75
+ text,
76
+ num_candidates=max(1, min(3, int(num_candidates))),
77
+ do_sample=True,
78
+ temperature=0.3,
79
+ top_p=0.95,
80
+ seed=0,
81
+ use_cache=True,
82
+ structured_fallback=bool(structured_fallback),
83
+ skip_model_for_structured=True,
84
+ )
85
+ except Exception as exc:
86
+ message = html.escape(f"{type(exc).__name__}: {exc}")
87
+ return (
88
+ f"> Conversion failed.\n>\n> `{message}`",
89
+ "",
90
+ _status(f"ERROR / {type(exc).__name__}", "error", f"{time.perf_counter() - started:.1f}S"),
91
+ )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92
 
93
+ metrics = result["metrics"]
94
+ numbers = f"{metrics['numbers_matched']}/{metrics['numbers_in']} NUMBERS"
95
+ overlap = f"{metrics['content_overlap']:.0%} CONTENT"
96
+ routes = {
97
+ "model": "PASITA + RERANK",
98
+ "structured": "HTML/TABLE CONVERTER",
99
+ "passthrough": "SOURCE PASSTHROUGH",
 
 
 
 
 
 
 
 
 
 
 
 
 
100
  }
101
+ source_label = routes.get(result.get("route"), "PASITA + RERANK")
102
+ taken = result["seconds"] if result["seconds"] else time.perf_counter() - started
103
+ detail = f"{source_label} / {numbers} / {overlap} / {taken:.1f}S"
104
+ return result["markdown"], result["markdown"], _status("DONE", "ready", detail)
105
+
106
+
107
+ def reset() -> tuple[str, str, str, str]:
108
+ return EMPTY_MD, "", _status("READY"), _counter("")
109
+
110
+
111
+ BUSY_JS = """(text, num_candidates, structured_fallback, md, source, status) => {
112
+ const n = Number(num_candidates) || 1;
113
+ const label = n > 1 ? 'PASITA ' + n + ' SAMPLES / RERANK' : 'PASITA SAMPLE / RERANK';
114
+ return [md, source,
115
+ '<div class="p-status busy"><span class="p-dot"></span><span class="p-text">CONVERTING</span><span class="p-detail">' + label + '</span></div>'];
116
+ }"""
117
+
118
+ COUNTER_JS = """(text) => {
119
+ const t = (text || '').trim();
120
+ const words = t ? t.split(/\\s+/).length : 0;
121
+ const chars = (text || '').length;
122
+ return '<span class="p-label">PLAIN TEXT</span><span class="p-count">' + chars.toLocaleString() + ' CHARS / ' + words.toLocaleString() + ' WORDS</span>';
123
+ }"""
124
+
125
+
126
+ HEADER = f"""
127
+ <header class="p-head">
128
+ <div class="p-brand">
129
+ <span class="p-glyph">{GLYPH}</span>
130
+ <span class="p-name">PASITA</span>
131
+ <span class="p-tag">/ 88M FROM SCRATCH</span>
132
+ </div>
133
+ <div class="p-actions">
134
+ <a class="p-btn" href="{MODEL_CARD}" target="_blank" rel="noopener">MODEL CARD</a>
135
+ </div>
136
+ </header>
137
+ """
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
138
 
139
+ FOOTER = f"""
140
+ <footer class="p-foot">
141
+ <span>PASITA v1 converts plain text to Markdown, in Spanish and English. It works best on medium and long documents.</span>
142
+ <span>
143
+ <a href="{MODEL_CARD}" target="_blank" rel="noopener">MODEL</a> /
144
+ <a href="?view=api" target="_blank" rel="noopener">API</a> /
145
+ RUNS ON ZEROGPU
146
+ </span>
147
+ </footer>
148
+ """
149
 
150
+ EXAMPLES = [
151
+ [
152
+ """RESUMEN MENSUAL DE OPERACIONES: MARZO 2026
153
+
154
+ Resumen ejecutivo
155
+ Los pedidos gestionados alcanzaron 18.420 unidades, un 6,1% más que en febrero. El tiempo medio de preparación bajó a 2,4 horas.
156
+
157
+ Detalle por almacén
158
+ Madrid: 8.120 pedidos (+7,4%)
159
+ Barcelona: 5.480 pedidos (+3,9%)
160
+ Lisboa: 4.820 pedidos (+6,7%)
161
+
162
+ Incidencias
163
+ Se registraron 96 incidencias de embalaje, el 0,52% del total. La previsión para abril es de 19.100 pedidos."""
164
+ ],
165
+ [
166
+ """Team sync, 2026-08-14
167
+ Attendees: Ana, Marc, Priya, Tom
168
+
169
+ Decisions
170
+ 1. Ship the new onboarding flow on Sept 2.
171
+ 2. Postpone the pricing experiment to Q4.
172
+ 3. Marc owns the migration checklist; due Aug 22.
173
+
174
+ Open questions
175
+ - Do we need a second staging cluster?
176
+ - Who signs off on the SOC2 evidence?"""
177
+ ],
178
+ [
179
+ """<h2>Shipping policy</h2><p>Orders placed before 2:00 PM CET ship the same business day.</p><ul><li>Standard delivery: 3 to 5 business days</li><li>Express delivery: 1 business day</li><li>Free shipping over 49 EUR</li></ul><p>Returns are accepted within 30 days with the original packaging.</p>"""
180
+ ],
181
+ [
182
+ """year,region,revenue,growth
183
+ 2019,North America,5120.4,4.1%
184
+ 2019,Europe,4380.9,3.7%
185
+ 2019,Latin America,2105.2,2.9%
186
+ 2020,North America,4875.6,-4.8%
187
+ 2020,Europe,4210.3,-3.9%
188
+ 2020,Latin America,1988.1,-5.6%
189
+ 2021,North America,5402.7,10.8%
190
+ 2021,Europe,4605.5,9.4%
191
+ 2021,Latin America,2240.8,12.7%
192
+ 2022,North America,5610.2,3.8%
193
+ 2022,Europe,4720.6,2.5%
194
+ 2022,Latin America,2385.4,6.5%
195
+ 2023,North America,5894.9,5.1%
196
+ 2023,Europe,4903.7,3.9%
197
+ 2023,Latin America,2560.3,7.3%"""
198
+ ],
199
+ ]
200
+
201
+ CSS = """
202
+ @import url('https://fonts.googleapis.com/css2?family=Hanken+Grotesk:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap');
203
+
204
+ :root, .gradio-container {
205
+ --bg:#050608; --ink:#f4f5f7; --muted:#9aa0ab; --muted-2:#7f8698;
206
+ --warm:#ff7a30; --cool:#7aa0ff; --obsidian:#0a0a0c;
207
+ --hair:rgba(255,255,255,0.14); --hair-soft:rgba(255,255,255,0.08);
208
+ --panel:rgba(255,255,255,0.03); --panel-hover:rgba(255,255,255,0.06);
209
+ --danger:#ff7a6b;
210
+ --font-display:"Hanken Grotesk", system-ui, sans-serif;
211
+ --font-mono:"JetBrains Mono", ui-monospace, monospace;
212
+
213
+ --body-background-fill:transparent;
214
+ --background-fill-primary:transparent;
215
+ --background-fill-secondary:transparent;
216
+ --block-background-fill:transparent;
217
+ --block-border-color:transparent;
218
+ --block-border-width:0px;
219
+ --block-radius:0px;
220
+ --border-color-primary:var(--hair);
221
+ --body-text-color:var(--ink);
222
+ --body-text-color-subdued:var(--muted);
223
+ --input-background-fill:transparent;
224
+ --input-border-color:transparent;
225
+ --input-radius:12px;
226
+ --color-accent:var(--warm);
227
+ --color-accent-soft:rgba(255,122,48,0.15);
228
+ --button-primary-background-fill:var(--warm);
229
+ --button-primary-background-fill-hover:#ff9757;
230
+ --button-primary-text-color:#0b0705;
231
+ --button-secondary-background-fill:transparent;
232
+ --button-secondary-text-color:var(--muted);
233
+ --button-secondary-border-color:var(--hair);
234
+ --button-large-radius:100px;
235
+ --button-small-radius:100px;
236
+ --shadow-drop:none; --shadow-drop-lg:none;
237
+ --font:var(--font-display); --font-mono:var(--font-mono);
238
+ }
239
 
240
+ body, .gradio-container {
241
+ background:
242
+ radial-gradient(120% 80% at 50% 118%, rgba(255,110,40,0.10), rgba(255,110,40,0) 55%),
243
+ radial-gradient(90% 70% at 88% -10%, rgba(80,120,230,0.10), rgba(80,120,230,0) 55%),
244
+ radial-gradient(140% 120% at 50% 50%, #0a0c11 0%, #050608 55%, #030305 100%) !important;
245
+ background-attachment: fixed !important;
246
+ color: var(--ink);
247
+ font-family: var(--font-display);
248
+ -webkit-font-smoothing: antialiased;
249
+ }
250
+ .gradio-container { max-width: 1240px !important; margin: 0 auto; padding: 0 20px 26px; }
251
+ footer[aria-label="Gradio footer navigation"] { display: none !important; }
252
+ :focus-visible { outline: 2px solid var(--cool) !important; outline-offset: 2px; }
253
 
254
+ .p-head {
255
+ display: flex; align-items: center; justify-content: space-between; gap: 16px;
256
+ padding: 22px 0 16px; border-bottom: 1px solid var(--hair-soft); margin-bottom: 20px;
257
+ }
258
+ .p-brand { display: flex; align-items: center; gap: 10px; }
259
+ .p-glyph { display: inline-flex; width: 22px; height: 22px; color: var(--warm); }
260
+ .p-glyph svg { width: 100%; height: 100%; }
261
+ .p-name { font-weight: 700; letter-spacing: 0.14em; font-size: 1.02rem; }
262
+ .p-tag { font-family: var(--font-mono); font-size: 0.62rem; letter-spacing: 0.14em; color: var(--muted-2); margin-top: 2px; }
263
+ .p-actions { display: flex; align-items: center; gap: 10px; }
264
+ .p-btn {
265
+ font-family: var(--font-mono); font-size: 0.72rem; letter-spacing: 0.1em;
266
+ color: var(--ink); text-decoration: none; padding: 0 18px; min-height: 44px;
267
+ display: inline-flex; align-items: center; border-radius: 999px;
268
+ background: rgba(255,255,255,0.04); border: 1px solid var(--hair);
269
+ transition: background .25s ease, color .25s ease, border-color .25s ease;
270
+ }
271
+ .p-btn:hover { background: var(--ink); color: #08090c; border-color: var(--ink); }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
272
 
273
+ .p-status {
274
+ display: flex; align-items: center; gap: 9px; margin: -6px 0 16px;
275
+ font-family: var(--font-mono); font-size: 11px; letter-spacing: 0.08em;
276
+ text-transform: uppercase; color: var(--muted); min-height: 20px;
277
+ }
278
+ .p-dot { width: 7px; height: 7px; border-radius: 50%; background: rgba(255,255,255,0.2); flex-shrink: 0; }
279
+ .p-status.ready .p-dot { background: var(--warm); box-shadow: 0 0 10px rgba(255,138,77,0.55); }
280
+ .p-status.busy .p-dot { background: #ffcd6b; animation: p-pulse 1.1s ease-in-out infinite; }
281
+ .p-status.error { color: var(--danger); }
282
+ .p-status.error .p-dot { background: var(--danger); box-shadow: 0 0 10px rgba(255,122,107,0.5); }
283
+ .p-detail { margin-left: auto; color: var(--muted-2); }
284
+ @keyframes p-pulse { 0%,100% { opacity: 1; } 50% { opacity: 0.3; } }
285
+
286
+ #p-shell { gap: 18px; align-items: stretch; }
287
+ .p-panel {
288
+ background: var(--panel); border: 1px solid var(--hair); border-radius: 16px;
289
+ padding: 14px 14px 12px; min-width: 0;
290
+ }
291
+ .p-label { font-family: var(--font-mono); font-size: 0.66rem; letter-spacing: 0.14em; text-transform: uppercase; color: var(--muted); }
292
+ .p-count { float: right; font-family: var(--font-mono); font-size: 0.66rem; letter-spacing: 0.04em; color: var(--muted-2); }
293
+
294
+ #p-text, #p-text > div { background: transparent !important; border: 0 !important; box-shadow: none !important; }
295
+ #p-text textarea {
296
+ background: transparent !important; border: 0 !important; color: var(--ink) !important;
297
+ font-family: var(--font-display) !important; font-size: 0.95rem; line-height: 1.65;
298
+ min-height: 360px; padding: 12px 2px 0;
299
+ }
300
+ #p-text textarea::placeholder { color: var(--muted-2) !important; }
301
+
302
+ #p-run, #p-clear { font-family: var(--font-mono) !important; font-size: 0.75rem !important; letter-spacing: 0.12em; min-height: 46px; }
303
+ #p-run { flex: 2; }
304
+ #p-clear { flex: 1; background: transparent !important; color: var(--muted) !important; border: 1px solid var(--hair) !important; }
305
+ #p-clear:hover { color: var(--ink) !important; border-color: rgba(255,255,255,0.3) !important; }
306
+
307
+ #p-examples .label { font-family: var(--font-mono) !important; font-size: 0.66rem !important; letter-spacing: 0.14em; text-transform: uppercase; color: var(--muted) !important; }
308
+ #p-examples .label svg { display: none; }
309
+ #p-examples .gallery-item {
310
+ background: var(--panel) !important; border: 1px solid var(--hair) !important; border-radius: 999px !important;
311
+ color: var(--muted) !important; font-size: 12px !important; min-height: 44px; padding: 8px 14px !important;
312
+ text-align: left; transition: background .2s ease, color .2s ease, border-color .2s ease;
313
+ }
314
+ #p-examples .gallery-item:hover { border-color: rgba(255,255,255,0.3) !important; color: var(--ink) !important; background: var(--panel-hover) !important; }
315
 
316
+ #p-tabs .tab-container[role="tablist"] {
317
+ display: inline-flex; gap: 4px; background: var(--panel); border: 1px solid var(--hair);
318
+ border-radius: 100px; padding: 3px; margin-bottom: 12px;
319
+ }
320
+ #p-tabs .tab-container[role="tablist"] button {
321
+ appearance: none; border: 0 !important; background: transparent !important; border-radius: 100px !important;
322
+ padding: 7px 14px !important; font-family: var(--font-mono) !important; font-size: 10.5px !important;
323
+ letter-spacing: 0.1em; text-transform: uppercase; color: var(--muted) !important;
324
+ }
325
+ #p-tabs .tab-container[role="tablist"] button.selected,
326
+ #p-tabs .tab-container[role="tablist"] button[aria-selected="true"] {
327
+ background: rgba(255,255,255,0.08) !important; color: var(--warm) !important;
328
+ }
329
+ #p-tabs .tab-container::after,
330
+ #p-tabs .tab-container button.selected::after { content: none !important; }
331
+
332
+ #p-rendered { min-height: 360px; font-size: 0.95rem; line-height: 1.72; color: var(--ink); }
333
+ #p-rendered .prose { font-size: 0.95rem; line-height: 1.72; }
334
+ #p-rendered h1, #p-rendered h2, #p-rendered h3, #p-rendered h4 { color: #fff; font-weight: 600; line-height: 1.25; margin: 1.25em 0 0.55em; }
335
+ #p-rendered h1 { font-size: 1.5em; }
336
+ #p-rendered h2 { font-size: 1.28em; }
337
+ #p-rendered h3 { font-size: 1.12em; }
338
+ #p-rendered p { margin: 0 0 0.9em; }
339
+ #p-rendered ul, #p-rendered ol { margin: 0 0 0.9em; padding-left: 1.5em; }
340
+ #p-rendered li { margin: 0.25em 0; }
341
+ #p-rendered li::marker { color: var(--muted-2); }
342
+ #p-rendered a { color: #8ab4ff; text-decoration: underline; text-underline-offset: 2px; }
343
+ #p-rendered a:hover { color: #aecbff; }
344
+ #p-rendered blockquote { margin: 0 0 0.9em; padding: 2px 0 2px 16px; border-left: 2px solid var(--hair); color: var(--muted); }
345
+ #p-rendered hr { border: 0; border-top: 1px solid var(--hair); margin: 1.3em 0; }
346
+ #p-rendered code { font-family: var(--font-mono); font-size: 0.85em; background: var(--panel); border: 1px solid var(--hair); border-radius: 5px; padding: 1px 5px; }
347
+ #p-rendered pre { margin: 0 0 0.9em; padding: 14px 16px; background: var(--obsidian); border: 1px solid var(--hair); border-radius: 12px; overflow-x: auto; }
348
+ #p-rendered pre code { background: none; border: 0; padding: 0; font-size: 0.82em; line-height: 1.6; }
349
+ #p-rendered table { border-collapse: collapse; margin: 0 0 0.9em; font-size: 0.92em; display: block; overflow-x: auto; }
350
+ #p-rendered th, #p-rendered td { border: 1px solid var(--hair); padding: 6px 11px; text-align: left; }
351
+ #p-rendered th { background: var(--panel); color: var(--ink); font-weight: 600; }
352
+
353
+ #p-source {
354
+ background: var(--obsidian) !important; border: 1px solid var(--hair-soft) !important;
355
+ border-radius: 12px !important; font-family: var(--font-mono) !important; min-height: 360px;
356
+ }
357
+ #p-source label[data-testid="block-label"] { display: none; }
358
+ #p-source .cm-editor { background: transparent !important; }
359
+ #p-source .cm-content, #p-source .cm-scroller { color: var(--ink) !important; caret-color: var(--warm) !important; }
360
+ #p-source .cm-gutters { background: transparent !important; color: var(--muted-2) !important; border-right: 1px solid var(--hair-soft) !important; }
361
+
362
+ #p-settings { background: transparent !important; border: 1px solid var(--hair-soft) !important; border-radius: 12px !important; margin-top: 4px; }
363
+ #p-settings > .label-wrap { font-family: var(--font-mono) !important; font-size: 0.68rem !important; letter-spacing: 0.12em; text-transform: uppercase; color: var(--muted) !important; }
364
+ #p-settings > .label-wrap .icon { color: var(--muted-2) !important; }
365
+ #p-settings .p-label { margin-bottom: 4px; }
366
+
367
+ .p-foot {
368
+ display: flex; flex-wrap: wrap; justify-content: space-between; gap: 14px;
369
+ margin-top: 22px; padding-top: 16px; border-top: 1px solid var(--hair-soft);
370
+ font-family: var(--font-mono); font-size: 0.68rem; letter-spacing: 0.05em; line-height: 1.7; color: var(--muted-2);
371
+ }
372
+ .p-foot a { color: var(--muted); text-decoration: none; }
373
+ .p-foot a:hover { color: var(--warm); }
374
+
375
+ @media (max-width: 820px) {
376
+ .gradio-container { padding: 0 14px 20px; }
377
+ #p-shell { flex-direction: column !important; }
378
+ .p-tag { display: none; }
379
+ #p-text textarea { min-height: 220px; }
380
+ #p-rendered, #p-source { min-height: 240px; }
381
+ .p-status { flex-wrap: wrap; }
382
+ .p-detail { margin-left: 0; width: 100%; }
383
+ .p-foot { flex-direction: column; gap: 8px; }
384
+ }
385
 
386
+ @media (prefers-reduced-motion: reduce) {
387
+ *, *::before, *::after { animation: none !important; transition: none !important; }
388
+ }
389
+ """
 
 
390
 
391
+ theme = gr.themes.Base(
392
+ primary_hue=gr.themes.colors.orange,
393
+ neutral_hue=gr.themes.colors.gray,
394
+ font=[gr.themes.GoogleFont("Hanken Grotesk"), "system-ui", "sans-serif"],
395
+ font_mono=[gr.themes.GoogleFont("JetBrains Mono"), "ui-monospace", "monospace"],
396
+ )
397
 
398
+ with gr.Blocks(title="PASITA: plain text to Markdown") as demo:
399
+ gr.HTML(HEADER)
400
+ status = gr.HTML(_status("READY"), elem_id="p-status")
401
+
402
+ with gr.Row(elem_id="p-shell"):
403
+ with gr.Column(scale=1, min_width=320, elem_classes=["p-panel"]):
404
+ label_in = gr.HTML(_counter(""))
405
+ text_in = gr.Textbox(
406
+ show_label=False,
407
+ container=False,
408
+ placeholder="Paste OCR output, pasted HTML, meeting notes, a report or an article... (SHIFT + ENTER to convert)",
409
+ lines=18,
410
+ max_lines=32,
411
+ elem_id="p-text",
412
+ )
413
+ with gr.Accordion("SETTINGS", open=False, elem_id="p-settings"):
414
+ candidates = gr.Slider(
415
+ minimum=1, maximum=3, value=3, step=1,
416
+ label="PASITA CANDIDATES",
417
+ info="More samples improve the reranker. 3 is the measured default.",
418
+ )
419
+ structured = gr.Checkbox(
420
+ value=True,
421
+ label="DETERMINISTIC CONVERTER FOR HTML AND TABLES",
422
+ info="PASITA does not handle raw HTML or data dumps at this size.",
423
+ )
424
+ with gr.Row():
425
+ run = gr.Button("CONVERT TO MARKDOWN", variant="primary", elem_id="p-run")
426
+ clear = gr.Button("CLEAR", elem_id="p-clear")
427
+ with gr.Column(scale=1, min_width=320, elem_classes=["p-panel"]):
428
+ with gr.Tabs(elem_id="p-tabs"):
429
+ with gr.Tab("RENDERED", elem_id="p-tab-rendered"):
430
+ md_out = gr.Markdown(EMPTY_MD, buttons=["copy"], height=520, elem_id="p-rendered")
431
+ with gr.Tab("SOURCE", elem_id="p-tab-source"):
432
+ source = gr.Code(
433
+ value="", language="markdown", buttons=["copy", "download"],
434
+ wrap_lines=True, lines=24, max_lines=24, elem_id="p-source",
435
+ )
436
+
437
+ gr.Examples(
438
+ examples=EXAMPLES,
439
+ example_labels=["ES REPORT", "EN MEETING", "HTML", "CSV"],
440
+ inputs=[text_in],
441
+ outputs=[md_out, source, status],
442
+ fn=convert,
443
+ cache_examples=True,
444
+ cache_mode="lazy",
445
+ examples_per_page=4,
446
+ label="EXAMPLES",
447
+ elem_id="p-examples",
448
  )
449
+ gr.HTML(FOOTER)
450
+
451
+ run.click(
452
+ convert,
453
+ inputs=[text_in, candidates, structured],
454
+ outputs=[md_out, source, status],
455
+ api_name="convert",
456
+ js=BUSY_JS,
457
+ show_progress="minimal",
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
458
  )
459
+ text_in.submit(
460
+ convert,
461
+ inputs=[text_in, candidates, structured],
462
+ outputs=[md_out, source, status],
463
+ api_visibility="undocumented",
464
+ js=BUSY_JS,
465
+ show_progress="minimal",
 
466
  )
467
+ text_in.change(None, inputs=[text_in], outputs=[label_in], js=COUNTER_JS, api_visibility="undocumented")
468
+ clear.click(reset, inputs=None, outputs=[md_out, source, status, label_in], api_visibility="undocumented")
 
 
 
 
 
 
469
 
470
+ demo.queue()
471
+ demo.launch(theme=theme, css=CSS, mcp_server=True)