File size: 2,557 Bytes
1d26c54 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | """Document rewriting with the desktop generation policy and incremental reports."""
import copy
import sys
import threading
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent / 'engine'))
import documents, generation, geometry, rewrite_policy
def transform(engine, text, g, style, options, output, cancel=None):
cancel = cancel or threading.Event()
locations = documents.spans(text, limit=700)
records = {}
report = dict(policy=rewrite_policy.VERSION, identity=engine.identity,
input=text, geometry=g, style=style, sampling=options,
spans=locations, records=records, complete=False)
output = Path(output)
def save():
report['text'] = documents.assemble(text, locations, records)
geometry.atomic(output, report)
output.with_suffix('.txt').write_text(report['text'], encoding='utf-8')
if output.suffix.lower() != '.json':
raise ValueError('The report output must end in .json; rewritten text uses the matching .txt path.')
save()
for index, (a, b) in enumerate(locations):
if cancel.is_set():
break
engine.progress(f'Rewriting passage {index + 1}/{len(locations)}')
limit = documents.rewrite_allowance(text, locations, records, index, multiplier=1.25)
target = geometry.point(g, (a + b) / (2 * len(text)))
instruction = rewrite_policy.prompt(style) + (
f'\nAim for at most {b-a} characters; finish before the hard limit of '
f'{limit} characters, including spaces and punctuation.')
opts = copy.deepcopy(options)
opts['max_tokens'] = min(2048, max(opts['max_tokens'], limit))
def generate(adjusted, attempt):
return engine.generate(
text[a:b], g=adjusted, coordinates=target, system=instruction,
max_chars=limit, soft_chars=max(1, int((b-a)*.85)),
cancel=cancel, **opts)
try:
result = generation.rewrite(text[a:b], generate, g, True, cancel,
engine.progress, max_chars=limit)
except InterruptedError:
break
records[str(index)] = dict(result, target=target, character_allowance=limit)
save()
report['complete'] = len(records) == len(locations)
report['failed'] = sum(bool(r.get('original_retained')) for r in records.values())
report['needs_review'] = sum(bool(r.get('warning') or r.get('error')) for r in records.values())
save()
return report
|