File size: 4,131 Bytes
b296ad4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 | """Generate new multilingual slot-preserving templates, with complete provenance."""
import argparse
import concurrent.futures
import json
import re
import time
import urllib.request
from pathlib import Path
from tinyquery.recipes import RECIPES, LANGUAGES
def slots(s):
return set(re.findall(r'\{([a-z_]+)\}', s))
def generate(op, seed, base, count):
source = RECIPES[op]
required = slots(source)
prompt = (
'Create diverse natural user requests that mean EXACTLY the same thing as the source. '
'Return a JSON object with keys en, noisy_en, hi, hinglish; each value is a list of '
f'{count} different strings. en: natural English, noisy_en: imperfect English that '
'still preserves every constraint, hi: natural Devanagari Hindi, hinglish: Romanized Hindi. '
'Preserve ALL literal {placeholder} slots exactly, including braces. Do not add slots. '
'Do not replace slots with example values. Do not add or remove conditions, columns, '
'sorting, limits, negation, or inclusivity. Vary sentence structure and tone. '
'Do not provide SQL or explanations. Source: ' + source
)
payload = {'model': 'Qwen/Qwen3.8-27B-FP8', 'messages': [{'role':'user','content':prompt}],
'temperature':0.8,'max_tokens':4096,'seed':seed,
'chat_template_kwargs':{'enable_thinking':False},
'response_format':{'type':'json_object'}}
request = urllib.request.Request(base+'/chat/completions', data=json.dumps(payload).encode(),
headers={'Content-Type':'application/json'})
started = time.time()
with urllib.request.urlopen(request, timeout=240) as response:
result = json.load(response)
raw = result['choices'][0]['message']['content']
parsed = json.loads(raw)
accepted = {}; rejected = []
for lang in LANGUAGES:
accepted[lang] = []
for text in parsed.get(lang, []):
if not isinstance(text,str) or slots(text) != required or len(text)>900:
rejected.append({'language':lang,'text':text,'reason':'slot_or_length'})
continue
if lang == 'hi' and not re.search('[\u0900-\u097f]', text):
rejected.append({'language':lang,'text':text,'reason':'missing_devanagari'})
continue
if text not in accepted[lang]: accepted[lang].append(text)
return {'op':op,'source':source,'templates':accepted,'rejected':rejected,
'teacher':payload['model'],'revision':'017b9c7af6b5689d5dd426a76e0bc077eb5ca20a',
'request':payload,'raw_response':raw,'usage':result.get('usage'),
'elapsed_seconds':time.time()-started,'finish_reason':result['choices'][0]['finish_reason']}
def main():
parser=argparse.ArgumentParser()
parser.add_argument('--out',default='data/tinyquery/templates.jsonl')
parser.add_argument('--base',default='http://127.0.0.1:18000/v1')
parser.add_argument('--count',type=int,default=12)
parser.add_argument('--workers',type=int,default=16)
args=parser.parse_args()
path=Path(args.out); path.parent.mkdir(parents=True,exist_ok=True)
done=set()
if path.exists():
done={json.loads(line)['op'] for line in path.read_text().splitlines()}
started=time.time()
with path.open('a') as output, concurrent.futures.ThreadPoolExecutor(args.workers) as pool:
jobs={pool.submit(generate,op,900+i,args.base,args.count):op
for i,op in enumerate(RECIPES) if op not in done}
for future in concurrent.futures.as_completed(jobs):
op=jobs[future]
try:
result=future.result()
output.write(json.dumps(result,ensure_ascii=False)+'\n'); output.flush()
print(json.dumps({'op':op,'accepted':{k:len(v) for k,v in result['templates'].items()},
'seconds':round(time.time()-started,1)}),flush=True)
except Exception as exc:
print(json.dumps({'op':op,'error':str(exc)}),flush=True)
if __name__ == '__main__': main()
|