"""Synthetic multilingual opinionated texts for sentiment training, Qwen3.8-27B via vLLM (Apache-2.0 outputs). Each prompt asks for 10 texts in one language, genre and topic: 3 clearly positive, 3 clearly negative, 2 neutral, 2 hard (mixed, sarcastic, subtle or negated). Labels come later from the teacher (sentiment/teacher_sent.py), not the generator. No benchmark items are shown to the generator. N=... SHARD=i OUT=dir python sentiment/gen_sent.py (vLLM venv) -> $OUT/gen_.jsonl {text, lang, genre, topic, intended} LONG=1: 4 long texts per prompt (120-350 words: reviews, posts, letters), 1 positive, 1 negative, 1 neutral, 1 hard - the v0 student trailed the teacher most on long reviews (e.g. mteb_cym, translated IMDB) because most texts were short. """ import json, os, random, re, sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from common import hard_exit LANGS = (["English"] * 6 + ["German", "French", "Spanish", "Portuguese (Brazilian)", "Italian", "Arabic (Modern Standard)", "Egyptian Arabic", "Hindi", "Hinglish (Hindi in Latin letters mixed with English)", "Chinese (Simplified)", "Chinese (Traditional)", "Japanese", "Korean", "Russian"] * 2 + ["Dutch", "Polish", "Ukrainian", "Czech", "Slovak", "Bulgarian", "Croatian", "Serbian", "Slovenian", "Romanian", "Hungarian", "Greek", "Turkish", "Hebrew", "Persian", "Urdu", "Bengali", "Tamil", "Telugu", "Marathi", "Vietnamese", "Thai", "Indonesian", "Malay", "Filipino (Tagalog)", "Swahili", "Finnish", "Swedish", "Norwegian", "Danish", "Welsh", "Basque", "Catalan", "Maltese", "Lithuanian", "Latvian", "Estonian", "Uyghur", "Algerian Arabic (Darja)", "Moroccan Arabic (Darija)", "European Portuguese", "Latin American Spanish", "Amharic", "Hausa", "Yoruba", "Nepali", "Punjabi", "Gujarati", "Kazakh", "Azerbaijani", "Georgian", "Armenian", "Albanian", "Icelandic", "Irish"]) GENRES = ["product review on a shopping site", "restaurant or cafe review", "hotel or travel review", "movie, series or book review", "app store review", "short social media post", "reply in a social media thread", "comment under a news article", "customer support message or complaint", "chat message to a friend", "forum post", "short email to a company", "answer in a customer satisfaction survey", "comment about a sports match", "opinion about a politician or policy", "comment on a video or song", "message to a landlord, school or employer", "post about a personal life event"] TOPICS = ["electronics", "clothing and shoes", "food and groceries", "a local business", "public transport", "a bank or insurance", "a phone or internet provider", "healthcare and doctors", "a game", "music", "a celebrity", "the weather", "work and colleagues", "school or university", "family", "a football club", "a city or country", "cars", "housing and rent", "a delivery service", "software and websites", "cosmetics", "furniture", "a concert or event", "pets", "prices and the economy", "a TV show", "an airline", "a government office", "a new law", "a hobby"] def parse_json(t): t = re.sub(r"^```(?:json)?|```$", "", t.strip(), flags=re.M).strip() s, e = t.find("{"), t.rfind("}") try: return json.loads(t[s: e + 1]) except Exception: return None def prompt(lang, genre, topic, length): return f"""Write 10 different realistic texts in {lang}. Each is a {genre} about {topic}, written by different ordinary people. - 3 clearly POSITIVE, 3 clearly NEGATIVE, 2 NEUTRAL (factual, a plain question or statement without emotion), 2 HARD cases (mixed feelings, sarcasm, subtle or negated sentiment, or polite criticism). - Length: {length}. Vary tone, vocabulary and style as real people do (slang, emojis, typos or informal grammar where natural for this kind of text). Write naturally in {lang}, as a native speaker; do not translate from English. - Do not mention these instructions. Return only JSON: {{"items": [{{"text": "...", "type": "positive|negative|neutral|hard"}}, ...]}}""" def prompt_long(lang, genre, topic, _): return f"""Write 4 different realistic LONG texts in {lang}, each 120-350 words. Each is a {genre} about {topic}, written by different ordinary people with their own details (names, places, numbers, what happened). - 1 clearly POSITIVE, 1 clearly NEGATIVE, 1 NEUTRAL (informational, a description or question without emotion), 1 HARD case (mixed feelings that balance out, sarcasm, or criticism that turns positive at the end or the reverse). - Write naturally in {lang}, as a native speaker; do not translate from English. Do not mention these instructions. Return only JSON: {{"items": [{{"text": "...", "type": "positive|negative|neutral|hard"}}, ...]}}""" if __name__ == "__main__": from vllm import LLM, SamplingParams n, shard, out = int(os.environ.get("N", 100)), int(os.environ.get("SHARD", 0)), os.environ["OUT"] rng = random.Random(1300 + shard) LEN = ["a few words to one sentence", "one or two sentences", "two to four sentences", "a short paragraph (3-6 sentences)"] jobs = [(l, rng.choice(GENRES), rng.choice(TOPICS), rng.choice(LEN)) for l in LANGS for _ in range(n)] llm = LLM(os.environ.get("MODEL", "Qwen/Qwen3.8-27B-FP8"), max_model_len=6144, gpu_memory_utilization=0.92, max_num_seqs=512, limit_mm_per_prompt={"image": 0, "video": 0}) long = os.environ.get("LONG") == "1" outs = llm.chat([[{"role": "user", "content": (prompt_long if long else prompt)(*j)}] for j in jobs], SamplingParams(temperature=0.95, top_p=0.95, max_tokens=3500 if long else 2500, seed=31 + shard), chat_template_kwargs={"enable_thinking": False}) os.makedirs(out, exist_ok=True) k = 0 with open(f"{out}/gen_{'long_' if long else ''}{shard}.jsonl", "w") as f: for (l, g, t, _), o in zip(jobs, outs): j = parse_json(o.outputs[0].text) for it in (j or {}).get("items", []) if isinstance(j, dict) else []: if isinstance(it, dict) and isinstance(it.get("text"), str) and 3 <= len(it["text"]) <= (4000 if long else 1500): f.write(json.dumps(dict(text=it["text"].strip(), lang=l, genre=g, topic=t, intended=str(it.get("type", ""))), ensure_ascii=False) + "\n"); k += 1 print("items", k, "from", len(jobs), "prompts", flush=True) hard_exit(0)