"""Hand-labelled routing test set: about 50 English and 50 German questions. Labels follow the app's CLASSIFY prompt. Where the prompt leaves room, a field lists every answer it allows, so a model is only marked wrong for a reading the prompt rules out. Per item: q the question hist prior turns (user, assistant, ...), optional search needs_search cx allowed complexity values shape allowed answer_shape values ("" is prose) img wants_image, or None when either is fine rec "none" (must be empty), "any" (anything), "set" (must be set), "hard" (day/week/month), "nothard" (empty or year) lens_in corpora that must be among `lenses` (searched questions only) lens_out corpora that must not be keys groups of terms; every group needs one hit in the queries (lowercase substring), searched questions only sq groups of terms the standalone question must contain (follow-ups) place accepted `place` substrings for weather (any one) python make_testset.py # writes testset.jsonl """ import json import os S, ST, R = "simple", "standard", "research" ANYCX = [S, ST, R] def q(text, lang, search=True, cx=ANYCX, shape=("",), img=False, rec="any", lens_in=(), lens_out=(), keys=(), sq=(), place="", hist=(), tag=""): return {"q": text, "lang": lang, "search": search, "cx": list(cx), "shape": list(shape), "img": img, "rec": rec, "lens_in": list(lens_in), "lens_out": list(lens_out), "keys": [list(k) for k in keys], "sq": [list(k) for k in sq], "place": list(place), "hist": list(hist), "tag": tag} NOSHAPE_OR_EXPL = ("", "explainer") EN, DE = "en", "de" ITEMS = [ # ------------------------------------------------------------ no search q("thanks, that helped a lot!", EN, search=False, img=False, tag="smalltalk"), q("hi there", EN, search=False, tag="smalltalk"), q("Translate into German: 'The meeting has been moved to Thursday afternoon.'", EN, search=False, tag="transform"), q("Rewrite this to sound more polite: send me the report by tomorrow or else.", EN, search=False, tag="transform"), q("What is 17% of 2,340?", EN, search=False, tag="arithmetic"), q("Why does this Python print 3 instead of 4?\n\nx = [1, 2, 3]\nx.append(4)\nprint(len(x[:-1]))", EN, search=False, tag="code"), q("Summarise this in one sentence: Our Q3 revenue rose 12% on stronger subscription sales, " "while hardware margins fell because of higher component costs.", EN, search=False, tag="transform"), q("danke dir, super erklärt", DE, search=False, tag="smalltalk"), q("Guten Morgen!", DE, search=False, tag="smalltalk"), q("Übersetze ins Englische: Ich komme heute etwas später, der Zug hat Verspätung.", DE, search=False, tag="transform"), q("Formuliere das freundlicher: Ihre Rechnung ist seit drei Wochen überfällig, zahlen Sie endlich.", DE, search=False, tag="transform"), q("Was ist 15 % von 480 Euro?", DE, search=False, tag="arithmetic"), q("Was macht diese SQL-Abfrage? SELECT name, COUNT(*) FROM orders GROUP BY name HAVING COUNT(*) > 5", DE, search=False, tag="code"), q("Kannst du deine letzte Antwort kürzer zusammenfassen?", DE, search=False, tag="conversation", hist=[{"role": "user", "content": "Wie funktioniert eine Wärmepumpe?"}, {"role": "assistant", "content": "Eine Wärmepumpe entzieht der Umgebung Wärme und hebt sie " "mit einem Verdichter auf ein höheres Temperaturniveau, um damit zu heizen."}]), # ------------------------------------------------------------ lookups q("What is the Bitcoin price right now?", EN, cx=[S], shape=["figure"], rec="set", lens_out=["scholar"], keys=[["bitcoin", "btc"]], tag="figure"), q("How many people live in Tokyo?", EN, cx=[S], shape=["figure", ""], rec="nothard", lens_out=["scholar"], keys=[["tokyo"]], tag="figure"), q("When did the Berlin Wall fall?", EN, cx=[S], shape=[""], rec="none", lens_out=["scholar", "news"], keys=[["berlin wall"]], tag="fact"), q("What's the latest stable version of Python?", EN, cx=[S], shape=["", "figure"], rec="set", lens_out=["scholar"], keys=[["python"]], tag="version"), q("weather in Hamburg tomorrow", EN, cx=[S], shape=["weather"], place=("hamburg",), lens_out=["scholar"], keys=[["hamburg"]], tag="weather"), q("Will it rain in Seattle this weekend?", EN, cx=[S], shape=["weather"], place=("seattle",), lens_out=["scholar"], keys=[["seattle"]], tag="weather"), q("Who won the last Champions League final?", EN, cx=[S], shape=["match", ""], rec="nothard", lens_out=["scholar"], keys=[["champions league"]], tag="match"), q("Wie viel kostet ein Liter Diesel aktuell in Deutschland?", DE, cx=[S], shape=["figure"], rec="set", lens_out=["scholar"], keys=[["diesel"]], tag="figure"), q("Wie hoch ist der Leitzins der EZB?", DE, cx=[S], shape=["figure"], rec="set", lens_out=["scholar"], keys=[["leitzins", "ezb", "ecb", "interest rate"]], tag="figure"), q("Wann wurde Goethe geboren?", DE, cx=[S], shape=[""], rec="none", lens_out=["scholar", "news"], keys=[["goethe"]], tag="fact"), q("Wetter in München morgen", DE, cx=[S], shape=["weather"], place=("münchen","munich"), lens_out=["scholar"], keys=[["münchen", "munich"]], tag="weather"), q("Wie wird das Wetter am Wochenende in Köln?", DE, cx=[S], shape=["weather"], place=("köln","cologne"), lens_out=["scholar"], keys=[["köln", "cologne"]], tag="weather"), q("Wie hat Bayern gegen Dortmund gespielt?", DE, cx=[S], shape=["match"], rec="set", lens_out=["scholar"], keys=[["bayern"], ["dortmund", "bvb"]], tag="match"), q("Welche Version hat Node.js LTS gerade?", DE, cx=[S], shape=["", "figure"], rec="set", lens_out=["scholar"], keys=[["node"]], tag="version"), # ------------------------------------------------------------ profile / image q("Who is Jensen Huang?", EN, cx=[S, ST], shape=["profile"], img=True, rec="nothard", lens_out=["scholar"], keys=[["jensen huang"]], tag="profile"), q("What is the company Anduril?", EN, cx=[S, ST], shape=["profile"], img=None, rec="nothard", lens_out=["scholar"], keys=[["anduril"]], tag="profile"), q("what is Obsidian, the notes app?", EN, cx=[S, ST], shape=["profile"], img=None, rec="nothard", lens_out=["scholar"], keys=[["obsidian"]], tag="profile"), q("What does an okapi look like?", EN, cx=[S], shape=["image", "profile"], img=True, rec="none", lens_out=["news"], keys=[["okapi"]], tag="image"), q("show me the flag of Bhutan", EN, cx=[S], shape=["image"], img=True, rec="none", lens_out=["scholar", "news"], keys=[["bhutan"]], tag="image"), q("Wer ist Annalena Baerbock?", DE, cx=[S, ST], shape=["profile"], img=True, rec="nothard", lens_out=["scholar"], keys=[["baerbock"]], tag="profile"), q("Was ist die Firma Celonis?", DE, cx=[S, ST], shape=["profile"], img=None, rec="nothard", lens_out=["scholar"], keys=[["celonis"]], tag="profile"), q("Wer war Hedy Lamarr?", DE, cx=[S, ST], shape=["profile"], img=True, rec="none", lens_out=["scholar", "news"], keys=[["lamarr"]], tag="profile"), q("Wie sieht ein Axolotl aus?", DE, cx=[S], shape=["image", "profile"], img=True, rec="none", lens_out=["news"], keys=[["axolotl"]], tag="image"), q("Zeig mir ein Bild vom Kölner Dom", DE, cx=[S], shape=["image"], img=True, rec="none", lens_out=["scholar", "news"], keys=[["dom", "cathedral"]], tag="image"), q("Wie sieht es mit der Rente mit 63 aus, gibt es die noch?", DE, cx=[S, ST], shape=[""], img=False, rec="set", lens_out=["scholar"], keys=[["rente"]], tag="idiom"), # ------------------------------------------------------------ explainer q("How do transformers work in machine learning?", EN, cx=[ST, S], shape=["explainer"], rec="none", keys=[["transformer"]], tag="explainer"), q("Why is the sky blue?", EN, cx=[S, ST], shape=["explainer"], rec="none", lens_out=["news"], keys=[["sky", "rayleigh"]], tag="explainer"), q("Explain how public key cryptography works", EN, cx=[S, ST], shape=["explainer"], rec="none", lens_out=["news"], keys=[["public key", "public-key", "asymmetric"]], tag="explainer"), q("What does impedance mean in electronics?", EN, cx=[S, ST], shape=["explainer"], rec="none", lens_out=["news"], keys=[["impedance"]], tag="explainer"), q("Wie funktionieren Autoencoder?", DE, cx=[S, ST], shape=["explainer"], rec="none", lens_out=["news"], keys=[["autoencoder"]], tag="explainer"), q("Warum wird Brot hart, wenn es alt wird?", DE, cx=[S, ST], shape=["explainer"], rec="none", lens_out=["news"], keys=[["brot", "bread"]], tag="explainer"), q("Was bedeutet Inflation eigentlich genau?", DE, cx=[S, ST], shape=["explainer"], rec="none", lens_out=["news"], keys=[["inflation"]], tag="explainer"), q("Wie funktioniert eine Wärmepumpe?", DE, cx=[S, ST], shape=["explainer", "diagram"], rec="none", lens_out=["news"], keys=[["wärmepumpe", "heat pump"]], tag="explainer"), # ------------------------------------------------------------ steps q("How do I install Docker on Ubuntu 24.04?", EN, cx=[S, ST], shape=["steps"], rec="nothard", lens_out=["scholar", "news"], keys=[["docker"], ["ubuntu"]], tag="steps"), q("How do I renew a US passport by mail?", EN, cx=[S, ST], shape=["steps"], rec="nothard", lens_out=["scholar"], keys=[["passport"]], tag="steps"), q("How do I set up SSH keys for GitHub?", EN, cx=[S, ST], shape=["steps"], rec="nothard", lens_out=["scholar", "news"], keys=[["ssh"], ["github"]], tag="steps"), q("Wie beantrage ich einen neuen Personalausweis?", DE, cx=[S, ST], shape=["steps"], rec="nothard", lens_out=["scholar"], keys=[["personalausweis"]], tag="steps"), q("Wie melde ich ein Gewerbe an?", DE, cx=[S, ST], shape=["steps"], rec="nothard", lens_out=["scholar"], keys=[["gewerbe"]], tag="steps"), q("Wie richte ich einen Cronjob unter Linux ein?", DE, cx=[S, ST], shape=["steps"], rec="nothard", lens_out=["scholar", "news"], keys=[["cron"]], tag="steps"), # ------------------------------------------------------------ compare -> research q("PostgreSQL vs MySQL for a small web app?", EN, cx=[R], shape=["comparison"], rec="nothard", lens_out=["scholar", "news"], keys=[["postgres"], ["mysql"]], tag="comparison"), q("Should I buy an iPhone 17 or a Pixel 10?", EN, cx=[R], shape=["comparison"], img=None, rec="nothard", lens_out=["scholar"], keys=[["iphone"], ["pixel"]], tag="comparison"), q("Compare Rust, Go and Zig for systems programming", EN, cx=[R], shape=["table"], rec="nothard", lens_out=["news"], keys=[["rust"], ["go", "golang"], ["zig"]], tag="table"), q("ETF oder Festgeld, was ist 2026 sinnvoller?", DE, cx=[R], shape=["comparison"], rec="set", lens_out=["scholar"], keys=[["etf"], ["festgeld"]], tag="comparison"), q("Vergleiche Tesla Model 3, VW ID.3 und BYD Dolphin", DE, cx=[R], shape=["table"], img=None, rec="nothard", lens_out=["scholar"], keys=[["model 3", "tesla"], ["id.3", "id3"], ["dolphin", "byd"]], tag="table"), q("Was ist besser für Einsteiger, Fuji oder Sony Kameras?", DE, cx=[R], shape=["comparison"], img=None, rec="nothard", lens_out=["scholar"], keys=[["fuji"], ["sony"]], tag="comparison"), # ------------------------------------------------------------ evidence -> scholar q("Does creatine actually help with muscle growth?", EN, cx=[ST, R], shape=["", "explainer"], rec="nothard", lens_in=["scholar"], keys=[["creatine"]], tag="evidence"), q("What does the research say about intermittent fasting and weight loss?", EN, cx=[R], shape=[""], rec="nothard", lens_in=["scholar"], keys=[["fasting"]], tag="evidence"), q("Is aspartame harmful?", EN, cx=[ST, R], shape=[""], rec="nothard", lens_in=["scholar"], keys=[["aspartame"]], tag="evidence"), q("how much vitamin D should an adult take per day", EN, cx=[S, ST], shape=["", "figure"], rec="nothard", lens_in=["scholar"], keys=[["vitamin d"]], tag="evidence"), q("Bringt Kreatin was?", DE, cx=[ST, R], shape=["", "explainer"], rec="nothard", lens_in=["scholar"], keys=[["creatine", "kreatin"]], tag="evidence"), q("Was sagt die Studienlage zu Melatonin bei Schlafstörungen?", DE, cx=[R], shape=[""], rec="nothard", lens_in=["scholar"], keys=[["melatonin"]], tag="evidence"), q("Ist Süßstoff schädlich?", DE, cx=[ST, R], shape=[""], rec="nothard", lens_in=["scholar"], keys=[["sweetener", "süßstoff", "aspartame", "sucralose"]], tag="evidence"), q("Warum wirkt Koffein bei mir nicht mehr?", DE, cx=[S, ST], shape=["explainer", ""], rec="nothard", lens_in=["scholar"], keys=[["caffeine", "koffein"]], tag="evidence"), # ------------------------------------------------------------ docs / API q("How do I paginate results with the GitHub REST API?", EN, cx=[S, ST], shape=["steps", ""], rec="nothard", lens_in=["docs"], lens_out=["scholar", "news"], keys=[["github"], ["paginat"]], tag="docs"), q("What does useEffect's cleanup function do in React?", EN, cx=[S, ST], shape=["explainer", ""], rec="nothard", lens_in=["docs"], lens_out=["scholar", "news"], keys=[["useeffect"]], tag="docs"), q("Wie setze ich in FastAPI einen Timeout für einen Request?", DE, cx=[S, ST], shape=["steps", ""], rec="nothard", lens_in=["docs"], lens_out=["scholar", "news"], keys=[["fastapi"], ["timeout"]], tag="docs"), q("Welche Parameter hat pandas read_csv für das Trennzeichen?", DE, cx=[S], shape=[""], rec="nothard", lens_in=["docs"], lens_out=["scholar", "news"], keys=[["read_csv"]], tag="docs"), # ------------------------------------------------------------ news / recent q("What happened at the UN climate conference this week?", EN, cx=[ST, R], shape=[""], rec="hard", lens_in=["news"], lens_out=["scholar"], keys=[["climate", "cop"]], tag="news"), q("latest news on the OpenAI board", EN, cx=[ST, S], shape=[""], rec="hard", lens_in=["news"], lens_out=["scholar"], keys=[["openai"]], tag="news"), q("Was ist heute an der Börse passiert?", DE, cx=[S, ST], shape=[""], rec="hard", lens_in=["news"], lens_out=["scholar"], keys=[["börse", "dax", "stock"]], tag="news"), q("Was gibt es Neues zum Bürgergeld?", DE, cx=[S, ST], shape=[""], rec="set", lens_in=["news"], lens_out=["scholar"], keys=[["bürgergeld"]], tag="news"), # ------------------------------------------------------------ prose with structure q("How did the 2008 financial crisis come about?", EN, cx=[ST, R], shape=[""], rec="none", lens_out=["news"], keys=[["2008", "financial crisis"]], tag="history"), q("Is a heat pump worth it in an old house?", EN, cx=[ST, R], shape=[""], rec="nothard", lens_out=["news"], keys=[["heat pump"]], tag="decision"), q("What are the side effects of ibuprofen?", EN, cx=[S, ST], shape=[""], rec="none", lens_out=["news"], keys=[["ibuprofen"]], tag="list"), q("Wie kam es zum Ukrainekrieg?", DE, cx=[ST, R], shape=[""], rec="nothard", lens_out=["scholar"], keys=[["ukraine"]], tag="history"), q("Lohnt sich eine Balkonsolaranlage?", DE, cx=[ST, R], shape=[""], rec="nothard", keys=[["balkon", "balcony", "solar"]], tag="decision"), q("Welche Unterlagen brauche ich für einen Bauantrag?", DE, cx=[S, ST], shape=["", "steps"], rec="nothard", lens_out=["scholar", "news"], keys=[["bauantrag"]], tag="list"), # ------------------------------------------------------------ series q("How has the price of gold changed over the last ten years?", EN, cx=[S, ST], shape=["series"], rec="nothard", lens_out=["scholar"], keys=[["gold"]], tag="series"), q("Wie hat sich die Einwohnerzahl Berlins seit 1990 entwickelt?", DE, cx=[S, ST], shape=["series"], rec="none", lens_out=["scholar", "news"], keys=[["berlin"]], tag="series"), # ------------------------------------------------------------ research, multi-part q("What is the current state of research on solid-state batteries?", EN, cx=[R], shape=[""], rec="nothard", lens_in=["scholar"], keys=[["solid-state", "solid state"]], tag="survey"), q("Welche Förderungen gibt es 2026 für Wärmepumpen, wie hoch sind sie und wie beantrage ich sie?", DE, cx=[R], shape=["", "steps"], rec="set", lens_out=["scholar"], keys=[["wärmepumpe", "heat pump"], ["förder"]], tag="multipart"), # ------------------------------------------------------------ follow-ups q("and how much does it cost?", EN, cx=[S, ST], shape=["figure", ""], rec="any", lens_out=["scholar"], keys=[["steam deck"]], sq=[["steam deck"]], tag="followup", hist=[{"role": "user", "content": "What is the Steam Deck OLED?"}, {"role": "assistant", "content": "The Steam Deck OLED is Valve's handheld gaming PC with an " "OLED screen, released in November 2023."}]), q("who founded it?", EN, cx=[S], shape=["", "profile"], img=None, rec="nothard", lens_out=["scholar"], keys=[["mistral"]], sq=[["mistral"]], tag="followup", hist=[{"role": "user", "content": "What is Mistral AI?"}, {"role": "assistant", "content": "Mistral AI is a French company that builds large language " "models, based in Paris."}]), q("is it safe during pregnancy?", EN, cx=[ST, R], shape=[""], rec="nothard", lens_in=["scholar"], keys=[["ibuprofen"]], sq=[["ibuprofen"]], tag="followup", hist=[{"role": "user", "content": "What is ibuprofen used for?"}, {"role": "assistant", "content": "Ibuprofen is a nonsteroidal anti-inflammatory drug used for " "pain, fever and inflammation."}]), q("und wie viel kostet das?", DE, cx=[S, ST], shape=["figure", ""], rec="any", lens_out=["scholar"], keys=[["deutschlandticket", "deutschland-ticket"]], sq=[["deutschlandticket", "deutschland-ticket"]], tag="followup", hist=[{"role": "user", "content": "Was ist das Deutschlandticket?"}, {"role": "assistant", "content": "Das Deutschlandticket ist ein bundesweit gültiges Monatsabo " "für den öffentlichen Nahverkehr."}]), q("Und wo liegt die?", DE, cx=[S], shape=["", "profile", "image"], img=None, rec="none", lens_out=["scholar", "news"], keys=[["zugspitze"]], sq=[["zugspitze"]], tag="followup", hist=[{"role": "user", "content": "Wie hoch ist die Zugspitze?"}, {"role": "assistant", "content": "Die Zugspitze ist 2962 Meter hoch und damit Deutschlands " "höchster Berg."}]), q("Gibt es dazu Studien?", DE, cx=[ST, R], shape=[""], rec="nothard", lens_in=["scholar"], keys=[["ashwagandha"]], sq=[["ashwagandha"]], tag="followup", hist=[{"role": "user", "content": "Was ist Ashwagandha?"}, {"role": "assistant", "content": "Ashwagandha ist eine Pflanze aus der ayurvedischen Medizin, " "die als Nahrungsergänzungsmittel gegen Stress verkauft wird."}]), # ------------------------------------------------------------ misc everyday q("best budget mechanical keyboard", EN, cx=[ST, R], shape=["", "table"], img=None, rec="nothard", lens_out=["scholar"], keys=[["keyboard"]], tag="recommend"), q("Can I bring a power bank on a plane?", EN, cx=[S, ST], shape=[""], rec="nothard", lens_out=["scholar"], keys=[["power bank"]], tag="rule"), q("What time zone is Brazil in?", EN, cx=[S], shape=[""], rec="none", lens_out=["scholar", "news"], keys=[["brazil"]], tag="fact"), q("recipe for a classic carbonara", EN, cx=[S, ST], shape=["steps", ""], rec="none", lens_out=["scholar", "news"], keys=[["carbonara"]], tag="recipe"), q("How long is a US driver's license valid in Germany?", EN, cx=[S, ST], shape=[""], rec="nothard", lens_out=["scholar"], keys=[["license", "licence"]], tag="rule"), q("Wie lange muss ich Steuerunterlagen aufbewahren?", DE, cx=[S], shape=[""], rec="nothard", lens_out=["scholar", "news"], keys=[["aufbewahr", "steuer"]], tag="rule"), q("Rezept für Schweinebraten mit Kruste", DE, cx=[S, ST], shape=["steps", ""], rec="none", lens_out=["scholar", "news"], keys=[["schweinebraten"]], tag="recipe"), q("Darf mein Vermieter die Kaution einbehalten, wenn ich Löcher in der Wand habe?", DE, cx=[S, ST], shape=[""], rec="nothard", lens_out=["scholar"], keys=[["kaution"]], tag="rule"), q("Welche Zeitzone hat Japan?", DE, cx=[S], shape=[""], rec="none", lens_out=["scholar", "news"], keys=[["japan"]], tag="fact"), q("Was kann man in Lissabon an einem Wochenende unternehmen?", DE, cx=[S, ST], shape=[""], img=None, rec="nothard", lens_out=["scholar"], keys=[["lissabon", "lisbon"]], tag="recommend"), q("Was ist der Unterschied zwischen GmbH und UG?", DE, cx=[ST, R], shape=["comparison"], rec="nothard", lens_out=["scholar", "news"], keys=[["gmbh"], ["ug"]], tag="comparison"), q("Wie viele Kalorien hat eine Banane?", DE, cx=[S], shape=["figure"], rec="none", lens_out=["news"], keys=[["banane", "banana"]], tag="figure"), q("What's the difference between a virus and a bacterium?", EN, cx=[ST, S, R], shape=["comparison", "explainer"], rec="none", lens_out=["news"], keys=[["virus"], ["bacteri"]], tag="comparison"), q("Which documents do I need to open a bank account in the UK as a foreigner?", EN, cx=[S, ST], shape=["", "steps"], rec="nothard", lens_out=["scholar", "news"], keys=[["bank account"]], tag="list"), ] if __name__ == "__main__": out = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testset.jsonl") with open(out, "w") as f: for item in ITEMS: f.write(json.dumps(item, ensure_ascii=False) + "\n") langs = [i["lang"] for i in ITEMS] print(f"{len(ITEMS)} items: {langs.count('en')} en, {langs.count('de')} de -> {out}")