Blame for training/eval/make_testset.py

"""Hand-labelled routing test set: about 50 English and 50 German questions.

Labels follow the app's CLASSIFY prompt. Where the prompt leaves room, a field
lists every answer it allows, so a model is only marked wrong for a reading
the prompt rules out.

Per item:
  q          the question
  hist       prior turns (user, assistant, ...), optional
  search     needs_search
  cx         allowed complexity values
  shape      allowed answer_shape values ("" is prose)
  img        wants_image, or None when either is fine
  rec        "none" (must be empty), "any" (anything), "set" (must be set),
             "hard" (day/week/month), "nothard" (empty or year)
  lens_in    corpora that must be among `lenses` (searched questions only)
  lens_out   corpora that must not be
  keys       groups of terms; every group needs one hit in the queries
             (lowercase substring), searched questions only
  sq         groups of terms the standalone question must contain (follow-ups)
  place      accepted `place` substrings for weather (any one)

    python make_testset.py   # writes testset.jsonl
"""

import json
import os

S, ST, R = "simple", "standard", "research"
ANYCX = [S, ST, R]


def q(text, lang, search=True, cx=ANYCX, shape=("",), img=False, rec="any",
      lens_in=(), lens_out=(), keys=(), sq=(), place="", hist=(), tag=""):
    return {"q": text, "lang": lang, "search": search, "cx": list(cx), "shape": list(shape),
            "img": img, "rec": rec, "lens_in": list(lens_in), "lens_out": list(lens_out),
            "keys": [list(k) for k in keys], "sq": [list(k) for k in sq], "place": list(place),
            "hist": list(hist), "tag": tag}



NOSHAPE_OR_EXPL = ("", "explainer")
EN, DE = "en", "de"

ITEMS = [
    # ------------------------------------------------------------ no search
    q("thanks, that helped a lot!", EN, search=False, img=False, tag="smalltalk"),
    q("hi there", EN, search=False, tag="smalltalk"),
    q("Translate into German: 'The meeting has been moved to Thursday afternoon.'", EN,
      search=False, tag="transform"),
    q("Rewrite this to sound more polite: send me the report by tomorrow or else.", EN,
      search=False, tag="transform"),
    q("What is 17% of 2,340?", EN, search=False, tag="arithmetic"),
    q("Why does this Python print 3 instead of 4?\n\nx = [1, 2, 3]\nx.append(4)\nprint(len(x[:-1]))",
      EN, search=False, tag="code"),
    q("Summarise this in one sentence: Our Q3 revenue rose 12% on stronger subscription sales, "
      "while hardware margins fell because of higher component costs.", EN, search=False, tag="transform"),
    q("danke dir, super erklärt", DE, search=False, tag="smalltalk"),
    q("Guten Morgen!", DE, search=False, tag="smalltalk"),
    q("Übersetze ins Englische: Ich komme heute etwas später, der Zug hat Verspätung.", DE,
      search=False, tag="transform"),
    q("Formuliere das freundlicher: Ihre Rechnung ist seit drei Wochen überfällig, zahlen Sie endlich.",
      DE, search=False, tag="transform"),
    q("Was ist 15 % von 480 Euro?", DE, search=False, tag="arithmetic"),
    q("Was macht diese SQL-Abfrage? SELECT name, COUNT(*) FROM orders GROUP BY name HAVING COUNT(*) > 5",
      DE, search=False, tag="code"),
    q("Kannst du deine letzte Antwort kürzer zusammenfassen?", DE, search=False, tag="conversation",
      hist=[{"role": "user", "content": "Wie funktioniert eine Wärmepumpe?"},
            {"role": "assistant", "content": "Eine Wärmepumpe entzieht der Umgebung Wärme und hebt sie "
             "mit einem Verdichter auf ein höheres Temperaturniveau, um damit zu heizen."}]),

    # ------------------------------------------------------------ lookups
    q("What is the Bitcoin price right now?", EN, cx=[S], shape=["figure"], rec="set",
      lens_out=["scholar"], keys=[["bitcoin", "btc"]], tag="figure"),
    q("How many people live in Tokyo?", EN, cx=[S], shape=["figure", ""], rec="nothard",
      lens_out=["scholar"], keys=[["tokyo"]], tag="figure"),
    q("When did the Berlin Wall fall?", EN, cx=[S], shape=[""], rec="none",
      lens_out=["scholar", "news"], keys=[["berlin wall"]], tag="fact"),
    q("What's the latest stable version of Python?", EN, cx=[S], shape=["", "figure"], rec="set",
      lens_out=["scholar"], keys=[["python"]], tag="version"),
    q("weather in Hamburg tomorrow", EN, cx=[S], shape=["weather"], place=("hamburg",),
      lens_out=["scholar"], keys=[["hamburg"]], tag="weather"),
    q("Will it rain in Seattle this weekend?", EN, cx=[S], shape=["weather"], place=("seattle",),
      lens_out=["scholar"], keys=[["seattle"]], tag="weather"),
    q("Who won the last Champions League final?", EN, cx=[S], shape=["match", ""], rec="nothard",
      lens_out=["scholar"], keys=[["champions league"]], tag="match"),
    q("Wie viel kostet ein Liter Diesel aktuell in Deutschland?", DE, cx=[S], shape=["figure"], rec="set",
      lens_out=["scholar"], keys=[["diesel"]], tag="figure"),
    q("Wie hoch ist der Leitzins der EZB?", DE, cx=[S], shape=["figure"], rec="set",
      lens_out=["scholar"], keys=[["leitzins", "ezb", "ecb", "interest rate"]], tag="figure"),
    q("Wann wurde Goethe geboren?", DE, cx=[S], shape=[""], rec="none",
      lens_out=["scholar", "news"], keys=[["goethe"]], tag="fact"),
    q("Wetter in München morgen", DE, cx=[S], shape=["weather"], place=("münchen","munich"),
      lens_out=["scholar"], keys=[["münchen", "munich"]], tag="weather"),
    q("Wie wird das Wetter am Wochenende in Köln?", DE, cx=[S], shape=["weather"], place=("köln","cologne"),
      lens_out=["scholar"], keys=[["köln", "cologne"]], tag="weather"),
    q("Wie hat Bayern gegen Dortmund gespielt?", DE, cx=[S], shape=["match"], rec="set",
      lens_out=["scholar"], keys=[["bayern"], ["dortmund", "bvb"]], tag="match"),
    q("Welche Version hat Node.js LTS gerade?", DE, cx=[S], shape=["", "figure"], rec="set",
      lens_out=["scholar"], keys=[["node"]], tag="version"),

    # ------------------------------------------------------------ profile / image
    q("Who is Jensen Huang?", EN, cx=[S, ST], shape=["profile"], img=True, rec="nothard",
      lens_out=["scholar"], keys=[["jensen huang"]], tag="profile"),
    q("What is the company Anduril?", EN, cx=[S, ST], shape=["profile"], img=None, rec="nothard",
      lens_out=["scholar"], keys=[["anduril"]], tag="profile"),
    q("what is Obsidian, the notes app?", EN, cx=[S, ST], shape=["profile"], img=None, rec="nothard",
      lens_out=["scholar"], keys=[["obsidian"]], tag="profile"),
    q("What does an okapi look like?", EN, cx=[S], shape=["image", "profile"], img=True, rec="none",
      lens_out=["news"], keys=[["okapi"]], tag="image"),
    q("show me the flag of Bhutan", EN, cx=[S], shape=["image"], img=True, rec="none",
      lens_out=["scholar", "news"], keys=[["bhutan"]], tag="image"),
    q("Wer ist Annalena Baerbock?", DE, cx=[S, ST], shape=["profile"], img=True, rec="nothard",
      lens_out=["scholar"], keys=[["baerbock"]], tag="profile"),
    q("Was ist die Firma Celonis?", DE, cx=[S, ST], shape=["profile"], img=None, rec="nothard",
      lens_out=["scholar"], keys=[["celonis"]], tag="profile"),
    q("Wer war Hedy Lamarr?", DE, cx=[S, ST], shape=["profile"], img=True, rec="none",
      lens_out=["scholar", "news"], keys=[["lamarr"]], tag="profile"),
    q("Wie sieht ein Axolotl aus?", DE, cx=[S], shape=["image", "profile"], img=True, rec="none",
      lens_out=["news"], keys=[["axolotl"]], tag="image"),
    q("Zeig mir ein Bild vom Kölner Dom", DE, cx=[S], shape=["image"], img=True, rec="none",
      lens_out=["scholar", "news"], keys=[["dom", "cathedral"]], tag="image"),
    q("Wie sieht es mit der Rente mit 63 aus, gibt es die noch?", DE, cx=[S, ST], shape=[""], img=False,
      rec="set", lens_out=["scholar"], keys=[["rente"]], tag="idiom"),

    # ------------------------------------------------------------ explainer
    q("How do transformers work in machine learning?", EN, cx=[ST, S], shape=["explainer"],
      rec="none", keys=[["transformer"]], tag="explainer"),
    q("Why is the sky blue?", EN, cx=[S, ST], shape=["explainer"], rec="none",
      lens_out=["news"], keys=[["sky", "rayleigh"]], tag="explainer"),
    q("Explain how public key cryptography works", EN, cx=[S, ST], shape=["explainer"], rec="none",
      lens_out=["news"], keys=[["public key", "public-key", "asymmetric"]], tag="explainer"),
    q("What does impedance mean in electronics?", EN, cx=[S, ST], shape=["explainer"], rec="none",
      lens_out=["news"], keys=[["impedance"]], tag="explainer"),
    q("Wie funktionieren Autoencoder?", DE, cx=[S, ST], shape=["explainer"], rec="none",
      lens_out=["news"], keys=[["autoencoder"]], tag="explainer"),
    q("Warum wird Brot hart, wenn es alt wird?", DE, cx=[S, ST], shape=["explainer"], rec="none",
      lens_out=["news"], keys=[["brot", "bread"]], tag="explainer"),
    q("Was bedeutet Inflation eigentlich genau?", DE, cx=[S, ST], shape=["explainer"], rec="none",
      lens_out=["news"], keys=[["inflation"]], tag="explainer"),
    q("Wie funktioniert eine Wärmepumpe?", DE, cx=[S, ST], shape=["explainer", "diagram"], rec="none",
      lens_out=["news"], keys=[["wärmepumpe", "heat pump"]], tag="explainer"),

    # ------------------------------------------------------------ steps
    q("How do I install Docker on Ubuntu 24.04?", EN, cx=[S, ST], shape=["steps"], rec="nothard",
      lens_out=["scholar", "news"], keys=[["docker"], ["ubuntu"]], tag="steps"),
    q("How do I renew a US passport by mail?", EN, cx=[S, ST], shape=["steps"], rec="nothard",
      lens_out=["scholar"], keys=[["passport"]], tag="steps"),
    q("How do I set up SSH keys for GitHub?", EN, cx=[S, ST], shape=["steps"], rec="nothard",
      lens_out=["scholar", "news"], keys=[["ssh"], ["github"]], tag="steps"),
    q("Wie beantrage ich einen neuen Personalausweis?", DE, cx=[S, ST], shape=["steps"], rec="nothard",
      lens_out=["scholar"], keys=[["personalausweis"]], tag="steps"),
    q("Wie melde ich ein Gewerbe an?", DE, cx=[S, ST], shape=["steps"], rec="nothard",
      lens_out=["scholar"], keys=[["gewerbe"]], tag="steps"),
    q("Wie richte ich einen Cronjob unter Linux ein?", DE, cx=[S, ST], shape=["steps"], rec="nothard",
      lens_out=["scholar", "news"], keys=[["cron"]], tag="steps"),

    # ------------------------------------------------------------ compare -> research
    q("PostgreSQL vs MySQL for a small web app?", EN, cx=[R], shape=["comparison"], rec="nothard",
      lens_out=["scholar", "news"], keys=[["postgres"], ["mysql"]], tag="comparison"),
    q("Should I buy an iPhone 17 or a Pixel 10?", EN, cx=[R], shape=["comparison"], img=None, rec="nothard",
      lens_out=["scholar"], keys=[["iphone"], ["pixel"]], tag="comparison"),
    q("Compare Rust, Go and Zig for systems programming", EN, cx=[R], shape=["table"], rec="nothard",
      lens_out=["news"], keys=[["rust"], ["go", "golang"], ["zig"]], tag="table"),
    q("ETF oder Festgeld, was ist 2026 sinnvoller?", DE, cx=[R], shape=["comparison"], rec="set",
      lens_out=["scholar"], keys=[["etf"], ["festgeld"]], tag="comparison"),
    q("Vergleiche Tesla Model 3, VW ID.3 und BYD Dolphin", DE, cx=[R], shape=["table"], img=None,
      rec="nothard", lens_out=["scholar"], keys=[["model 3", "tesla"], ["id.3", "id3"], ["dolphin", "byd"]],
      tag="table"),
    q("Was ist besser für Einsteiger, Fuji oder Sony Kameras?", DE, cx=[R], shape=["comparison"], img=None,
      rec="nothard", lens_out=["scholar"], keys=[["fuji"], ["sony"]], tag="comparison"),

    # ------------------------------------------------------------ evidence -> scholar
    q("Does creatine actually help with muscle growth?", EN, cx=[ST, R], shape=["", "explainer"],
      rec="nothard", lens_in=["scholar"], keys=[["creatine"]], tag="evidence"),
    q("What does the research say about intermittent fasting and weight loss?", EN, cx=[R], shape=[""],
      rec="nothard", lens_in=["scholar"], keys=[["fasting"]], tag="evidence"),
    q("Is aspartame harmful?", EN, cx=[ST, R], shape=[""], rec="nothard", lens_in=["scholar"],
      keys=[["aspartame"]], tag="evidence"),
    q("how much vitamin D should an adult take per day", EN, cx=[S, ST], shape=["", "figure"],
      rec="nothard", lens_in=["scholar"], keys=[["vitamin d"]], tag="evidence"),
    q("Bringt Kreatin was?", DE, cx=[ST, R], shape=["", "explainer"], rec="nothard", lens_in=["scholar"],
      keys=[["creatine", "kreatin"]], tag="evidence"),
    q("Was sagt die Studienlage zu Melatonin bei Schlafstörungen?", DE, cx=[R], shape=[""], rec="nothard",
      lens_in=["scholar"], keys=[["melatonin"]], tag="evidence"),
    q("Ist Süßstoff schädlich?", DE, cx=[ST, R], shape=[""], rec="nothard", lens_in=["scholar"],
      keys=[["sweetener", "süßstoff", "aspartame", "sucralose"]], tag="evidence"),
    q("Warum wirkt Koffein bei mir nicht mehr?", DE, cx=[S, ST], shape=["explainer", ""], rec="nothard",
      lens_in=["scholar"], keys=[["caffeine", "koffein"]], tag="evidence"),

    # ------------------------------------------------------------ docs / API
    q("How do I paginate results with the GitHub REST API?", EN, cx=[S, ST], shape=["steps", ""],
      rec="nothard", lens_in=["docs"], lens_out=["scholar", "news"], keys=[["github"], ["paginat"]],
      tag="docs"),
    q("What does useEffect's cleanup function do in React?", EN, cx=[S, ST], shape=["explainer", ""],
      rec="nothard", lens_in=["docs"], lens_out=["scholar", "news"], keys=[["useeffect"]], tag="docs"),
    q("Wie setze ich in FastAPI einen Timeout für einen Request?", DE, cx=[S, ST], shape=["steps", ""],
      rec="nothard", lens_in=["docs"], lens_out=["scholar", "news"], keys=[["fastapi"], ["timeout"]],
      tag="docs"),
    q("Welche Parameter hat pandas read_csv für das Trennzeichen?", DE, cx=[S], shape=[""],
      rec="nothard", lens_in=["docs"], lens_out=["scholar", "news"], keys=[["read_csv"]], tag="docs"),

    # ------------------------------------------------------------ news / recent
    q("What happened at the UN climate conference this week?", EN, cx=[ST, R], shape=[""], rec="hard",
      lens_in=["news"], lens_out=["scholar"], keys=[["climate", "cop"]], tag="news"),
    q("latest news on the OpenAI board", EN, cx=[ST, S], shape=[""], rec="hard", lens_in=["news"],
      lens_out=["scholar"], keys=[["openai"]], tag="news"),
    q("Was ist heute an der Börse passiert?", DE, cx=[S, ST], shape=[""], rec="hard", lens_in=["news"],
      lens_out=["scholar"], keys=[["börse", "dax", "stock"]], tag="news"),
    q("Was gibt es Neues zum Bürgergeld?", DE, cx=[S, ST], shape=[""], rec="set", lens_in=["news"],
      lens_out=["scholar"], keys=[["bürgergeld"]], tag="news"),

    # ------------------------------------------------------------ prose with structure
    q("How did the 2008 financial crisis come about?", EN, cx=[ST, R], shape=[""], rec="none",
      lens_out=["news"], keys=[["2008", "financial crisis"]], tag="history"),
    q("Is a heat pump worth it in an old house?", EN, cx=[ST, R], shape=[""], rec="nothard",
      lens_out=["news"], keys=[["heat pump"]], tag="decision"),
    q("What are the side effects of ibuprofen?", EN, cx=[S, ST], shape=[""], rec="none",
      lens_out=["news"], keys=[["ibuprofen"]], tag="list"),
    q("Wie kam es zum Ukrainekrieg?", DE, cx=[ST, R], shape=[""], rec="nothard", lens_out=["scholar"],
      keys=[["ukraine"]], tag="history"),
    q("Lohnt sich eine Balkonsolaranlage?", DE, cx=[ST, R], shape=[""], rec="nothard",
      keys=[["balkon", "balcony", "solar"]], tag="decision"),
    q("Welche Unterlagen brauche ich für einen Bauantrag?", DE, cx=[S, ST], shape=["", "steps"],
      rec="nothard", lens_out=["scholar", "news"], keys=[["bauantrag"]], tag="list"),

    # ------------------------------------------------------------ series
    q("How has the price of gold changed over the last ten years?", EN, cx=[S, ST], shape=["series"],
      rec="nothard", lens_out=["scholar"], keys=[["gold"]], tag="series"),
    q("Wie hat sich die Einwohnerzahl Berlins seit 1990 entwickelt?", DE, cx=[S, ST], shape=["series"],
      rec="none", lens_out=["scholar", "news"], keys=[["berlin"]], tag="series"),

    # ------------------------------------------------------------ research, multi-part
    q("What is the current state of research on solid-state batteries?", EN, cx=[R], shape=[""],
      rec="nothard", lens_in=["scholar"], keys=[["solid-state", "solid state"]], tag="survey"),
    q("Welche Förderungen gibt es 2026 für Wärmepumpen, wie hoch sind sie und wie beantrage ich sie?", DE,
      cx=[R], shape=["", "steps"], rec="set", lens_out=["scholar"], keys=[["wärmepumpe", "heat pump"],
                                                                        ["förder"]], tag="multipart"),

    # ------------------------------------------------------------ follow-ups
    q("and how much does it cost?", EN, cx=[S, ST], shape=["figure", ""], rec="any", lens_out=["scholar"],
      keys=[["steam deck"]], sq=[["steam deck"]], tag="followup",
      hist=[{"role": "user", "content": "What is the Steam Deck OLED?"},
            {"role": "assistant", "content": "The Steam Deck OLED is Valve's handheld gaming PC with an "
             "OLED screen, released in November 2023."}]),
    q("who founded it?", EN, cx=[S], shape=["", "profile"], img=None, rec="nothard", lens_out=["scholar"],
      keys=[["mistral"]], sq=[["mistral"]], tag="followup",
      hist=[{"role": "user", "content": "What is Mistral AI?"},
            {"role": "assistant", "content": "Mistral AI is a French company that builds large language "
             "models, based in Paris."}]),
    q("is it safe during pregnancy?", EN, cx=[ST, R], shape=[""], rec="nothard", lens_in=["scholar"],
      keys=[["ibuprofen"]], sq=[["ibuprofen"]], tag="followup",
      hist=[{"role": "user", "content": "What is ibuprofen used for?"},
            {"role": "assistant", "content": "Ibuprofen is a nonsteroidal anti-inflammatory drug used for "
             "pain, fever and inflammation."}]),
    q("und wie viel kostet das?", DE, cx=[S, ST], shape=["figure", ""], rec="any", lens_out=["scholar"],
      keys=[["deutschlandticket", "deutschland-ticket"]], sq=[["deutschlandticket", "deutschland-ticket"]],
      tag="followup",
      hist=[{"role": "user", "content": "Was ist das Deutschlandticket?"},
            {"role": "assistant", "content": "Das Deutschlandticket ist ein bundesweit gültiges Monatsabo "
             "für den öffentlichen Nahverkehr."}]),
    q("Und wo liegt die?", DE, cx=[S], shape=["", "profile", "image"], img=None, rec="none",
      lens_out=["scholar", "news"], keys=[["zugspitze"]], sq=[["zugspitze"]], tag="followup",
      hist=[{"role": "user", "content": "Wie hoch ist die Zugspitze?"},
            {"role": "assistant", "content": "Die Zugspitze ist 2962 Meter hoch und damit Deutschlands "
             "höchster Berg."}]),
    q("Gibt es dazu Studien?", DE, cx=[ST, R], shape=[""], rec="nothard", lens_in=["scholar"],
      keys=[["ashwagandha"]], sq=[["ashwagandha"]], tag="followup",
      hist=[{"role": "user", "content": "Was ist Ashwagandha?"},
            {"role": "assistant", "content": "Ashwagandha ist eine Pflanze aus der ayurvedischen Medizin, "
             "die als Nahrungsergänzungsmittel gegen Stress verkauft wird."}]),

    # ------------------------------------------------------------ misc everyday
    q("best budget mechanical keyboard", EN, cx=[ST, R], shape=["", "table"], img=None, rec="nothard",
      lens_out=["scholar"], keys=[["keyboard"]], tag="recommend"),
    q("Can I bring a power bank on a plane?", EN, cx=[S, ST], shape=[""], rec="nothard",
      lens_out=["scholar"], keys=[["power bank"]], tag="rule"),
    q("What time zone is Brazil in?", EN, cx=[S], shape=[""], rec="none", lens_out=["scholar", "news"],
      keys=[["brazil"]], tag="fact"),
    q("recipe for a classic carbonara", EN, cx=[S, ST], shape=["steps", ""], rec="none",
      lens_out=["scholar", "news"], keys=[["carbonara"]], tag="recipe"),
    q("How long is a US driver's license valid in Germany?", EN, cx=[S, ST], shape=[""], rec="nothard",
      lens_out=["scholar"], keys=[["license", "licence"]], tag="rule"),
    q("Wie lange muss ich Steuerunterlagen aufbewahren?", DE, cx=[S], shape=[""], rec="nothard",
      lens_out=["scholar", "news"], keys=[["aufbewahr", "steuer"]], tag="rule"),
    q("Rezept für Schweinebraten mit Kruste", DE, cx=[S, ST], shape=["steps", ""], rec="none",
      lens_out=["scholar", "news"], keys=[["schweinebraten"]], tag="recipe"),
    q("Darf mein Vermieter die Kaution einbehalten, wenn ich Löcher in der Wand habe?", DE, cx=[S, ST],
      shape=[""], rec="nothard", lens_out=["scholar"], keys=[["kaution"]], tag="rule"),
    q("Welche Zeitzone hat Japan?", DE, cx=[S], shape=[""], rec="none", lens_out=["scholar", "news"],
      keys=[["japan"]], tag="fact"),
    q("Was kann man in Lissabon an einem Wochenende unternehmen?", DE, cx=[S, ST], shape=[""], img=None,
      rec="nothard", lens_out=["scholar"], keys=[["lissabon", "lisbon"]], tag="recommend"),
    q("Was ist der Unterschied zwischen GmbH und UG?", DE, cx=[ST, R], shape=["comparison"],
      rec="nothard", lens_out=["scholar", "news"], keys=[["gmbh"], ["ug"]], tag="comparison"),
    q("Wie viele Kalorien hat eine Banane?", DE, cx=[S], shape=["figure"], rec="none",
      lens_out=["news"], keys=[["banane", "banana"]], tag="figure"),
    q("What's the difference between a virus and a bacterium?", EN, cx=[ST, S, R],
      shape=["comparison", "explainer"], rec="none", lens_out=["news"], keys=[["virus"], ["bacteri"]],
      tag="comparison"),
    q("Which documents do I need to open a bank account in the UK as a foreigner?", EN, cx=[S, ST],
      shape=["", "steps"], rec="nothard", lens_out=["scholar", "news"], keys=[["bank account"]], tag="list"),
]

if __name__ == "__main__":
    out = os.path.join(os.path.dirname(os.path.abspath(__file__)), "testset.jsonl")
    with open(out, "w") as f:
        for item in ITEMS:
            f.write(json.dumps(item, ensure_ascii=False) + "\n")
    langs = [i["lang"] for i in ITEMS]
    print(f"{len(ITEMS)} items: {langs.count('en')} en, {langs.count('de')} de -> {out}")