tiny-scene-three / ml /gen.py
paralaif's picture
Add generated music and sound: Tone.js band, ambience, beat-synced animation, video export
4a0e872 verified
Raw History Blame Contribute Delete
35.9 kB
"""Synthetic (prompt -> scene language) pairs in English and Spanish.
sample(rng) returns (prompt_text, [target tokens]). Three kinds of targets:
<new> ... a full scene
<edit> <set|add|del|env|cam> ... operations on the current scene
<qa> lang=.. topic=.. a three.js question
"""
import random
import unicodedata
from lexicon import (ACT, AROUND_ACTS, CAM, COLOR, GENRE, LAYER, LOC, MOOD, MOTION, OVER_ACTS, REL, SHOT, SIZE, SPD,
TEMPO, TIME, TYPES, WEATHER)
from qa_topics import Q_EN, Q_ES, SUF_EN, SUF_ES, TOPICS
TYPE_KEYS, COLOR_KEYS, SIZE_KEYS, ACT_KEYS = list(TYPES), list(COLOR), list(SIZE), list(ACT)
NUM = {"en": {2: "two", 3: "three", 4: "four", 5: "five", 6: "six"},
"es": {2: "dos", 3: "tres", 4: "cuatro", 5: "cinco", 6: "seis"}}
MAX_PROMPT = 160
# ------------------------------------------------------------------ helpers
def agree(s, g, pl):
return s.replace("{o}", {("m", 0): "o", ("f", 0): "a", ("m", 1): "os", ("f", 1): "as"}[(g, int(pl))])
def es_form(forms, g, pl):
return forms[(0 if g == "m" else 1) + (2 if pl else 0)]
def an(word):
w = word.lower()
return "an" if (w[0] in "aeio" or (w[0] == "u" and not w.startswith("uf"))) else "a"
def es_join(prep, np):
"""'encima de' + 'el coche' -> 'encima del coche'; 'a' + 'el perro' -> 'al perro'."""
if np.startswith("el "):
if prep.endswith(" de") or prep == "de":
return prep[:-2] + "del " + np[3:]
if prep.endswith(" a") or prep == "a":
return prep[:-1] + "al " + np[3:]
return f"{prep} {np}"
def rand_ent(r, act=True, n=True):
e = {"type": r.choice(TYPE_KEYS)}
if r.random() < 0.45:
e["color"] = r.choice(COLOR_KEYS)
if r.random() < 0.2:
e["size"] = r.choice(SIZE_KEYS)
if n and r.random() < 0.15:
e["n"] = r.randint(2, 6)
if act and r.random() < 0.6:
e["act"] = r.choice(ACT_KEYS)
if e["act"] in MOTION and r.random() < 0.2:
e["spd"] = r.choice(["slow", "fast"])
return e
def ent_tokens(e, head="<ent>"):
t = [head, "type=" + e["type"]]
for k in ("color", "size", "n", "act", "spd", "rel"):
if k in e:
t.append(f"{k}={e[k]}")
return t
# ------------------------------------------------------------------ noun phrases
def np_en(r, e, det=None, act=True):
sing, plur = r.choice(TYPES[e["type"]].en)
n = e.get("n")
pl = n is not None
adjs = []
if "size" in e:
adjs.append(r.choice(SIZE[e["size"]].en))
if "color" in e:
adjs.append(r.choice(COLOR[e["color"]].en))
pre, post = None, ""
a = e.get("act") if act else None
if a:
A = ACT[a]
adv = " " + r.choice(SPD[e["spd"]].en) if "spd" in e else ""
ing = r.choice(A.ing)
x = r.random()
if x < 0.13 and " " not in ing and not adv:
pre = ing
elif x < 0.70:
post = f" {ing}{adv}"
elif x < 0.82:
post = f" that {'are' if pl else 'is'} {ing}{adv}"
elif x < 0.90:
post = f" {'are' if pl else 'is'} {ing}{adv}"
else:
post = f" {r.choice(A.base) if pl else r.choice(A.s3)}{adv}"
words = ([pre] if pre else []) + adjs + [plur if pl else sing]
if pl:
d = r.choice([NUM["en"][n], NUM["en"][n], str(n)])
if det == "the":
d = "the " + d if r.random() < 0.5 else "the"
else:
if det is None:
det = r.choices(["a", "the", "one", ""], [0.78, 0.12, 0.05, 0.05])[0]
d = an(words[0]) if det == "a" else det
return (d + " " if d else "") + " ".join(words) + post
def np_es(r, e, det=None, act=True):
sing, plur, g = r.choice(TYPES[e["type"]].es)
n = e.get("n")
pl = n is not None
adjs = []
if "color" in e:
adjs.append(es_form(r.choice(COLOR[e["color"]].es), g, pl))
if "size" in e:
adjs.append(es_form(r.choice(SIZE[e["size"]].es), g, pl))
if len(adjs) == 2 and r.random() < 0.25:
adjs.reverse()
post = ""
a = e.get("act") if act else None
if a:
A = ACT[a]
adv = " " + r.choice(SPD[e["spd"]].es) if "spd" in e else ""
ger = agree(r.choice(A.ger), g, pl)
x = r.random()
if x < 0.62:
post = f" {ger}{adv}"
elif x < 0.88:
post = f" que {agree(r.choice(A.e3p if pl else A.e3), g, pl)}{adv}"
else:
post = f" que {'están' if pl else 'está'} {ger}{adv}"
if pl:
d = r.choice([NUM["es"][n], NUM["es"][n], str(n)])
if det == "the":
art = "los" if g == "m" else "las"
d = f"{art} {d}" if r.random() < 0.5 else art
else:
if det is None:
det = r.choices(["a", "the", ""], [0.82, 0.13, 0.05])[0]
d = {"a": "un" if g == "m" else "una", "the": "el" if g == "m" else "la", "": ""}[det]
return (d + " " if d else "") + " ".join([plur if pl else sing] + adjs) + post, g, pl
def NP(r, lang, e, det=None, act=True):
return np_en(r, e, det, act) if lang == "en" else np_es(r, e, det, act)[0]
# ------------------------------------------------------------------ new scenes
PREFIX = {
"en": ["create ", "make ", "generate ", "show me ", "i want ", "i want to see ", "a scene with ", "scene of ",
"an animation of ", "draw ", "render ", "give me ", "build ", "animate "],
"es": ["crea ", "genera ", "haz ", "muéstrame ", "quiero ", "quiero ver ", "una escena con ", "escena de ",
"una animación de ", "dibuja ", "dame ", "construye ", "hazme ", "anima "],
}
CONJ = {"en": [" and ", " and ", ", ", " with ", " and also ", " plus "],
"es": [" y ", " y ", ", ", " con ", " y también ", " más "]}
def modifiers(r, lang, p=(0.55, 0.25, 0.2, 0.15, 0.15)):
"""-> (list of phrases, dict of chosen keys)"""
out, keys = [], {}
for name, table, prob in (("loc", LOC, p[0]), ("time", TIME, p[1]), ("weather", WEATHER, p[2]),
("shot", SHOT, p[3]), ("cam", CAM, p[4])):
if r.random() < prob:
k = r.choice(list(table))
keys[name] = k
out.append((name, r.choice(getattr(table[k], lang))))
return out, keys
SHOT_PRE = {
"en": {"close": ["close-up of", "a close-up of", "closeup of", "close up shot of"],
"medium": ["medium shot of", "a medium shot of"],
"wide": ["wide shot of", "a wide shot of", "wide angle shot of", "long shot of"],
"top": ["aerial view of", "top view of", "top-down view of", "bird's eye view of", "an aerial shot of"],
"low": ["low angle shot of", "a low angle view of"]},
"es": {"close": ["primer plano de", "un primer plano de", "plano cerrado de"],
"medium": ["plano medio de", "un plano medio de"],
"wide": ["plano general de", "un plano general de", "plano abierto de", "vista general de"],
"top": ["vista aérea de", "una vista aérea de", "vista cenital de", "toma aérea de"],
"low": ["ángulo bajo de", "contrapicado de", "vista desde abajo de"]},
}
def mod_tokens(keys):
return [f"{k}={keys[k]}" for k in ("loc", "time", "weather", "shot", "cam") if k in keys]
def new_scene(r, lang):
k = r.choices([1, 2, 3], [0.5, 0.4, 0.1])[0]
ents = []
while len(ents) < k:
e = rand_ent(r)
if all(e["type"] != o["type"] for o in ents):
ents.append(e)
rels = []
for i in range(k - 1):
rel = r.choice(list(REL)) if r.random() < 0.6 else None
a = ents[i]
if rel in ("ride", "chase"):
a.pop("act", None), a.pop("spd", None)
if rel == "chase":
ents[i + 1].pop("act", None), ents[i + 1].pop("spd", None)
elif rel == "over" and a.get("act") not in OVER_ACTS:
a["act"] = r.choice(OVER_ACTS)
elif rel == "around" and a.get("act") not in AROUND_ACTS:
a["act"] = r.choice(AROUND_ACTS)
if a.get("act") not in MOTION:
a.pop("spd", None)
if rel:
a["rel"] = rel
rels.append(rel)
segs = []
for i, e in enumerate(ents):
text = ""
if lang == "en":
s = np_en(r, e)
else:
s, g, pl = np_es(r, e)
if i:
rel = rels[i - 1]
if rel is None:
text += r.choice(CONJ[lang]) + s
else:
prep = r.choice(getattr(REL[rel], lang))
if lang == "es":
if rel == "on" and prep == "sobre" and ents[i - 1].get("act") in OVER_ACTS:
prep = "encima de"
prep = agree(prep, pg, ppl)
text += " " + es_join(prep, s)
else:
text += f" {prep} {s}"
else:
text = s
segs.append(text)
if lang == "es":
pg, ppl = g, pl
mods, keys = modifiers(r, lang)
mus_toks = []
if r.random() < 0.2:
phrase, mus_toks = music_phrase(r, lang)
mods.append(("mus", phrase))
r.shuffle(mods)
shot_pre = None
if "shot" in keys and r.random() < 0.3: # "a wide shot of a train ..."
mods = [m for m in mods if m[0] != "shot"]
shot_pre = r.choice(SHOT_PRE[lang][keys["shot"]])
mid = None
if k > 1 and rels[0] is None and "loc" in keys and r.random() < 0.25: # "a shark in the ocean and a duck"
mid = next(m for m in mods if m[0] == "loc")
mods = [m for m in mods if m is not mid]
segs[0] += " " + mid[1]
text = "".join(segs)
loc_first = [m for m in mods if m[0] == "loc"] if r.random() < 0.18 else []
rest = [m[1] for m in mods if m not in loc_first]
sep = r.choice([" ", " ", ", "])
if shot_pre:
text = es_join(shot_pre, text) if lang == "es" else f"{shot_pre} {text}"
if loc_first:
text = loc_first[0][1] + r.choice([", ", " ", ": "]) + text
if rest:
text += r.choice([" ", " ", ", "]) + sep.join(rest)
if r.random() < 0.3 and not shot_pre:
text = r.choice(PREFIX[lang]) + text
tokens = ["<new>"] + mod_tokens(keys) + mus_toks
for e in ents:
tokens += ent_tokens(e)
return text, tokens
# ------------------------------------------------------------------ music
def music_adj(r, lang, kmin=0):
"""Random mood/genre/tempo choice -> (adjective string, tokens). The string may be empty."""
n = r.choices([0, 1, 2, 3], [0.1 if kmin == 0 else 0, 0.5, 0.3, 0.1])[0]
ks = r.sample(["mood", "genre", "tempo"], n)
at = {k: r.choice(list({"mood": MOOD, "genre": GENRE, "tempo": TEMPO}[k])) for k in ks}
w = {k: r.choice(getattr({"mood": MOOD, "genre": GENRE, "tempo": TEMPO}[k][v], lang)) for k, v in at.items()}
order = ["tempo", "mood", "genre"] if lang == "en" else ["genre", "mood", "tempo"]
words = [w[k] for k in order if k in w]
if len(words) > 1 and r.random() < 0.2:
r.shuffle(words)
if lang == "es" and len(words) == 3 and r.random() < 0.5:
words = words[:2] + ["y"] + words[2:]
return " ".join(words), [f"{k}={at[k]}" for k in ("mood", "genre", "tempo") if k in at]
MUS_NEW = {
"en": ["with {m}", "with {m}", "and {m} playing", "with {m} in the background", "to {m}", "with {m} playing",
"while {m} plays", "set to {m}"],
"es": ["con {m}", "con {m}", "y {m} de fondo", "con {m} de fondo", "mientras suena {m}", "al ritmo de {m}",
"con {m} sonando"],
}
MUS_OFF_NEW = {"en": ["without music", "with no music", "in silence", "no music"],
"es": ["sin música", "en silencio", "sin sonido de música"]}
def music_phrase(r, lang):
"""Music clause of a new scene -> (phrase, tokens)."""
if r.random() < 0.1:
return r.choice(MUS_OFF_NEW[lang]), ["<mus>", "mus=off"]
adj, toks = music_adj(r, lang)
if lang == "en":
if adj and r.random() < 0.15:
return f"with {an(adj)} {adj} soundtrack", ["<mus>"] + toks
m = f"{adj} music" if adj else "music"
else:
if adj and r.random() < 0.15:
return f"con banda sonora {adj}", ["<mus>"] + toks
m = f"música {adj}" if adj else "música"
return r.choice(MUS_NEW[lang]).replace("{m}", m), ["<mus>"] + (toks or ["mus=on"])
E_MUS = {
"set_en": ["make the music {a}", "change the music to {a}", "{m}", "play {m}", "i want {m}", "put on some {m}",
"switch to {m}", "now {m}", "can you play {m}", "the music should be {a}", "play something {a}",
"add {m}", "give me {m}"],
"set_es": ["pon {m}", "cambia la música a {a}", "{m}", "quiero {m}", "haz la música {a}", "que la música sea {a}",
"que suene {m}", "ahora {m}", "cambia a {m}", "pon algo {a}", "añade {m}", "ponme {m}"],
"up_en": ["make the music faster", "speed up the music", "faster music", "music faster", "increase the tempo",
"raise the tempo", "a faster beat", "speed the song up"],
"down_en": ["make the music slower", "slow down the music", "slower music", "music slower", "lower the tempo",
"decrease the tempo", "a slower beat", "slow the song down"],
"up_es": ["haz la música más rápida", "acelera la música", "música más rápida", "sube el tempo",
"más rápida la música", "que la música vaya más rápido", "un ritmo más rápido"],
"down_es": ["haz la música más lenta", "ralentiza la música", "música más lenta", "baja el tempo",
"más lenta la música", "que la música vaya más lento", "un ritmo más lento"],
"on_en": ["play music", "add music", "turn on the music", "start the music", "music please", "i want music",
"put on some music", "turn the music back on", "unmute the music"],
"off_en": ["stop the music", "turn off the music", "no music", "mute the music", "remove the music", "silence",
"without music", "kill the music", "i don't want music"],
"on_es": ["pon música", "añade música", "enciende la música", "que suene música", "quiero música",
"activa la música", "vuelve a poner la música", "música por favor"],
"off_es": ["quita la música", "para la música", "sin música", "apaga la música", "silencia la música",
"silencio", "detén la música", "no quiero música"],
"lon_en": ["add {l}", "add {l} back", "bring back {l}", "turn on {l}", "unmute {l}", "i want {l}", "with {l}"],
"loff_en": ["remove {l}", "no {l}", "without {l}", "mute {l}", "turn off {l}", "drop {l}", "take out {l}"],
"lon_es": ["añade {l}", "pon {l}", "vuelve a poner {l}", "activa {l}", "quiero {l}", "con {l}"],
"loff_es": ["quita {l}", "sin {l}", "silencia {l}", "apaga {l}", "elimina {l}", "saca {l}"],
"vup_en": ["louder", "turn up the music", "make the music louder", "volume up", "raise the volume",
"turn it up", "more volume"],
"vdown_en": ["quieter", "turn down the music", "make the music quieter", "volume down", "lower the volume",
"turn it down", "less volume"],
"vup_es": ["más fuerte", "sube el volumen", "sube la música", "más volumen", "música más fuerte",
"súbele al volumen"],
"vdown_es": ["más flojo", "baja el volumen", "baja la música", "menos volumen", "música más baja",
"bájale al volumen"],
}
def music_edit(r, lang):
f = lambda key: r.choice(E_MUS[f"{key}_{lang}"])
kind = r.choices(["set", "tempo", "onoff", "layer", "vol"], [40, 15, 20, 15, 10])[0]
if kind == "set":
adj, toks = music_adj(r, lang, kmin=1)
m = f"{adj} music" if lang == "en" else f"música {adj}"
return f("set").replace("{m}", m).replace("{a}", adj), ["<mus>"] + toks
if kind == "tempo":
d = r.choice(["up", "down"])
return f(d), ["<mus>", f"tempo={d}"]
if kind == "onoff":
d = r.choice(["on", "off"])
return f(d), ["<mus>", f"mus={d}"]
if kind == "layer":
k, d = r.choice(list(LAYER)), r.choice(["on", "off"])
text = f("l" + d).replace("{l}", r.choice(getattr(LAYER[k], lang)))
return text.replace(" a el ", " al "), ["<mus>", f"{k}={d}"]
d = r.choice(["up", "down"])
return f("v" + d), ["<mus>", f"vol={d}"]
# ------------------------------------------------------------------ edits
def ref_en(r, e):
sing, plur = r.choice(TYPES[e["type"]].en)
return "the " + (plur if r.random() < 0.12 else sing)
def ref_es(r, e):
sing, plur, g = r.choice(TYPES[e["type"]].es)
if r.random() < 0.12:
return ("los " if g == "m" else "las ") + plur, g, True
return ("el " if g == "m" else "la ") + sing, g, False
E = {}
E["color_en"] = ["make {ref} {c}", "paint {ref} {c}", "change {ref} to {c}", "turn {ref} {c}", "color {ref} {c}",
"{ref} should be {c}", "i want {ref} {c}", "change the color of {ref} to {c}", "{ref} in {c}",
"now {ref} is {c}", "make {ref} {c} instead", "can you make {ref} {c}", "{ref} {c}"]
E["color_it_en"] = ["make it {c}", "paint it {c}", "turn it {c}", "change it to {c}", "color it {c}", "make that {c}",
"i want it {c}", "now make it {c}"]
E["color_all_en"] = ["make everything {c}", "paint everything {c}", "all of them {c}", "make them all {c}",
"turn everything {c}", "color all of them {c}"]
E["color_es"] = ["haz {ref} {c}", "pinta {ref} de {cm}", "cambia {ref} a {cm}", "pon {ref} {c}", "que {ref} sea {c}",
"quiero {ref} {c}", "cambia el color de {ref} a {cm}", "{ref} {c}", "ahora {ref} es {c}",
"vuelve {ref} {c}", "{ref} de color {cm}", "puedes hacer {ref} {c}", "pon {ref} de color {cm}"]
E["color_it_es"] = ["hazlo {cm}", "hazla {cf}", "píntalo de {cm}", "ponlo {cm}", "cámbialo a {cm}", "píntala de {cm}",
"lo quiero {cm}", "ahora hazlo {cm}", "ponlo de color {cm}"]
E["color_all_es"] = ["haz todo {cm}", "pinta todo de {cm}", "todos {cmp}", "que todos sean {cmp}", "pon todo {cm}",
"todo de color {cm}"]
E["size_en"] = ["make {ref} {s}", "{ref} should be {s}", "i want {ref} {s}", "turn {ref} {s}"]
E["up_en"] = ["make {ref} bigger", "make {ref} larger", "enlarge {ref}", "grow {ref}", "{ref} bigger",
"increase the size of {ref}", "scale up {ref}"]
E["down_en"] = ["make {ref} smaller", "shrink {ref}", "{ref} smaller", "reduce the size of {ref}", "scale down {ref}"]
E["size_es"] = ["haz {ref} {s}", "que {ref} sea {s}", "pon {ref} {s}", "quiero {ref} {s}"]
E["up_es"] = ["haz {ref} más grande", "agranda {ref}", "{ref} más grande", "aumenta el tamaño de {ref}",
"haz crecer {ref}", "que {ref} sea más grande"]
E["down_es"] = ["haz {ref} más pequeñ{o}", "achica {ref}", "reduce {ref}", "encoge {ref}", "{ref} más pequeñ{o}",
"{ref} más chic{o}", "reduce el tamaño de {ref}"]
E["act_en"] = ["make {ref} {base}", "{ref} should {base}", "now {ref} {s3}", "{ref} {s3} now", "have {ref} {base}",
"let {ref} {base}", "{ref} is {ing} now", "i want {ref} to {base}", "{ref} {ing}",
"{ref} starts to {base}", "can {ref} {base}", "make {ref} start {ing}"]
E["act_es"] = ["haz que {ref} {subj}", "que {ref} {subj}", "ahora {ref} {e3}", "{ref} {e3} ahora",
"pon {aref} a {inf}", "quiero que {ref} {subj}", "{ref} {ger}", "que {ref} empiece a {inf}",
"{ref} debe {inf}", "haz {inf} {aref}", "ahora {ref} está {ger}"]
E["fast_en"] = ["make {ref} faster", "speed up {ref}", "{ref} faster", "make {ref} go faster", "{ref} should move faster"]
E["slow_en"] = ["make {ref} slower", "slow down {ref}", "{ref} slower", "make {ref} go slower", "{ref} should move slowly"]
E["normal_en"] = ["{ref} at normal speed", "reset the speed of {ref}", "make {ref} move at normal speed"]
E["fast_es"] = ["haz {ref} más rápido", "acelera {ref}", "{ref} más rápido", "que {ref} vaya más rápido",
"{ref} más veloz"]
E["slow_es"] = ["haz {ref} más lento", "{ref} más despacio", "frena {ref}", "que {ref} vaya más lento",
"{ref} más lento"]
E["normal_es"] = ["{ref} a velocidad normal", "reinicia la velocidad de {ref}", "que {ref} vaya a velocidad normal"]
E["add_en"] = ["add {np}", "add {np}", "put {np}", "add {np} too", "i also want {np}", "also {np}", "now add {np}",
"include {np}", "place {np}", "insert {np}", "add {np} as well", "put {np} there", "and {np}",
"can you add {np}", "throw in {np}"]
E["add_es"] = ["añade {np}", "agrega {np}", "pon {np}", "añade también {np}", "también {np}", "quiero también {np}",
"ahora añade {np}", "incluye {np}", "coloca {np}", "mete {np}", "suma {np}", "y {np}",
"puedes añadir {np}", "ponle {np}", "agrégale {np}"]
E["del_en"] = ["remove {ref}", "delete {ref}", "erase {ref}", "take away {ref}", "get rid of {ref}", "without {ref}",
"take {ref} out", "no {ref}", "i don't want {ref}", "drop {ref}", "remove {ref} please"]
E["del_es"] = ["quita {ref}", "elimina {ref}", "borra {ref}", "saca {ref}", "remueve {ref}", "sin {ref}",
"ya no quiero {ref}", "quita {aref}", "fuera {ref}", "no quiero {ref}", "elimina {aref}"]
E["del_it_en"] = ["remove it", "delete it", "erase it", "get rid of it", "take it away"]
E["del_it_es"] = ["quítalo", "bórralo", "elimínalo", "sácalo", "quítala"]
E["del_all_en"] = ["clear the scene", "remove everything", "delete everything", "start over", "empty the scene",
"clear all", "erase everything", "reset the scene"]
E["del_all_es"] = ["borra todo", "limpia la escena", "quita todo", "elimina todo", "empieza de nuevo",
"vacía la escena", "reinicia la escena"]
E["loc_en"] = ["now {p}", "move the scene {p}", "put them {p}", "now they are {p}", "set the scene {p}", "take them {p}",
"{p}", "change the location to {bare}", "switch to {bare}", "go to {bare}", "change the background to {bare}",
"change the place to {bare}", "move it {p}"]
E["loc_es"] = ["ahora {p}", "ponlos {p}", "que estén {p}", "que sea {p}", "{p}", "cambia el lugar a {bare}",
"cambia el fondo a {bare}", "cambia el escenario a {bare}", "llévalos a {bare}", "vamos a {bare}",
"mueve la escena a {bare}", "ponlo {p}"]
TIME_W = {
"en": {"day": ["day", "daytime", "noon"], "night": ["night", "nighttime", "midnight"],
"sunset": ["sunset", "dusk", "evening"], "dawn": ["dawn", "sunrise", "morning"]},
"es": {"day": ["el día", "el mediodía"], "night": ["la noche", "la medianoche"],
"sunset": ["el atardecer", "el anochecer"], "dawn": ["el amanecer", "la madrugada"]},
}
E["time_en"] = ["make it {w}", "change it to {w}", "switch to {w}", "set the time to {w}", "now it is {w}", "now {p}",
"i want it {p}", "{p}", "change the time to {w}", "turn it into {w}"]
E["time_es"] = ["que sea {p}", "hazlo {p}", "ponlo {p}", "ahora es {p}", "ahora {p}", "{p}", "cambia la hora a {w}",
"cambia a {w}", "pasa a {w}", "lo quiero {p}"]
TIME_X = {"es": {"night": ["que anochezca", "que se haga de noche", "oscurece la escena"],
"dawn": ["que amanezca"], "day": ["que se haga de día", "que salga el sol"], "sunset": ["que atardezca"]},
"en": {"night": ["make it dark", "turn off the sun"], "day": ["make it bright", "let the sun come out"],
"dawn": ["let the sun rise"], "sunset": ["let the sun set"]}}
WEATHER_E = {
"en": {"rain": ["make it rain", "add rain", "start the rain", "let it rain", "i want rain", "now it rains", "rain"],
"snow": ["make it snow", "add snow", "let it snow", "now it snows", "start snowing", "add snowfall"],
"fog": ["add fog", "make it foggy", "add some mist", "i want fog", "fog"],
"storm": ["add a storm", "make it stormy", "start a thunderstorm", "add lightning", "i want a storm"],
"clear": ["stop the rain", "remove the rain", "stop the snow", "remove the fog", "clear the weather",
"clear the sky", "no more rain", "make it sunny", "stop the storm", "good weather",
"remove the snow", "no more fog", "stop raining", "stop snowing"]},
"es": {"rain": ["que llueva", "haz que llueva", "pon lluvia", "añade lluvia", "agrega lluvia", "ahora llueve",
"quiero lluvia", "lluvia"],
"snow": ["que nieve", "haz que nieve", "pon nieve", "añade nieve", "ahora nieva", "que caiga nieve"],
"fog": ["pon niebla", "añade niebla", "agrega neblina", "que haya niebla", "niebla"],
"storm": ["pon una tormenta", "añade una tormenta", "que haya tormenta", "agrega rayos", "quiero una tormenta"],
"clear": ["para la lluvia", "quita la lluvia", "que deje de llover", "quita la nieve", "que deje de nevar",
"quita la niebla", "despeja el cielo", "quita la tormenta", "buen tiempo", "cielo despejado",
"que pare de llover", "ya no quiero lluvia", "que salga el sol y pare la lluvia"]},
}
SHOT_E = {
"en": {"in": ["zoom in", "closer", "get closer", "move the camera closer", "bring the camera closer",
"zoom in a bit", "come closer"],
"out": ["zoom out", "move the camera back", "farther", "pull back", "zoom out a little", "further away",
"move the camera away"],
"close": ["switch to a close-up", "make it a close-up", "show a close-up"],
"wide": ["make it a wide shot", "show the whole scene", "switch to a wide shot"],
"medium": ["normal view", "reset the camera", "default camera"],
"top": ["view from above", "show it from above", "camera from above", "look from above"],
"low": ["camera from below", "view from below", "look from below"]},
"es": {"in": ["acerca la cámara", "más cerca", "acércate", "haz zoom", "zoom in", "acerca un poco",
"acerca más la cámara"],
"out": ["aleja la cámara", "más lejos", "aléjate", "zoom out", "aleja un poco", "aleja más la cámara"],
"close": ["pon un primer plano", "cambia a primer plano", "muestra un primer plano"],
"wide": ["muestra toda la escena", "cambia a plano general", "pon un plano abierto"],
"medium": ["vista normal", "reinicia la cámara", "cámara por defecto"],
"top": ["cámara desde arriba", "míralo desde arriba", "vista desde arriba"],
"low": ["cámara desde abajo", "míralo desde abajo", "vista desde abajo"]},
}
CAM_E = {
"en": {"static": ["stop the camera", "keep the camera still", "fix the camera", "freeze the camera"],
"orbit": ["orbit the camera", "rotate the camera", "spin the camera around", "make the camera orbit",
"camera orbit"],
"zoomin": ["animate a zoom in", "keep zooming in", "make the camera zoom in slowly"],
"zoomout": ["animate a zoom out", "keep zooming out", "make the camera zoom out slowly"],
"pan": ["pan the camera", "camera pan", "make the camera pan"],
"follow": ["follow {ref}", "make the camera follow {ref}", "camera follows {ref}", "track {ref}",
"follow it", "follow them with the camera"]},
"es": {"static": ["detén la cámara", "para la cámara", "deja la cámara quieta", "fija la cámara"],
"orbit": ["gira la cámara", "orbita la cámara", "que la cámara gire", "rota la cámara",
"que la cámara orbite"],
"zoomin": ["que la cámara se acerque lentamente", "anima un zoom in", "sigue acercando la cámara"],
"zoomout": ["que la cámara se aleje lentamente", "anima un zoom out", "sigue alejando la cámara"],
"pan": ["panea la cámara", "haz un paneo", "que la cámara panee", "mueve la cámara de lado"],
"follow": ["sigue {aref}", "que la cámara siga {aref}", "cámara siguiendo {aref}", "síguelo",
"síguelos con la cámara"]},
}
def bare(lang, p):
w = p.split(" ")
if w[0] in ("in", "on", "at", "en") and len(w) > 1:
return " ".join(w[1:])
return p
def edit_op(r, lang):
"""-> (text, tokens) for one edit operation."""
kind = r.choices(["color", "size", "act", "spd", "add", "del", "loc", "time", "weather", "shot", "cam", "mus"],
[16, 8, 14, 5, 16, 12, 7, 6, 7, 6, 5, 16])[0]
if kind == "mus":
return music_edit(r, lang)
e = {"type": r.choice(TYPE_KEYS)}
if lang == "en":
ref, g, pl = ref_en(r, e), "m", False
aref = ref
else:
ref, g, pl = ref_es(r, e)
aref = es_join("a", ref)
f = lambda key: r.choice(E[f"{key}_{lang}"])
def fill(t, **kw):
t = t.replace("{ref}", ref).replace("{aref}", aref)
for k, v in kw.items():
t = t.replace("{" + k + "}", v)
if lang == "es":
t = agree(t, g, pl).replace(" de el ", " del ").replace(" a el ", " al ")
return t
if kind == "color":
c = r.choice(COLOR_KEYS)
who = r.choices(["ref", "it", "all"], [0.72, 0.18, 0.10])[0]
key = {"ref": "color", "it": "color_it", "all": "color_all"}[who]
if lang == "en":
text = fill(f(key), c=r.choice(COLOR[c].en))
else:
cf = r.choice(COLOR[c].es)
text = fill(f(key), c=es_form(cf, g, pl), cm=cf[0], cf=cf[1], cmp=cf[2])
ty = {"ref": e["type"], "it": "it", "all": "all"}[who]
return text, ["<set>", f"type={ty}", f"color={c}"]
if kind == "size":
s = r.choice(SIZE_KEYS + ["up", "up", "down", "down"])
if s in ("up", "down"):
text = fill(f(s))
elif lang == "en":
text = fill(f("size"), s=r.choice(SIZE[s].en))
else:
text = fill(f("size"), s=es_form(r.choice(SIZE[s].es), g, pl))
return text, ["<set>", f"type={e['type']}", f"size={s}"]
if kind == "act":
a = r.choice(ACT_KEYS)
A = ACT[a]
if lang == "en":
text = fill(f("act"), base=r.choice(A.base), s3=r.choice(A.base if pl else A.s3), ing=r.choice(A.ing))
else:
text = fill(f("act"), subj=r.choice(A.subjp if pl else A.subj), e3=r.choice(A.e3p if pl else A.e3),
inf=r.choice(A.inf), ger=r.choice(A.ger))
return text, ["<set>", f"type={e['type']}", f"act={a}"]
if kind == "spd":
s = r.choices(["fast", "slow", "normal"], [4, 4, 1])[0]
return fill(f(s)), ["<set>", f"type={e['type']}", f"spd={s}"]
if kind == "add":
ne = rand_ent(r)
if lang == "en":
if "n" not in ne and r.random() < 0.08:
sing = r.choice(TYPES[ne["type"]].en)[0]
return f"add another {sing}", ["<add>", f"type={ne['type']}"]
np = np_en(r, ne, det=None if "n" in ne else "a")
else:
if "n" not in ne and r.random() < 0.08:
sing, _, g2 = r.choice(TYPES[ne["type"]].es)
return f"{r.choice(['añade', 'agrega', 'pon'])} {'otro' if g2 == 'm' else 'otra'} {sing}", \
["<add>", f"type={ne['type']}"]
np = np_es(r, ne, det=None if "n" in ne else "a")[0]
return f("add").replace("{np}", np), ent_tokens(ne, "<add>")
if kind == "del":
who = r.choices(["ref", "it", "all"], [0.72, 0.08, 0.20])[0]
if who == "ref":
return fill(f("del")), ["<del>", f"type={e['type']}"]
return f(f"del_{who}"), ["<del>", f"type={who}"]
if kind == "loc":
k = r.choice(list(LOC))
p = r.choice(getattr(LOC[k], lang))
return fill(f("loc"), p=p, bare=bare(lang, p)).replace(" a el ", " al "), ["<env>", f"loc={k}"]
if kind == "time":
k = r.choice(list(TIME))
if r.random() < 0.2:
return r.choice(TIME_X[lang][k]), ["<env>", f"time={k}"]
p = r.choice(getattr(TIME[k], lang))
return fill(f("time"), p=p, w=r.choice(TIME_W[lang][k])).replace(" a el ", " al "), ["<env>", f"time={k}"]
if kind == "weather":
k = r.choice(list(WEATHER))
if r.random() < 0.2:
return r.choice(getattr(WEATHER[k], lang)), ["<env>", f"weather={k}"]
return r.choice(WEATHER_E[lang][k]), ["<env>", f"weather={k}"]
if kind == "shot":
k = r.choice(list(SHOT_E[lang]))
if k in SHOT and r.random() < 0.4:
return r.choice(getattr(SHOT[k], lang)), ["<cam>", f"shot={k}"]
return r.choice(SHOT_E[lang][k]), ["<cam>", f"shot={k}"]
k = r.choice(list(CAM))
if r.random() < 0.35:
return r.choice(getattr(CAM[k], lang)), ["<cam>", f"cam={k}"]
return fill(r.choice(CAM_E[lang][k])), ["<cam>", f"cam={k}"]
def edit(r, lang):
n = 1 if r.random() < 0.82 else 2
texts, tokens = [], ["<edit>"]
for _ in range(n):
t, k = edit_op(r, lang)
texts.append(t)
tokens += k
join = {"en": [" and ", ", ", " and then ", ". "], "es": [" y ", ", ", " y luego ", ". "]}[lang]
text = r.choice(join).join(texts)
if r.random() < 0.08:
text = {"en": ["please ", "now ", "ok ", "can you "], "es": ["por favor ", "ahora ", "ok ", "puedes "]}[lang][r.randrange(4)] + text
if r.random() < 0.06:
text += {"en": " please", "es": " por favor"}[lang]
return text, tokens
# ------------------------------------------------------------------ questions
def qa(r, lang):
key = r.choice(list(TOPICS))
en, es, qen, qes = TOPICS[key][:4]
terms = (en if lang == "en" else es).split("|")
extra = (qen if lang == "en" else qes).split("|")
if r.random() < 0.45:
text = r.choice(extra)
else:
text = r.choice(Q_EN if lang == "en" else Q_ES).replace("{t}", r.choice(terms))
if lang == "es":
text = text.replace(" de el ", " del ")
if "three" not in text and key not in ("help", "objects", "actions", "about"):
text += r.choice(SUF_EN if lang == "en" else SUF_ES)
if r.random() < 0.5:
text += "?"
return text, ["<qa>", f"lang={lang}", f"topic={key}"]
# ------------------------------------------------------------------ noise
def strip_accents(s):
return "".join(c for c in unicodedata.normalize("NFD", s.replace("ñ", "\0"))
if unicodedata.category(c) != "Mn").replace("\0", "ñ")
def typo(r, s):
idx = [i for i, c in enumerate(s) if c.isalpha()]
if len(idx) < 4:
return s
i = r.choice(idx)
k = r.random()
if k < 0.3 and i + 1 < len(s) and s[i + 1].isalpha():
return s[:i] + s[i + 1] + s[i] + s[i + 2:]
if k < 0.6:
return s[:i] + s[i + 1:]
if k < 0.8:
return s[:i] + s[i] + s[i:]
return s[:i] + r.choice("abcdefghijklmnopqrstuvwxyz") + s[i + 1:]
def noise(r, s, lang):
if lang == "es" and r.random() < 0.45:
s = strip_accents(s)
if r.random() < 0.22:
s = typo(r, s)
if r.random() < 0.3:
s = typo(r, s)
if r.random() < 0.05:
s = s.replace(",", "")
if r.random() < 0.06 and not s.endswith("?"):
s += "."
return s
def sample(r, clean=False):
while True:
lang = "en" if r.random() < 0.5 else "es"
kind = r.choices([new_scene, edit, qa], [0.55, 0.33, 0.12])[0]
text, tokens = kind(r, lang)
if not clean:
text = noise(r, text, lang)
if len(text) <= MAX_PROMPT:
return text, tokens
if __name__ == "__main__":
import sys
rr = random.Random(int(sys.argv[1]) if len(sys.argv) > 1 else 0)
for _ in range(int(sys.argv[2]) if len(sys.argv) > 2 else 40):
t, k = sample(rr)
print(f"{t}\n -> {' '.join(k)}")