Upload jobs.py with huggingface_hub
Browse files
jobs.py
CHANGED
|
@@ -55,6 +55,24 @@ def _namespace(token: str) -> str:
|
|
| 55 |
return whoami(token=token)["name"]
|
| 56 |
|
| 57 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
def build_metadata(image_paths: list[str], captions: list[str], instance_prompt: str) -> list[dict]:
|
| 59 |
"""One `metadata.jsonl` row per image: {file_name, prompt}. Empty captions fall back to the
|
| 60 |
instance prompt (the trigger sentence). Files are renamed to a stable `0000.ext` order."""
|
|
@@ -72,7 +90,6 @@ def build_train_args(params: dict, hub_model_id: str) -> list[str]:
|
|
| 72 |
"""Turn UI params into the `train_dreambooth_lora_krea2.py` CLI. Krea weights are passed as
|
| 73 |
local dirs (pre-downloaded in the job); the dataset is a local imagefolder (image/prompt cols)."""
|
| 74 |
instance_prompt = (params.get("instance_prompt") or "TOK").strip()
|
| 75 |
-
val_prompt = (params.get("validation_prompt") or instance_prompt).strip()
|
| 76 |
args = [
|
| 77 |
"--pretrained_model_name_or_path", JOB_RAW,
|
| 78 |
"--validation_model_path", JOB_TURBO,
|
|
@@ -97,14 +114,9 @@ def build_train_args(params: dict, hub_model_id: str) -> list[str]:
|
|
| 97 |
"--push_to_hub",
|
| 98 |
"--hub_model_id", hub_model_id,
|
| 99 |
]
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
"--validation_epochs", str(int(params["validation_epochs"])),
|
| 104 |
-
"--num_validation_images", "2",
|
| 105 |
-
]
|
| 106 |
-
else:
|
| 107 |
-
args += ["--skip_final_inference"]
|
| 108 |
if params.get("lora_layers"):
|
| 109 |
args += ["--lora_layers", str(params["lora_layers"]).strip()]
|
| 110 |
if params.get("gradient_checkpointing", True):
|
|
@@ -127,7 +139,10 @@ def build_train_args(params: dict, hub_model_id: str) -> list[str]:
|
|
| 127 |
# UV job script (runs on HF Jobs GPU hardware)
|
| 128 |
# --------------------------------------------------------------------------------------
|
| 129 |
|
| 130 |
-
|
|
|
|
|
|
|
|
|
|
| 131 |
# requires-python = ">=3.10"
|
| 132 |
# dependencies = [
|
| 133 |
# "diffusers @ git+https://github.com/huggingface/diffusers.git@{ref}",
|
|
@@ -145,40 +160,165 @@ JOB_SCRIPT_TEMPLATE = '''# /// script
|
|
| 145 |
# "huggingface_hub[hf-xet]",
|
| 146 |
# ]
|
| 147 |
# ///
|
| 148 |
-
"""Auto-generated Krea 2 DreamBooth-LoRA job
|
| 149 |
-
import json, os, subprocess, sys, urllib.request
|
| 150 |
-
from pathlib import Path
|
| 151 |
-
|
| 152 |
-
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
| 153 |
-
|
| 154 |
REF = "{ref}"
|
| 155 |
-
RAW, TURBO, DATA, BNB = "{raw}", "{turbo}", "{data}", "{bnb}"
|
| 156 |
DATASET_REPO = {dataset_repo!r}
|
| 157 |
QUANT = {quant!r}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 158 |
TRAIN_ARGS = {train_args}
|
| 159 |
-
|
| 160 |
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 165 |
|
| 166 |
|
| 167 |
def main():
|
| 168 |
from huggingface_hub import snapshot_download
|
| 169 |
|
| 170 |
print("=== 1/4 download gated Krea 2 weights (krea token) ===", flush=True)
|
| 171 |
-
snapshot_download(
|
| 172 |
-
snapshot_download(
|
| 173 |
|
| 174 |
print("=== 2/4 download dataset (user token / HF_TOKEN env) ===", flush=True)
|
| 175 |
snapshot_download(DATASET_REPO, repo_type="dataset", local_dir=DATA)
|
| 176 |
|
| 177 |
if QUANT == "4bit":
|
| 178 |
-
Path(BNB).write_text(json.dumps({
|
| 179 |
"load_in_4bit": True, "bnb_4bit_quant_type": "nf4",
|
| 180 |
"bnb_4bit_compute_dtype": "bfloat16",
|
| 181 |
-
}
|
| 182 |
|
| 183 |
print("=== 3/4 fetch trainer script @ " + REF + " ===", flush=True)
|
| 184 |
urllib.request.urlretrieve(SCRIPT_URL, "/tmp/train_dreambooth_lora_krea2.py")
|
|
@@ -188,6 +328,13 @@ def main():
|
|
| 188 |
"/tmp/train_dreambooth_lora_krea2.py", *TRAIN_ARGS]
|
| 189 |
print(">>> " + " ".join(cmd), flush=True)
|
| 190 |
subprocess.run(cmd, check=True)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 191 |
print("=== DONE ===", flush=True)
|
| 192 |
|
| 193 |
|
|
@@ -226,22 +373,33 @@ def submit(params: dict, image_paths: list[str], captions: list[str],
|
|
| 226 |
api.create_repo(dataset_repo, repo_type="dataset", private=True, exist_ok=True, token=token)
|
| 227 |
api.upload_folder(repo_id=dataset_repo, repo_type="dataset", folder_path=str(tmp), token=token)
|
| 228 |
|
| 229 |
-
# render the job script
|
| 230 |
train_args = build_train_args(params, hub_model_id)
|
| 231 |
-
|
| 232 |
-
|
|
|
|
| 233 |
dataset_repo=dataset_repo, quant=params.get("quantization", "none"),
|
| 234 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 235 |
)
|
| 236 |
script_path = tmp / "job_train.py"
|
| 237 |
-
script_path.write_text(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 238 |
|
| 239 |
job = api.run_uv_job(
|
| 240 |
str(script_path),
|
| 241 |
flavor=flavor,
|
| 242 |
timeout=timeout,
|
| 243 |
-
|
| 244 |
-
secrets={"HF_TOKEN": token, "KREA_TOKEN": os.environ["KREA_TOKEN"]},
|
| 245 |
token=token,
|
| 246 |
)
|
| 247 |
job_id = getattr(job, "id", "") or ""
|
|
|
|
| 55 |
return whoami(token=token)["name"]
|
| 56 |
|
| 57 |
|
| 58 |
+
def _custom_prompts(params: dict, instance_prompt: str) -> list[str]:
|
| 59 |
+
"""Parse the optional custom eval-prompts textarea (one per line). `<trigger>` placeholders are
|
| 60 |
+
substituted with the actual trigger. Empty → [] → the job asks the LLM (or falls back)."""
|
| 61 |
+
raw = (params.get("custom_eval_prompts") or "").strip()
|
| 62 |
+
if not raw:
|
| 63 |
+
return []
|
| 64 |
+
trig = (instance_prompt or "").strip()
|
| 65 |
+
out = []
|
| 66 |
+
for line in raw.splitlines():
|
| 67 |
+
line = line.strip()
|
| 68 |
+
if not line:
|
| 69 |
+
continue
|
| 70 |
+
for ph in ("<trigger phrase>", "<trigger>", "<TOK>", "TOK"):
|
| 71 |
+
line = line.replace(ph, trig)
|
| 72 |
+
out.append(line)
|
| 73 |
+
return out
|
| 74 |
+
|
| 75 |
+
|
| 76 |
def build_metadata(image_paths: list[str], captions: list[str], instance_prompt: str) -> list[dict]:
|
| 77 |
"""One `metadata.jsonl` row per image: {file_name, prompt}. Empty captions fall back to the
|
| 78 |
instance prompt (the trigger sentence). Files are renamed to a stable `0000.ext` order."""
|
|
|
|
| 90 |
"""Turn UI params into the `train_dreambooth_lora_krea2.py` CLI. Krea weights are passed as
|
| 91 |
local dirs (pre-downloaded in the job); the dataset is a local imagefolder (image/prompt cols)."""
|
| 92 |
instance_prompt = (params.get("instance_prompt") or "TOK").strip()
|
|
|
|
| 93 |
args = [
|
| 94 |
"--pretrained_model_name_or_path", JOB_RAW,
|
| 95 |
"--validation_model_path", JOB_TURBO,
|
|
|
|
| 114 |
"--push_to_hub",
|
| 115 |
"--hub_model_id", hub_model_id,
|
| 116 |
]
|
| 117 |
+
# Previews are produced by our own post-training showcase step (a multi-prompt gallery + rich
|
| 118 |
+
# README), so the trainer itself always skips final inference.
|
| 119 |
+
args += ["--skip_final_inference"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
if params.get("lora_layers"):
|
| 121 |
args += ["--lora_layers", str(params["lora_layers"]).strip()]
|
| 122 |
if params.get("gradient_checkpointing", True):
|
|
|
|
| 139 |
# UV job script (runs on HF Jobs GPU hardware)
|
| 140 |
# --------------------------------------------------------------------------------------
|
| 141 |
|
| 142 |
+
# The script is rendered in two pieces: a small `.format()`-ed HEADER (PEP 723 metadata + config
|
| 143 |
+
# constants) and a verbatim BODY (raw string, never `.format()`-ed) so the showcase code's many
|
| 144 |
+
# dict / set literals don't need brace-escaping.
|
| 145 |
+
JOB_HEADER = '''# /// script
|
| 146 |
# requires-python = ">=3.10"
|
| 147 |
# dependencies = [
|
| 148 |
# "diffusers @ git+https://github.com/huggingface/diffusers.git@{ref}",
|
|
|
|
| 160 |
# "huggingface_hub[hf-xet]",
|
| 161 |
# ]
|
| 162 |
# ///
|
| 163 |
+
"""Auto-generated Krea 2 DreamBooth-LoRA job: trains on RAW, then renders a Turbo preview gallery."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
REF = "{ref}"
|
| 165 |
+
RAW, TURBO, DATA, BNB, OUT = "{raw}", "{turbo}", "{data}", "{bnb}", "{out}"
|
| 166 |
DATASET_REPO = {dataset_repo!r}
|
| 167 |
QUANT = {quant!r}
|
| 168 |
+
CONCEPT = {concept!r}
|
| 169 |
+
TRIGGER = {trigger!r}
|
| 170 |
+
MAKE_GALLERY = {make_gallery}
|
| 171 |
+
NUM_GALLERY = {num_gallery}
|
| 172 |
+
CUSTOM_PROMPTS = {custom_prompts}
|
| 173 |
+
HUB_MODEL_ID = {hub_model_id!r}
|
| 174 |
TRAIN_ARGS = {train_args}
|
| 175 |
+
'''
|
| 176 |
|
| 177 |
+
JOB_BODY = r'''
|
| 178 |
+
import json, os, re, subprocess, sys, urllib.request
|
| 179 |
+
from pathlib import Path
|
| 180 |
+
|
| 181 |
+
os.environ["HF_HUB_ENABLE_HF_TRANSFER"] = "1"
|
| 182 |
+
KREA_TOKEN = os.environ["KREA_TOKEN"] # gated Krea weights ONLY
|
| 183 |
+
CAPTION_TOKEN = os.environ.get("CAPTION_HF_TOKEN", "") # LLM eval-prompt generation (optional)
|
| 184 |
+
|
| 185 |
+
BASE_RAW_ID, TURBO_ID = "krea/Krea-2-Raw", "krea/Krea-2-Turbo"
|
| 186 |
+
CAPTION_MODEL, CAPTION_PROVIDER = "google/gemma-4-31B-it", "novita"
|
| 187 |
+
SCRIPT_URL = ("https://raw.githubusercontent.com/huggingface/diffusers/"
|
| 188 |
+
+ REF + "/examples/dreambooth/train_dreambooth_lora_krea2.py")
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
def gen_eval_prompts(n):
|
| 192 |
+
"""Ask the LLM for n diverse showcase prompts that include the trigger."""
|
| 193 |
+
from huggingface_hub import InferenceClient
|
| 194 |
+
if CONCEPT == "style":
|
| 195 |
+
instr = (
|
| 196 |
+
"Generate " + str(n) + " diverse, vivid text-to-image prompts to showcase an image "
|
| 197 |
+
"style. Each prompt must describe a COMPLETELY different subject and scene (an animal "
|
| 198 |
+
"in a landscape, a vehicle on a road, a person doing an action, an interior, etc.), be "
|
| 199 |
+
"1-2 sentences, and MUST end with this exact phrase: " + repr(TRIGGER) + ". Return ONLY "
|
| 200 |
+
"a JSON array of " + str(n) + " strings."
|
| 201 |
+
)
|
| 202 |
+
else:
|
| 203 |
+
instr = (
|
| 204 |
+
"Generate " + str(n) + " diverse, vivid text-to-image prompts to showcase a specific "
|
| 205 |
+
"subject. Each prompt must place the subject in a COMPLETELY different scene/context, "
|
| 206 |
+
"be 1-2 sentences, and MUST include this exact token: " + repr(TRIGGER) + ". Return ONLY "
|
| 207 |
+
"a JSON array of " + str(n) + " strings."
|
| 208 |
+
)
|
| 209 |
+
client = InferenceClient(model=CAPTION_MODEL, provider=CAPTION_PROVIDER, token=CAPTION_TOKEN)
|
| 210 |
+
r = client.chat_completion(messages=[{"role": "user", "content": instr}],
|
| 211 |
+
max_tokens=600, temperature=0.8)
|
| 212 |
+
txt = (r.choices[0].message.content or "").strip()
|
| 213 |
+
m = re.search(r"\[.*\]", txt, re.S)
|
| 214 |
+
if m:
|
| 215 |
+
prompts = [str(p).strip() for p in json.loads(m.group(0)) if str(p).strip()]
|
| 216 |
+
else:
|
| 217 |
+
prompts = [ln.strip().lstrip("-*0123456789. ").strip(' "') for ln in txt.splitlines()]
|
| 218 |
+
prompts = [p for p in prompts if p]
|
| 219 |
+
return prompts[:n] or [TRIGGER]
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def fallback_prompts():
|
| 223 |
+
if CONCEPT == "style":
|
| 224 |
+
return [
|
| 225 |
+
"A golden labrador running across a sunny farm, a pickup truck on a dusty road far behind, " + TRIGGER,
|
| 226 |
+
"A fishing boat moored in a narrow canal between tall old buildings, " + TRIGGER,
|
| 227 |
+
"A deer grazing in a dense forest with the bright sun overhead, " + TRIGGER,
|
| 228 |
+
]
|
| 229 |
+
return [
|
| 230 |
+
"A photo of " + TRIGGER + " on a wooden table indoors.",
|
| 231 |
+
"A photo of " + TRIGGER + " outdoors on a patch of grass.",
|
| 232 |
+
"A close-up photo of " + TRIGGER + " against a plain background.",
|
| 233 |
+
]
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def esc(s):
|
| 237 |
+
return str(s).replace("\\", "\\\\").replace('"', '\\"')
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
def write_readme(rows):
|
| 241 |
+
widget = "\n".join('- text: "' + esc(p) + '"\n output:\n url: ' + fn for fn, p in rows)
|
| 242 |
+
gallery = "\n".join("\n\n> *"' + p.replace('"', "'") + '"*\n' for fn, p in rows)
|
| 243 |
+
trig = ("the phrase `" + TRIGGER + "`") if CONCEPT == "style" else ("the token `" + TRIGGER + "`")
|
| 244 |
+
md = (
|
| 245 |
+
"---\n"
|
| 246 |
+
"base_model: " + BASE_RAW_ID + "\n"
|
| 247 |
+
"tags:\n- text-to-image\n- diffusers\n- lora\n- krea2\n- template:sd-lora\n"
|
| 248 |
+
"license: apache-2.0\n"
|
| 249 |
+
"instance_prompt: " + json.dumps(TRIGGER) + "\n"
|
| 250 |
+
"widget:\n" + widget + "\n"
|
| 251 |
+
"---\n\n"
|
| 252 |
+
"# Krea 2 LoRA — " + HUB_MODEL_ID + "\n\n"
|
| 253 |
+
"<Gallery />\n\n"
|
| 254 |
+
"A DreamBooth-LoRA for **Krea 2**, trained on **Krea 2 RAW** and shown on **Krea 2 Turbo**. "
|
| 255 |
+
"The samples below were generated with this LoRA on Turbo (8 steps).\n\n"
|
| 256 |
+
"## Trigger\n\nUse " + trig + " to invoke the concept.\n\n"
|
| 257 |
+
"## Samples\n\n" + gallery + "\n"
|
| 258 |
+
"## Use it with diffusers\n\n"
|
| 259 |
+
"```py\n"
|
| 260 |
+
"import torch\n"
|
| 261 |
+
"from diffusers import Krea2Pipeline\n\n"
|
| 262 |
+
'pipe = Krea2Pipeline.from_pretrained("' + TURBO_ID + '", torch_dtype=torch.bfloat16).to("cuda")\n'
|
| 263 |
+
'pipe.load_lora_weights("' + HUB_MODEL_ID + '")\n'
|
| 264 |
+
'image = pipe("' + esc(rows[0][1]) + '", num_inference_steps=8, guidance_scale=0.0, mu=1.15).images[0]\n'
|
| 265 |
+
'image.save("output.png")\n'
|
| 266 |
+
"```\n"
|
| 267 |
+
)
|
| 268 |
+
Path(os.path.join(OUT, "README.md")).write_text(md)
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def showcase():
|
| 272 |
+
import torch
|
| 273 |
+
from diffusers import Krea2Pipeline
|
| 274 |
+
from huggingface_hub import HfApi
|
| 275 |
+
|
| 276 |
+
prompts = list(CUSTOM_PROMPTS or [])
|
| 277 |
+
if not prompts and CAPTION_TOKEN:
|
| 278 |
+
try:
|
| 279 |
+
prompts = gen_eval_prompts(NUM_GALLERY)
|
| 280 |
+
except Exception as e:
|
| 281 |
+
print("=== eval-prompt generation failed, using fallbacks ===", repr(e), flush=True)
|
| 282 |
+
if not prompts:
|
| 283 |
+
prompts = fallback_prompts()
|
| 284 |
+
prompts = prompts[:max(1, NUM_GALLERY)]
|
| 285 |
+
|
| 286 |
+
print("=== showcase: load Turbo + trained LoRA, render " + str(len(prompts)) + " samples ===", flush=True)
|
| 287 |
+
pipe = Krea2Pipeline.from_pretrained(TURBO, torch_dtype=torch.bfloat16).to("cuda")
|
| 288 |
+
pipe.load_lora_weights(OUT)
|
| 289 |
+
|
| 290 |
+
rows = []
|
| 291 |
+
for i, p in enumerate(prompts):
|
| 292 |
+
print(">>> sample " + str(i) + ": " + p, flush=True)
|
| 293 |
+
image = pipe(p, num_inference_steps=8, guidance_scale=0.0, mu=1.15).images[0]
|
| 294 |
+
fn = "sample_" + str(i) + ".png"
|
| 295 |
+
image.save(os.path.join(OUT, fn))
|
| 296 |
+
rows.append((fn, p))
|
| 297 |
+
|
| 298 |
+
write_readme(rows)
|
| 299 |
+
|
| 300 |
+
api = HfApi() # ambient HF_TOKEN = user
|
| 301 |
+
for fn, _ in rows:
|
| 302 |
+
api.upload_file(path_or_fileobj=os.path.join(OUT, fn), path_in_repo=fn, repo_id=HUB_MODEL_ID)
|
| 303 |
+
api.upload_file(path_or_fileobj=os.path.join(OUT, "README.md"), path_in_repo="README.md", repo_id=HUB_MODEL_ID)
|
| 304 |
+
print("=== showcase: pushed gallery + README to " + HUB_MODEL_ID + " ===", flush=True)
|
| 305 |
|
| 306 |
|
| 307 |
def main():
|
| 308 |
from huggingface_hub import snapshot_download
|
| 309 |
|
| 310 |
print("=== 1/4 download gated Krea 2 weights (krea token) ===", flush=True)
|
| 311 |
+
snapshot_download(BASE_RAW_ID, local_dir=RAW, token=KREA_TOKEN)
|
| 312 |
+
snapshot_download(TURBO_ID, local_dir=TURBO, token=KREA_TOKEN)
|
| 313 |
|
| 314 |
print("=== 2/4 download dataset (user token / HF_TOKEN env) ===", flush=True)
|
| 315 |
snapshot_download(DATASET_REPO, repo_type="dataset", local_dir=DATA)
|
| 316 |
|
| 317 |
if QUANT == "4bit":
|
| 318 |
+
Path(BNB).write_text(json.dumps({
|
| 319 |
"load_in_4bit": True, "bnb_4bit_quant_type": "nf4",
|
| 320 |
"bnb_4bit_compute_dtype": "bfloat16",
|
| 321 |
+
}))
|
| 322 |
|
| 323 |
print("=== 3/4 fetch trainer script @ " + REF + " ===", flush=True)
|
| 324 |
urllib.request.urlretrieve(SCRIPT_URL, "/tmp/train_dreambooth_lora_krea2.py")
|
|
|
|
| 328 |
"/tmp/train_dreambooth_lora_krea2.py", *TRAIN_ARGS]
|
| 329 |
print(">>> " + " ".join(cmd), flush=True)
|
| 330 |
subprocess.run(cmd, check=True)
|
| 331 |
+
|
| 332 |
+
if MAKE_GALLERY:
|
| 333 |
+
try:
|
| 334 |
+
showcase()
|
| 335 |
+
except Exception as e:
|
| 336 |
+
print("=== showcase failed (LoRA already trained + pushed) ===", repr(e), flush=True)
|
| 337 |
+
|
| 338 |
print("=== DONE ===", flush=True)
|
| 339 |
|
| 340 |
|
|
|
|
| 373 |
api.create_repo(dataset_repo, repo_type="dataset", private=True, exist_ok=True, token=token)
|
| 374 |
api.upload_folder(repo_id=dataset_repo, repo_type="dataset", folder_path=str(tmp), token=token)
|
| 375 |
|
| 376 |
+
# render the job script (formatted header + verbatim body)
|
| 377 |
train_args = build_train_args(params, hub_model_id)
|
| 378 |
+
custom_prompts = _custom_prompts(params, instance_prompt=params.get("instance_prompt", ""))
|
| 379 |
+
header = JOB_HEADER.format(
|
| 380 |
+
ref=DIFFUSERS_REF, raw=JOB_RAW, turbo=JOB_TURBO, data=JOB_DATA, bnb=JOB_BNB, out=JOB_OUT,
|
| 381 |
dataset_repo=dataset_repo, quant=params.get("quantization", "none"),
|
| 382 |
+
concept=params.get("concept_type", "style"),
|
| 383 |
+
trigger=(params.get("instance_prompt") or "").strip(),
|
| 384 |
+
make_gallery=bool(params.get("make_gallery", True)),
|
| 385 |
+
num_gallery=int(params.get("num_gallery_images", 3) or 3),
|
| 386 |
+
custom_prompts=repr(custom_prompts),
|
| 387 |
+
hub_model_id=hub_model_id, train_args=json.dumps(train_args),
|
| 388 |
)
|
| 389 |
script_path = tmp / "job_train.py"
|
| 390 |
+
script_path.write_text(header + JOB_BODY)
|
| 391 |
+
|
| 392 |
+
# HF_TOKEN = user (push + dataset); KREA_TOKEN = gated Krea weights only;
|
| 393 |
+
# CAPTION_HF_TOKEN = LLM eval-prompt generation for the showcase (optional).
|
| 394 |
+
secrets = {"HF_TOKEN": token, "KREA_TOKEN": os.environ["KREA_TOKEN"]}
|
| 395 |
+
if os.environ.get("CAPTION_HF_TOKEN"):
|
| 396 |
+
secrets["CAPTION_HF_TOKEN"] = os.environ["CAPTION_HF_TOKEN"]
|
| 397 |
|
| 398 |
job = api.run_uv_job(
|
| 399 |
str(script_path),
|
| 400 |
flavor=flavor,
|
| 401 |
timeout=timeout,
|
| 402 |
+
secrets=secrets,
|
|
|
|
| 403 |
token=token,
|
| 404 |
)
|
| 405 |
job_id = getattr(job, "id", "") or ""
|