ReCoEdit-Demo / app.py
Matteoooo46's picture
Restore original Space app
30bf72d verified
Raw History Blame Contribute Delete
12.4 kB
from __future__ import annotations
import glob
import os
import re
import time
from pathlib import Path
import gradio as gr
import spaces
import torch
from diffsynth.pipelines.qwen_image import ModelConfig, QwenImagePipeline
from huggingface_hub import snapshot_download
from peft import PeftModel
from PIL import Image, ImageOps
from transformers import AutoProcessor, Qwen3VLForConditionalGeneration
from apg_guidance import patch_pipeline_for_apg
BASE_MODEL_ID = os.getenv("BASE_MODEL_ID", "Qwen/Qwen-Image-Edit-2511")
IMAGE_LORA_ID = os.getenv("IMAGE_LORA_ID", "Matteoooo46/ReCoEdit-RL")
REWRITER_BASE_ID = os.getenv("REWRITER_BASE_ID", "Qwen/Qwen3-VL-8B-Instruct")
REWRITER_LORA_ID = os.getenv("REWRITER_LORA_ID", "Matteoooo46/ReCoEdit-rewriter")
HF_TOKEN = os.getenv("HF_TOKEN")
VRAM_LIMIT_GB = float(os.getenv("VRAM_LIMIT_GB", "72"))
NO_TEXT_SUFFIX = (
",画面中不要出现任何多余的文字、字幕、水印、签名、乱码或无关logo,"
"仅保留商品原本的品牌文字且保持清晰一致。"
)
NEGATIVE_PROMPT = (
"text, watermark, signature, letters, writing, words, typography, "
"logo, brand name, garbled characters, subtitles"
)
PROMPT_REWRITE_SYSTEM = """
你是一个电商商品图像编辑 prompt 改写助手。你的任务是根据商品参考图和 original prompt,生成一段更清晰、更具体、更适合图像编辑模型执行的中文 prompt。
请遵守以下原则:
1. 以 original prompt 的编辑意图为主。保留并扩写场景、背景、光线、氛围、构图、人物、动作、道具和风格要求,不要改变原始编辑目标。
2. 适度补充商品本体的类别、形状、颜色、材质、logo、文字、图案和关键结构细节,以帮助模型保持商品一致性。
3. 参考图只用于识别商品本体。不要描述参考图的原始背景、摆放方式、手、模特或装饰物,除非 original prompt 明确要求保留。
4. 最终 prompt 中,编辑目标约占 50%,商品细节约占 40%,质量约束约占 10%;总长度控制在 120~150 个中文字符,不要列小标题或分点。
5. 保持商品颜色、形状、logo、文字、图案和材质与参考图一致;不要新增无关文字、水印、乱码;不要让商品变形、镜像、错色或丢失关键标识。只描述单帧静态画面。
只输出一条最终 prompt,格式如下:
i2v描述:<优化后的 prompt>
""".strip()
def _download(repo_id: str) -> str:
return snapshot_download(repo_id, token=HF_TOKEN)
def _weight_files(root: str, subfolder: str) -> list[str]:
folder = Path(root) / subfolder
files = sorted(str(path) for path in folder.glob("*.safetensors"))
if not files:
raise FileNotFoundError(f"在 {folder} 中没有找到 safetensors 权重。")
return files
def _adapter_file(root: str) -> str:
preferred = Path(root) / "adapter_model.safetensors"
if preferred.exists():
return str(preferred)
files = sorted(glob.glob(str(Path(root) / "**" / "*.safetensors"), recursive=True))
if len(files) != 1:
raise FileNotFoundError(f"无法在 {root} 中唯一确定图像 LoRA 权重。")
return files[0]
print("正在下载并加载 ReCoEdit 模型……")
base_dir = _download(BASE_MODEL_ID)
image_lora_dir = _download(IMAGE_LORA_ID)
vram_config = {
"offload_dtype": "disk",
"offload_device": "disk",
"onload_dtype": torch.float8_e4m3fn,
"onload_device": "cpu",
"preparing_dtype": torch.float8_e4m3fn,
"preparing_device": "cuda",
"computation_dtype": torch.bfloat16,
"computation_device": "cuda",
}
image_pipe = QwenImagePipeline.from_pretrained(
torch_dtype=torch.bfloat16,
device="cuda",
model_configs=[
ModelConfig(path=_weight_files(base_dir, "transformer"), **vram_config),
ModelConfig(path=_weight_files(base_dir, "text_encoder"), **vram_config),
ModelConfig(
path=str(Path(base_dir) / "vae" / "diffusion_pytorch_model.safetensors"),
**vram_config,
),
],
tokenizer_config=None,
processor_config=ModelConfig(path=str(Path(base_dir) / "processor")),
vram_limit=VRAM_LIMIT_GB,
)
patch_pipeline_for_apg(image_pipe, eta=0.0, beta=-0.5, norm_threshold=None)
image_lora_loaded = False
rewriter = Qwen3VLForConditionalGeneration.from_pretrained(
REWRITER_BASE_ID,
torch_dtype=torch.bfloat16,
device_map="cuda:0",
token=HF_TOKEN,
trust_remote_code=True,
)
rewriter.eval()
rewriter_lora_loaded = False
rewriter_processor = AutoProcessor.from_pretrained(
REWRITER_BASE_ID,
token=HF_TOKEN,
trust_remote_code=True,
)
print("ReCoEdit 模型加载完成。")
def _clean_rewritten_prompt(text: str) -> str:
match = re.search(r"i2v描述\s*[::]\s*(.+)", text or "", flags=re.DOTALL)
value = match.group(1) if match else (text or "")
return re.sub(r"\s+", " ", value.strip().strip("`").strip())
def _load_reference_images(files) -> list[Image.Image]:
if not files:
raise gr.Error("请至少上传一张商品参考图。")
if len(files) > 4:
raise gr.Error("最多支持 4 张参考图。")
images = []
for file in files:
path = file if isinstance(file, str) else getattr(file, "path", None) or file.name
image = ImageOps.exif_transpose(Image.open(path)).convert("RGB")
image.thumbnail((1536, 1536), Image.Resampling.LANCZOS)
images.append(image)
return images
def rewrite_prompt(original_prompt: str, images: list[Image.Image]) -> tuple[str, str]:
content = [{"type": "image", "image": image.copy()} for image in images]
content.append(
{
"type": "text",
"text": PROMPT_REWRITE_SYSTEM + "\n\noriginal prompt:" + original_prompt,
}
)
messages = [{"role": "user", "content": content}]
chat_text = rewriter_processor.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
inputs = rewriter_processor(
text=[chat_text],
images=images,
return_tensors="pt",
).to(rewriter.device)
with torch.inference_mode():
generated_ids = rewriter.generate(
**inputs,
max_new_tokens=512,
do_sample=False,
)
output_ids = generated_ids[0][inputs["input_ids"].shape[1] :]
raw_answer = rewriter_processor.tokenizer.decode(
output_ids,
skip_special_tokens=True,
).strip()
rewritten = _clean_rewritten_prompt(raw_answer) or original_prompt
return rewritten, raw_answer
def _parse_size(size: str) -> tuple[int, int]:
width, height = size.split("×")
return int(width), int(height)
def _gpu_duration(_files, _prompt, _size, steps, *_args):
# ZeroGPU charges xlarge requests at 2x duration, so the raw request must
# stay at or below 150 seconds to remain within the 300-second limit.
return min(150, max(120, int(steps) * 3))
def _ensure_image_lora():
"""Load the image LoRA only after ZeroGPU has assigned a real GPU."""
global image_lora_loaded
if not image_lora_loaded:
image_pipe.load_lora(image_pipe.dit, _adapter_file(image_lora_dir))
image_lora_loaded = True
def _ensure_rewriter_lora():
"""Load the prompt-rewriter LoRA after ZeroGPU has assigned a real GPU."""
global rewriter, rewriter_lora_loaded
if not rewriter_lora_loaded:
rewriter = PeftModel.from_pretrained(
rewriter,
REWRITER_LORA_ID,
token=HF_TOKEN,
)
rewriter.eval()
rewriter_lora_loaded = True
@spaces.GPU(size="xlarge", duration=_gpu_duration)
def generate(files, prompt, size, steps, cfg_scale, seed, progress=gr.Progress()):
prompt = (prompt or "").strip()
if not prompt:
raise gr.Error("请输入希望如何编辑商品图。")
images = _load_reference_images(files)
width, height = _parse_size(size)
started = time.time()
progress(0.03, desc="正在载入 ReCoEdit 图像权重")
_ensure_image_lora()
progress(0.06, desc="正在载入提示词改写权重")
_ensure_rewriter_lora()
progress(0.08, desc="正在分析商品与编辑目标")
try:
rewritten, raw_answer = rewrite_prompt(prompt, images)
except Exception as exc:
print(f"Prompt 改写失败,回退到原始提示词:{exc}")
rewritten, raw_answer = prompt, "改写模型不可用,本次已使用原始提示词。"
progress(0.25, desc="提示词已优化,正在生成图片")
final_prompt = rewritten + NO_TEXT_SUFFIX
with torch.inference_mode():
result = image_pipe(
prompt=final_prompt,
negative_prompt=NEGATIVE_PROMPT,
edit_image=images,
seed=int(seed),
num_inference_steps=int(steps),
height=height,
width=width,
edit_image_auto_resize=True,
zero_cond_t=True,
cfg_scale=float(cfg_scale),
)
elapsed = time.time() - started
details = (
f"优化后的提示词:\n{rewritten}\n\n"
f"耗时:{elapsed:.1f} 秒|尺寸:{width}×{height}|"
f"步数:{int(steps)}|CFG:{float(cfg_scale):.1f}|Seed:{int(seed)}"
)
progress(1.0, desc="生成完成")
return details, result
CSS = """
.gradio-container {max-width: 1260px !important;}
.hero {padding: 12px 0 8px;}
.hero h1 {font-size: 2.2rem; margin-bottom: .3rem;}
.hero p {color: var(--body-text-color-subdued); font-size: 1.02rem;}
#generate-button {min-height: 48px; font-weight: 700;}
#result-image {min-height: 560px;}
.model-note {border-left: 4px solid #7c3aed; padding-left: 12px;}
"""
with gr.Blocks(
theme=gr.themes.Soft(primary_hue="violet", secondary_hue="blue"),
css=CSS,
title="ReCoEdit — 商品图像编辑",
) as demo:
gr.Markdown(
"""
<div class="hero">
<h1>ReCoEdit 商品图像编辑</h1>
<p>上传商品参考图并描述目标画面。模型会先优化编辑提示词,再生成保持商品一致性的结果。</p>
</div>
"""
)
with gr.Row(equal_height=False):
with gr.Column(scale=5):
reference_files = gr.File(
label="商品参考图(1–4 张)",
file_count="multiple",
file_types=["image"],
type="filepath",
)
prompt_input = gr.Textbox(
label="编辑指令",
placeholder="例如:把这款口红放在高级黑色梳妆台上,暖色侧光,商业广告摄影风格……",
lines=6,
)
with gr.Accordion("生成参数", open=False):
size_input = gr.Dropdown(
["896×1152", "1024×1024", "1152×896"],
value="896×1152",
label="输出尺寸",
)
steps_input = gr.Slider(20, 50, value=40, step=1, label="推理步数")
cfg_input = gr.Slider(1.0, 7.5, value=4.0, step=0.5, label="APG / CFG 强度")
seed_input = gr.Number(value=1, precision=0, label="随机种子")
generate_button = gr.Button("开始生成", variant="primary", elem_id="generate-button")
gr.Markdown(
"""
<div class="model-note">
当前流程:Qwen3-VL Rewriter LoRA → Qwen-Image-Edit-2511 + ReCoEdit-RL LoRA → APG。
</div>
"""
)
with gr.Column(scale=7):
image_output = gr.Image(
label="生成结果",
type="pil",
format="png",
interactive=False,
elem_id="result-image",
)
prompt_output = gr.Textbox(
label="提示词优化结果与参数",
lines=7,
interactive=False,
)
generate_button.click(
fn=generate,
inputs=[
reference_files,
prompt_input,
size_input,
steps_input,
cfg_input,
seed_input,
],
outputs=[prompt_output, image_output],
api_name="generate",
)
demo.queue(default_concurrency_limit=1, max_size=12)
if __name__ == "__main__":
demo.launch()