Buckets:

hf-doc-build
/

doc

Files
xet
hf-doc-build/doc / diffusers /main /vi /using-diffusers /controlnet.html
mishig
3 days ago
download
raw
42.9 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;ControlNet&quot;,&quot;local&quot;:&quot;controlnet&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Multi-ControlNet&quot;,&quot;local&quot;:&quot;multi-controlnet&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Chế độ đoán&quot;,&quot;local&quot;:&quot;guess-mode&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
		<link href="/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/I9eooN77.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/Bupkjr9f.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CyvZ3FS5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/l0QnnOED.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DnVIJfC5.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/0.DteTvmQZ.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/CoSrdQax.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/nodes/391.CRavWgDP.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/DSPDl1gz.js" rel="modulepreload">
		<link href="/docs/diffusers/main/en/_app/immutable/chunks/D5eijdnr.js" rel="modulepreload">
		<!--18lijse--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;ControlNet&quot;,&quot;local&quot;:&quot;controlnet&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;Multi-ControlNet&quot;,&quot;local&quot;:&quot;multi-controlnet&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;Guess mode&quot;,&quot;local&quot;:&quot;guess-mode&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
		<link href="/docs/diffusers/main/en/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: inline-end; margin-inline-start: 10px; display: inline-flex; position: relative; z-index: 10;"><div dir="ltr" class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span dir="auto">Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="controlnet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#controlnet"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->ControlNet<!----></span></h1><!--]--><!----> <p><a href="https://huggingface.co/papers/2302.05543" rel="nofollow">ControlNet</a> điều hướng một mô hình khuếch tán được huấn luyện trước bằng một hình ảnh kiểm soát cấu trúc (đường viền, độ sâu hoặc tư thế) trong khi bạn vẫn giữ nguyên câu lệnh văn bản. Nó đóng băng mô hình cơ sở và thêm một mạng song song, trong đó các phần dư (residuals) của mạng này dẫn dắt bộ khử nhiễu (denoiser), nhờ đó bạn có được khả năng kiểm soát bố cục mà không cần tinh chỉnh (fine-tuning).</p> <p>Tải một ControlNet cho loại điều khiển bạn cần (ví dụ canny), sau đó truyền nó dưới dạng <code>controlnet=</code> vào <a href="/docs/diffusers/main/vi/api/pipelines/overview#diffusers.DiffusionPipeline.from_pretrained">from_pretrained()</a>. Sử dụng <code>controlnet_conditioning_scale</code> để thiết lập mức độ mạnh mà điều khiển định hướng quá trình tạo.</p> <blockquote class="tip"><p>ControlNet có sẵn cho nhiều mô hình như <a href="../api/pipelines/controlnet_flux">Flux</a>, <a href="../api/pipelines/controlnet_hunyuandit">Hunyuan-DiT</a>, <a href="../api/pipelines/controlnet_sd3">Stable Diffusion 3</a> và nhiều hơn nữa. Các ví dụ trong hướng dẫn này sử dụng Flux và Stable Diffusion XL. Đối với T2I-Adapter trên các checkpoint thuộc họ Stable Diffusion cũ hơn, hãy xem <a href="./legacy_adapters#t2i-adapter">Legacy adapters</a>. Ưu tiên sử dụng ControlNet cho các công việc tạo ảnh có kiểm soát mới.</p></blockquote> <p>Các tab bên dưới hiển thị ControlNet cho text-to-image, image-to-image và inpainting.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd border-gray-800 bg-black dark:bg-gray-700 text-white">text-to-image</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">image-to-image</div><div class="flex items-center border rounded-lg px-1.5 py-1 leading-none select-none text-smd text-gray-500 cursor-pointer opacity-90 hover:text-gray-700 dark:hover:text-gray-200 hover:shadow-sm">inpainting</div><!--]--></div> <div class="language-select"><div><p>Tạo ảnh canny bằng <a href="https://github.com/opencv/opencv-python" rel="nofollow">opencv-python</a>.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> cv2
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image

original_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/non-enhanced-prompt.png&quot;</span>
)

image = np.array(original_image)

low_threshold = <span class="hljs-number">100</span>
high_threshold = <span class="hljs-number">200</span>

image = cv2.Canny(image, low_threshold, high_threshold)
image = image[:, :, <span class="hljs-literal">None</span>]
image = np.concatenate([image, image, image], axis=<span class="hljs-number">2</span>)
canny_image = Image.fromarray(image)<!----></pre></div><!----> <p>Truyền ảnh canny vào pipeline. Sử dụng tham số <code>controlnet_conditioning_scale</code> để xác định mức độ ưu tiên gán cho điều khiển.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> FluxControlNetPipeline, FluxControlNetModel

controlnet = FluxControlNetModel.from_pretrained(
    <span class="hljs-string">&quot;InstantX/FLUX.1-dev-Controlnet-Canny&quot;</span>, dtype=torch.bfloat16
)
pipeline = FluxControlNetPipeline.from_pretrained(
    <span class="hljs-string">&quot;black-forest-labs/FLUX.1-dev&quot;</span>, controlnet=controlnet, dtype=torch.bfloat16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

prompt = <span class="hljs-string">&quot;&quot;&quot;
A photorealistic overhead image of a cat reclining sideways in a flamingo pool floatie holding a margarita. 
The cat is floating leisurely in the pool and completely relaxed and happy.
&quot;&quot;&quot;</span>

pipeline(
    prompt, 
    control_image=canny_image,
    controlnet_conditioning_scale=<span class="hljs-number">0.5</span>,
    num_inference_steps=<span class="hljs-number">50</span>, 
    guidance_scale=<span class="hljs-number">3.5</span>,
).images[<span class="hljs-number">0</span>]<!----></pre></div><!----> <div style="display: flex; gap: 10px; justify-content: space-around; align-items: flex-end;"><figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/non-enhanced-prompt.png" width="300" alt="Generated image (prompt only)"/> <figcaption style="text-align: center;">ảnh gốc</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/canny-cat.png" width="300" alt="Control image (Canny edges)"/> <figcaption style="text-align: center;">ảnh canny</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/canny-cat-generated.png" width="300" alt="Generated image (ControlNet + prompt)"/> <figcaption style="text-align: center;">hình ảnh được tạo</figcaption></figure></div><!----></div><!----> <div hidden=""><p>Tạo bản đồ độ sâu bằng pipeline ước lượng độ sâu từ Transformers.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DPTImageProcessor, DPTForDepthEstimation
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> ControlNetModel, StableDiffusionXLControlNetImg2ImgPipeline, AutoencoderKL
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image


device = <span class="hljs-string">&quot;cuda&quot;</span>  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>
depth_estimator = DPTForDepthEstimation.from_pretrained(<span class="hljs-string">&quot;Intel/dpt-hybrid-midas&quot;</span>).to(device)
feature_extractor = DPTImageProcessor.from_pretrained(<span class="hljs-string">&quot;Intel/dpt-hybrid-midas&quot;</span>)

<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_depth_map</span>(<span class="hljs-params">image</span>):
    image = feature_extractor(images=image, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).pixel_values.to(device)
    <span class="hljs-keyword">with</span> torch.no_grad(), torch.autocast(device):
        depth_map = depth_estimator(image).predicted_depth

    depth_map = torch.nn.functional.interpolate(
        depth_map.unsqueeze(<span class="hljs-number">1</span>),
        size=(<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>),
        mode=<span class="hljs-string">&quot;bicubic&quot;</span>,
        align_corners=<span class="hljs-literal">False</span>,
    )
    depth_min = torch.amin(depth_map, dim=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>], keepdim=<span class="hljs-literal">True</span>)
    depth_max = torch.amax(depth_map, dim=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>], keepdim=<span class="hljs-literal">True</span>)
    depth_map = (depth_map - depth_min) / (depth_max - depth_min)
    image = torch.cat([depth_map] * <span class="hljs-number">3</span>, dim=<span class="hljs-number">1</span>)
    image = image.permute(<span class="hljs-number">0</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">1</span>).cpu().numpy()[<span class="hljs-number">0</span>]
    image = Image.fromarray((image * <span class="hljs-number">255.0</span>).clip(<span class="hljs-number">0</span>, <span class="hljs-number">255</span>).astype(np.uint8))
    <span class="hljs-keyword">return</span> image

image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/non-enhanced-prompt.png&quot;</span>
).resize((<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>))
depth_image = get_depth_map(image)<!----></pre></div><!----> <p>Truyền bản đồ độ sâu vào pipeline. Sử dụng tham số <code>controlnet_conditioning_scale</code> để xác định mức độ ưu tiên gán cho điều khiển.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-py "><!---->controlnet = ControlNetModel.from_pretrained(
    <span class="hljs-string">&quot;diffusers/controlnet-depth-sdxl-1.0-small&quot;</span>,
    dtype=torch.float16,
)
vae = AutoencoderKL.from_pretrained(<span class="hljs-string">&quot;madebyollin/sdxl-vae-fp16-fix&quot;</span>, dtype=torch.float16)
pipeline = StableDiffusionXLControlNetImg2ImgPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
    controlnet=controlnet,
    vae=vae,
    dtype=torch.float16,
).to(device)

prompt = <span class="hljs-string">&quot;&quot;&quot;
A photorealistic overhead image of a cat reclining sideways in a flamingo pool floatie holding a margarita. 
The cat is floating leisurely in the pool and completely relaxed and happy.
&quot;&quot;&quot;</span>
controlnet_conditioning_scale = <span class="hljs-number">0.5</span>
pipeline(
    prompt,
    image=image,
    control_image=depth_image,
    controlnet_conditioning_scale=controlnet_conditioning_scale,
    strength=<span class="hljs-number">0.99</span>,
    num_inference_steps=<span class="hljs-number">100</span>,
).images[<span class="hljs-number">0</span>]<!----></pre></div><!----> <div style="display: flex; gap: 10px; justify-content: space-around; align-items: flex-end;"><figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/non-enhanced-prompt.png" width="300" alt="Generated image (prompt only)"/> <figcaption style="text-align: center;">ảnh gốc</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl_depth_image.png" width="300" alt="Depth map"/> <figcaption style="text-align: center;">bản đồ độ sâu</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl_depth_cat.png" width="300" alt="Generated image (ControlNet + prompt)"/> <figcaption style="text-align: center;">hình ảnh được tạo</figcaption></figure></div><!----></div><!----> <div hidden=""><p>Tạo một ảnh mask đánh dấu các pixel cần thay thế, sau đó truyền nó dưới dạng <code>mask_image</code> cùng với ảnh điều khiển.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> cv2
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLControlNetInpaintPipeline, ControlNetModel

init_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/non-enhanced-prompt.png&quot;</span>
)
init_image = init_image.resize((<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>))
mask_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat_mask.png&quot;</span>
)
mask_image = mask_image.resize((<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>))

<span class="hljs-keyword">def</span> <span class="hljs-title function_">make_canny_condition</span>(<span class="hljs-params">image</span>):
    image = np.array(image)
    image = cv2.Canny(image, <span class="hljs-number">100</span>, <span class="hljs-number">200</span>)
    image = image[:, :, <span class="hljs-literal">None</span>]
    image = np.concatenate([image, image, image], axis=<span class="hljs-number">2</span>)
    image = Image.fromarray(image)
    <span class="hljs-keyword">return</span> image

control_image = make_canny_condition(init_image)<!----></pre></div><!----> <p>Truyền mask và ảnh điều khiển vào pipeline. Sử dụng tham số <code>controlnet_conditioning_scale</code> để xác định mức độ ưu tiên gán cho điều khiển.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-py "><!---->controlnet = ControlNetModel.from_pretrained(
    <span class="hljs-string">&quot;diffusers/controlnet-canny-sdxl-1.0&quot;</span>, dtype=torch.float16
)
pipeline = StableDiffusionXLControlNetInpaintPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>, controlnet=controlnet, dtype=torch.float16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>
pipeline(
    <span class="hljs-string">&quot;a cute and fluffy bunny rabbit&quot;</span>,
    num_inference_steps=<span class="hljs-number">100</span>,
    strength=<span class="hljs-number">0.99</span>,
    controlnet_conditioning_scale=<span class="hljs-number">0.5</span>,
    image=init_image,
    mask_image=mask_image,
    control_image=control_image,
).images[<span class="hljs-number">0</span>]<!----></pre></div><!----> <div style="display: flex; gap: 10px; justify-content: space-around; align-items: flex-end;"><figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/non-enhanced-prompt.png" width="300" alt="Generated image (prompt only)"/> <figcaption style="text-align: center;">ảnh gốc</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat_mask.png" width="300" alt="Mask image"/> <figcaption style="text-align: center;">mask image</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl_rabbit_inpaint.png" width="300" alt="Generated image (ControlNet + prompt)"/> <figcaption style="text-align: center;">hình ảnh được tạo</figcaption></figure></div><!----></div><!----><!----></div><!----> <!--[1--><h2 class="relative group"><a id="multi-controlnet" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#multi-controlnet"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Multi-ControlNet<!----></span></h2><!--]--><!----> <p>Kết hợp nhiều ControlNet (ví dụ canny + depth) bằng cách truyền chúng dưới dạng một danh sách. Che mờ các vùng chồng lấn khi có thể và tinh chỉnh từng <code>controlnet_conditioning_scale</code>. Tạo <code>canny_image</code> bằng OpenCV Canny (như trong text-to-image) và <code>depth_image</code> bằng DPT (như trong image-to-image). Thay đổi kích thước mọi ảnh điều khiển về kích thước của pipeline trước khi gọi pipeline.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> cv2
<span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">import</span> numpy <span class="hljs-keyword">as</span> np
<span class="hljs-keyword">from</span> PIL <span class="hljs-keyword">import</span> Image
<span class="hljs-keyword">from</span> transformers <span class="hljs-keyword">import</span> DPTImageProcessor, DPTForDepthEstimation
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image

original_image = load_image(
    <span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/non-enhanced-prompt.png&quot;</span>
)
image = np.array(original_image)
canny_image = Image.fromarray(
    np.concatenate([cv2.Canny(image, <span class="hljs-number">100</span>, <span class="hljs-number">200</span>)[:, :, <span class="hljs-literal">None</span>]] * <span class="hljs-number">3</span>, axis=<span class="hljs-number">2</span>)
)

device = <span class="hljs-string">&quot;cuda&quot;</span>  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>
depth_estimator = DPTForDepthEstimation.from_pretrained(<span class="hljs-string">&quot;Intel/dpt-hybrid-midas&quot;</span>).to(device)
feature_extractor = DPTImageProcessor.from_pretrained(<span class="hljs-string">&quot;Intel/dpt-hybrid-midas&quot;</span>)

<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_depth_map</span>(<span class="hljs-params">image</span>):
    image = feature_extractor(images=image, return_tensors=<span class="hljs-string">&quot;pt&quot;</span>).pixel_values.to(device)
    <span class="hljs-keyword">with</span> torch.no_grad(), torch.autocast(device):
        depth_map = depth_estimator(image).predicted_depth

    depth_map = torch.nn.functional.interpolate(
        depth_map.unsqueeze(<span class="hljs-number">1</span>),
        size=(<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>),
        mode=<span class="hljs-string">&quot;bicubic&quot;</span>,
        align_corners=<span class="hljs-literal">False</span>,
    )
    depth_min = torch.amin(depth_map, dim=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>], keepdim=<span class="hljs-literal">True</span>)
    depth_max = torch.amax(depth_map, dim=[<span class="hljs-number">1</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>], keepdim=<span class="hljs-literal">True</span>)
    depth_map = (depth_map - depth_min) / (depth_max - depth_min)
    image = torch.cat([depth_map] * <span class="hljs-number">3</span>, dim=<span class="hljs-number">1</span>)
    image = image.permute(<span class="hljs-number">0</span>, <span class="hljs-number">2</span>, <span class="hljs-number">3</span>, <span class="hljs-number">1</span>).cpu().numpy()[<span class="hljs-number">0</span>]
    image = Image.fromarray((image * <span class="hljs-number">255.0</span>).clip(<span class="hljs-number">0</span>, <span class="hljs-number">255</span>).astype(np.uint8))
    <span class="hljs-keyword">return</span> image

depth_image = get_depth_map(original_image.resize((<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>)))

controlnets = [
    ControlNetModel.from_pretrained(
        <span class="hljs-string">&quot;diffusers/controlnet-canny-sdxl-1.0&quot;</span>, dtype=torch.float16
    ),
    ControlNetModel.from_pretrained(
        <span class="hljs-string">&quot;diffusers/controlnet-depth-sdxl-1.0-small&quot;</span>, dtype=torch.float16
    ),
]

vae = AutoencoderKL.from_pretrained(<span class="hljs-string">&quot;madebyollin/sdxl-vae-fp16-fix&quot;</span>, dtype=torch.float16)
pipeline = StableDiffusionXLControlNetPipeline.from_pretrained(
    <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>, controlnet=controlnets, vae=vae, dtype=torch.float16
).to(device)

prompt = <span class="hljs-string">&quot;&quot;&quot;
a relaxed rabbit sitting on a striped towel next to a pool with a tropical drink nearby, 
bright sunny day, vacation scene, 35mm photograph, film, professional, 4k, highly detailed
&quot;&quot;&quot;</span>
negative_prompt = <span class="hljs-string">&quot;lowres, bad anatomy, worst quality, low quality, deformed, ugly&quot;</span>

images = [canny_image.resize((<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>)), depth_image.resize((<span class="hljs-number">1024</span>, <span class="hljs-number">1024</span>))]

pipeline(
    prompt,
    negative_prompt=negative_prompt,
    image=images,
    num_inference_steps=<span class="hljs-number">100</span>,
    controlnet_conditioning_scale=[<span class="hljs-number">0.5</span>, <span class="hljs-number">0.5</span>],
).images[<span class="hljs-number">0</span>]<!----></pre></div><!----> <div style="display: flex; gap: 10px; justify-content: space-around; align-items: flex-end;"><figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/canny-cat.png" width="300" alt="Canny image"/> <figcaption style="text-align: center;">ảnh canny</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/multicontrolnet_depth.png" width="300" alt="Depth map"/> <figcaption style="text-align: center;">bản đồ độ sâu</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/sdxl_multi_controlnet.png" width="300" alt="Generated image (ControlNet + prompt)"/> <figcaption style="text-align: center;">hình ảnh được tạo</figcaption></figure></div> <!--[1--><h2 class="relative group"><a id="guess-mode" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#guess-mode"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span><!---->Chế độ đoán<!----></span></h2><!--]--><!----> <p><a href="https://github.com/lllyasviel/ControlNet/discussions/188" rel="nofollow">Chế độ Guess</a> tạo ra hình ảnh từ một tín hiệu điều khiển đơn lẻ khi bạn không có (hoặc không muốn dùng) prompt văn bản. Chế độ này nhân các phần dư của ControlNet theo độ sâu của khối, từ khoảng <code>0.1</code> ở các khối xuống sớm lên đến <code>1.0</code> ở khối giữa. Điều này giúp các lớp sớm không bị ràng buộc quá mức trong khi các khối sâu hơn vẫn duy trì cấu trúc.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5   text-gray-600  " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Đã sao chép</div><!----></button><!----></div> <pre class="language-py "><!----><span class="hljs-keyword">import</span> torch
<span class="hljs-keyword">from</span> diffusers.utils <span class="hljs-keyword">import</span> load_image
<span class="hljs-keyword">from</span> diffusers <span class="hljs-keyword">import</span> StableDiffusionXLControlNetPipeline, ControlNetModel

controlnet = ControlNetModel.from_pretrained(
  <span class="hljs-string">&quot;diffusers/controlnet-canny-sdxl-1.0&quot;</span>, dtype=torch.float16
)
pipeline = StableDiffusionXLControlNetPipeline.from_pretrained(
  <span class="hljs-string">&quot;stabilityai/stable-diffusion-xl-base-1.0&quot;</span>,
  controlnet=controlnet,
  dtype=torch.float16
).to(<span class="hljs-string">&quot;cuda&quot;</span>)  <span class="hljs-comment"># or &quot;mps&quot;, &quot;xpu&quot;, &quot;cpu&quot;</span>

canny_image = load_image(<span class="hljs-string">&quot;https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/canny-cat.png&quot;</span>)
pipeline(
  <span class="hljs-string">&quot;&quot;</span>,
  image=canny_image,
  guess_mode=<span class="hljs-literal">True</span>
).images[<span class="hljs-number">0</span>]<!----></pre></div><!----> <div style="display: flex; gap: 10px; justify-content: space-around; align-items: flex-end;"><figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/canny-cat.png" width="300" alt="Canny image"/> <figcaption style="text-align: center;">ảnh canny</figcaption></figure> <figure><img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/guess_mode.png" width="300" alt="Generated image (Guess mode)"/> <figcaption style="text-align: center;">hình ảnh được tạo</figcaption></figure></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/diffusers/blob/main/docs/source/en/using-diffusers/controlnet.md" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
			
			<script>
				{
					__sveltekit_1qsdbwf = {
						base: "/docs/diffusers/main/en",
						assets: "/docs/diffusers/main/en"
					};
					const element = document.currentScript.parentElement;
					Promise.all([
						import("/docs/diffusers/main/en/_app/immutable/entry/start.0S0_iEoG.js"),
						import("/docs/diffusers/main/en/_app/immutable/entry/app.CPdOxCbw.js")
					]).then(([kit, app]) => {
						kit.start(app, element, {
							node_ids: [0, 391],
							data: [null,null],
							form: null,
							error: null
						});
					});
				}
			</script>
		
Xet Storage Details

Size:: 42.9 kB
Xet hash:: 515e43b78f8a8982f4cd556bde527aa61f8048a76bec3b161f4f9db9309eee28
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.