Buckets:

download
raw
117 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;استقرار بهینه‌ی استنتاج&quot;,&quot;local&quot;:&quot;استقرار-بهینهی-استنتاج&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;راهنمای انتخاب چارچوب&quot;,&quot;local&quot;:&quot;راهنمای-انتخاب-چارچوب&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;مدیریت حافظه و عملکرد&quot;,&quot;local&quot;:&quot;مدیریت-حافظه-و-عملکرد&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;استقرار و یکپارچه‌سازی&quot;,&quot;local&quot;:&quot;استقرار-و-یکپارچهسازی&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;شروع کار&quot;,&quot;local&quot;:&quot;شروع-کار&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;نصب و راه‌اندازی پایه&quot;,&quot;local&quot;:&quot;نصب-و-راهاندازی-پایه&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;تولید پایه‌ی متن&quot;,&quot;local&quot;:&quot;تولید-پایهی-متن&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;کنترل پیشرفته‌ی تولید&quot;,&quot;local&quot;:&quot;کنترل-پیشرفتهی-تولید&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;انتخاب توکِن و نمونه‌برداری&quot;,&quot;local&quot;:&quot;انتخاب-توکن-و-نمونهبرداری&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;کنترل تکرار&quot;,&quot;local&quot;:&quot;کنترل-تکرار&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;کنترل طول و رشته‌های توقف&quot;,&quot;local&quot;:&quot;کنترل-طول-و-رشتههای-توقف&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;مدیریت حافظه&quot;,&quot;local&quot;:&quot;مدیریت-حافظه&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;منابع&quot;,&quot;local&quot;:&quot;منابع&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
<link href="/docs/course/pr_1364/fa/_app/immutable/entry/start.CqZXmSrv.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/oyrVqMhC.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/B3UUDp28.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/entry/app.B4UvgX5H.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/BioDZtGL.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/NIl2lQjl.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/DN6uzRQe.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/nodes/0.DcFj1eGj.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/u5kFQ1Zb.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/nodes/43.BefmZGJX.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/BjCHz3cj.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/D0FEdAwv.js" rel="modulepreload">
<!--116gsbn--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;استقرار بهینه‌ی استنتاج&quot;,&quot;local&quot;:&quot;استقرار-بهینهی-استنتاج&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;راهنمای انتخاب چارچوب&quot;,&quot;local&quot;:&quot;راهنمای-انتخاب-چارچوب&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;مدیریت حافظه و عملکرد&quot;,&quot;local&quot;:&quot;مدیریت-حافظه-و-عملکرد&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;استقرار و یکپارچه‌سازی&quot;,&quot;local&quot;:&quot;استقرار-و-یکپارچهسازی&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;شروع کار&quot;,&quot;local&quot;:&quot;شروع-کار&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;نصب و راه‌اندازی پایه&quot;,&quot;local&quot;:&quot;نصب-و-راهاندازی-پایه&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;تولید پایه‌ی متن&quot;,&quot;local&quot;:&quot;تولید-پایهی-متن&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;کنترل پیشرفته‌ی تولید&quot;,&quot;local&quot;:&quot;کنترل-پیشرفتهی-تولید&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;انتخاب توکِن و نمونه‌برداری&quot;,&quot;local&quot;:&quot;انتخاب-توکن-و-نمونهبرداری&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;کنترل تکرار&quot;,&quot;local&quot;:&quot;کنترل-تکرار&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3},{&quot;title&quot;:&quot;کنترل طول و رشته‌های توقف&quot;,&quot;local&quot;:&quot;کنترل-طول-و-رشتههای-توقف&quot;,&quot;sections&quot;:[],&quot;depth&quot;:3}],&quot;depth&quot;:2},{&quot;title&quot;:&quot;مدیریت حافظه&quot;,&quot;local&quot;:&quot;مدیریت-حافظه&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;منابع&quot;,&quot;local&quot;:&quot;منابع&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
<link href="/docs/course/pr_1364/fa/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div dir="rtl"><div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="استقرار-بهینهی-استنتاج" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#استقرار-بهینهی-استنتاج"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>استقرار بهینه‌ی استنتاج</span></h1><!--]--><!----> <p>در این بخش، چارچوب‌های پیشرفته‌ی بهینه‌سازی استقرار مدل‌های زبانی بزرگ را بررسی می‌کنیم: Text Generation Inference (TGI)، vLLM و llama.cpp. این برنامه‌ها عمدتاً در محیط‌های تولید برای ارائه‌ی مدل‌های زبانی بزرگ به کاربران استفاده می‌شوند. تمرکز این بخش بر نحوه‌ی استقرار این چارچوب‌ها در محیط تولید است، نه نحوه‌ی استفاده از آن‌ها برای استنتاج روی یک ماشین منفرد.</p> <p>بررسی خواهیم کرد که این ابزارها چگونه کارایی استنتاج را به حداکثر می‌رسانند و استقرار مدل‌های زبانی بزرگ را در محیط تولید ساده‌تر می‌کنند.</p> <!--[1--><h2 class="relative group"><a id="راهنمای-انتخاب-چارچوب" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#راهنمای-انتخاب-چارچوب"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>راهنمای انتخاب چارچوب</span></h2><!--]--><!----> <p>TGI، vLLM و llama.cpp هدف‌های مشابهی دارند، اما ویژگی‌های متمایزی دارند که باعث می‌شود هرکدام برای موارد استفاده‌ی متفاوتی مناسب‌تر باشند. بیایید تفاوت‌های کلیدی آن‌ها را با تمرکز بر عملکرد و یکپارچه‌سازی بررسی کنیم.</p> <!--[2--><h3 class="relative group"><a id="مدیریت-حافظه-و-عملکرد" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#مدیریت-حافظه-و-عملکرد"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>مدیریت حافظه و عملکرد</span></h3><!--]--><!----> <p><strong>TGI</strong> برای پایدار و قابل‌پیش‌بینی بودن در محیط تولید طراحی شده و از طول‌های ثابت رشته استفاده می‌کند تا مصرف حافظه را ثابت نگه دارد. TGI حافظه را با استفاده از Flash Attention 2 و تکنیک‌های بتچ‌سازی پیوسته مدیریت می‌کند. این یعنی می‌تواند محاسبات توجه را بسیار کارآمد انجام دهد و با تغذیه‌ی پیوسته‌ی کار، GPU را مشغول نگه دارد. سیستم می‌تواند در صورت نیاز بخش‌هایی از مدل را بین CPU و GPU جابه‌جا کند و به این ترتیب از مدل‌های بزرگ‌تر پشتیبانی کند.</p> <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/tgi/flash-attn.png" alt="Flash Attention"/> <blockquote class="tip"><p>Flash Attention تکنیکی است که با برطرف کردن گلوگاه‌های پهنای باند حافظه، مکانیزم توجه را در مدل‌های ترنسفورمر بهینه می‌کند. همان‌طور که پیش‌تر در <a href="/course/chapter1/8">بخش ۱.۸</a> گفتیم، مکانیزم توجه پیچیدگی و مصرف حافظه‌ی درجه‌دوم دارد و همین موضوع آن را برای رشته‌های طولانی ناکارآمد می‌کند.</p> <p>نوآوری اصلی در نحوه‌ی مدیریت انتقال حافظه است. توجه سنتی داده‌ها را بارها بین حافظه‌ی با پهنای باند بالا (HBM) و کش سریع SRAM جابه‌جا می‌کند و با بیکار گذاشتن GPU گلوگاه ایجاد می‌کند. Flash Attention داده‌ها را یک‌بار در SRAM بارگذاری می‌کند و همه‌ی محاسبات را همان‌جا انجام می‌دهد؛ در نتیجه انتقال‌های پرهزینه‌ی حافظه به حداقل می‌رسند.</p> <p>اگرچه مزایای آن در زمان تعلیم بیشترین مقدار را دارند، مصرف کمتر VRAM و کارایی بهتر Flash Attention باعث می‌شود برای استنتاج نیز ارزشمند باشد و ارائه‌ی سریع‌تر و مقیاس‌پذیرتر مدل‌های زبانی بزرگ را ممکن کند.</p><!----></blockquote><!----> <p><strong>vLLM</strong> با استفاده از PagedAttention رویکرد متفاوتی دارد. همان‌طور که رایانه حافظه‌ی خود را به صفحه‌ها تقسیم می‌کند، vLLM حافظه‌ی مدل را به بلوک‌های کوچک‌تر تقسیم می‌کند. این سیستم هوشمندانه باعث می‌شود بتواند درخواست‌هایی با اندازه‌های مختلف را انعطاف‌پذیرتر مدیریت کند و فضای حافظه را هدر ندهد. این چارچوب در اشتراک‌گذاری حافظه بین درخواست‌های مختلف بسیار خوب عمل می‌کند و تکه‌تکه‌شدن حافظه را کاهش می‌دهد؛ بنابراین کل سیستم کارآمدتر می‌شود.</p> <blockquote class="tip"><p>PagedAttention تکنیکی است که به یکی دیگر از گلوگاه‌های مهم استنتاج مدل‌های زبانی بزرگ، یعنی مدیریت حافظه‌ی کش KV، می‌پردازد. همان‌طور که در <a href="/course/chapter1/8">بخش ۱.۸</a> گفتیم، مدل هنگام تولید متن، کلیدها و مقادیر توجه (کش KV) را برای هر توکِن تولیدشده ذخیره می‌کند تا محاسبات تکراری کاهش یابند. کش KV، به‌خصوص برای رشته‌های طولانی یا چندین درخواست هم‌زمان، می‌تواند بسیار بزرگ شود.</p> <p>نوآوری اصلی vLLM در نحوه‌ی مدیریت این کش است:</p> <ol><li><strong>صفحه‌بندی حافظه</strong>: به‌جای اینکه کش KV به‌صورت یک بلوک بزرگ در نظر گرفته شود، به «صفحه‌های» با اندازه‌ی ثابت تقسیم می‌شود (مشابه حافظه‌ی مجازی در سیستم‌عامل‌ها).</li> <li><strong>ذخیره‌سازی غیرپیوسته</strong>: لازم نیست صفحه‌ها به‌صورت پیوسته در حافظه‌ی GPU ذخیره شوند؛ بنابراین تخصیص حافظه انعطاف‌پذیرتر است.</li> <li><strong>مدیریت جدول صفحه</strong>: یک جدول صفحه مشخص می‌کند کدام صفحه‌ها به کدام رشته تعلق دارند و جست‌وجو و دسترسی کارآمد را ممکن می‌کند.</li> <li><strong>اشتراک‌گذاری حافظه</strong>: در عملیاتی مانند نمونه‌برداری موازی، صفحه‌هایی که کش KV مربوط به پیام ورودی را ذخیره می‌کنند می‌توانند بین چندین رشته به اشتراک گذاشته شوند.</li></ol> <p>رویکرد PagedAttention در مقایسه با روش‌های سنتی می‌تواند توان عملیاتی را تا ۲۴ برابر افزایش دهد و برای استقرار مدل‌های زبانی بزرگ در محیط تولید تحولی اساسی ایجاد کند. اگر می‌خواهید درباره‌ی نحوه‌ی کار PagedAttention عمیق‌تر مطالعه کنید، می‌توانید <a href="https://docs.vllm.ai/en/latest/design/kernel/paged_attention.html" rel="nofollow">راهنمای مستندات vLLM</a> را بخوانید.</p><!----></blockquote><!----> <p><strong>llama.cpp</strong> یک پیاده‌سازی بسیار بهینه از C/C++ است که در ابتدا برای اجرای مدل‌های LLaMA روی سخت‌افزارهای مصرفی طراحی شد. این پروژه بر کارایی CPU تمرکز دارد، با امکان شتاب‌دهی GPU، و برای محیط‌های دارای منابع محدود ایدئال است. llama.cpp از تکنیک‌های کوانتیزه‌سازی برای کاهش اندازه و نیازمندی‌های حافظه‌ی مدل استفاده می‌کند و در عین حفظ عملکرد خوب، کرنل‌های بهینه‌شده‌ای برای معماری‌های مختلف CPU ارائه می‌دهد و از مدیریت پایه‌ی کش KV برای تولید کارآمد توکِن پشتیبانی می‌کند.</p> <blockquote class="tip"><p>کوانتیزه‌سازی در llama.cpp دقت وزن‌های مدل را از ممیز شناور ۳۲بیتی یا ۱۶بیتی به قالب‌های کم‌دقت‌تری مانند اعداد صحیح ۸بیتی (INT8)، ۴بیتی یا حتی پایین‌تر کاهش می‌دهد. این کار مصرف حافظه را به‌طور چشمگیری کاهش می‌دهد و سرعت استنتاج را با افت کیفیتی اندک افزایش می‌دهد.</p> <p>ویژگی‌های کلیدی کوانتیزه‌سازی در llama.cpp عبارت‌اند از:</p> <ol><li><strong>سطوح متعدد کوانتیزه‌سازی</strong>: از کوانتیزه‌سازی ۸بیتی، ۴بیتی، ۳بیتی و حتی ۲بیتی پشتیبانی می‌کند</li> <li><strong>قالب GGML/GGUF</strong>: از قالب‌های سفارشی تِنسور استفاده می‌کند که برای استنتاج کوانتیزه‌شده بهینه شده‌اند</li> <li><strong>دقت ترکیبی</strong>: می‌تواند سطوح مختلف کوانتیزه‌سازی را روی بخش‌های مختلف مدل اعمال کند</li> <li><strong>بهینه‌سازی‌های ویژه‌ی سخت‌افزار</strong>: مسیرهای کد بهینه‌شده برای معماری‌های مختلف CPU (AVX2، AVX-512، NEON) دارد</li></ol> <p>این رویکرد اجرای مدل‌های دارای میلیاردها پارامتر را روی سخت‌افزارهای مصرفی با حافظه‌ی محدود ممکن می‌کند و برای استقرارهای محلی و دستگاه‌های لبه‌ای عالی است.</p><!----></blockquote><!----> <!--[2--><h3 class="relative group"><a id="استقرار-و-یکپارچهسازی" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#استقرار-و-یکپارچهسازی"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>استقرار و یکپارچه‌سازی</span></h3><!--]--><!----> <p>حالا به سراغ تفاوت‌های استقرار و یکپارچه‌سازی میان این چارچوب‌ها برویم.</p> <p><strong>TGI</strong> در استقرارهای سازمانی و در مقیاس بزرگ، با قابلیت‌های آماده‌ی تولید خود، عملکردی عالی دارد. این چارچوب پشتیبانی داخلی از Kubernetes دارد و هر چیزی را که برای اجرای محیط تولید نیاز دارید، از جمله پایش با Prometheus و Grafana، مقیاس‌بندی خودکار و قابلیت‌های جامع ایمنی، در اختیار می‌گذارد. سیستم همچنین شامل ثبت گزارش در سطح سازمانی و اقدامات حفاظتی مختلفی مانند فیلتر کردن محتوا و محدودسازی نرخ است تا استقرار شما امن و پایدار بماند.</p> <p><strong>vLLM</strong> رویکردی انعطاف‌پذیرتر و سازگارتر با توسعه‌دهندگان برای استقرار دارد. هسته‌ی آن با پایتون ساخته شده و می‌تواند به‌سادگی جایگزین API شرکت OpenAI در برنامه‌های موجود شما شود. این چارچوب بر ارائه‌ی عملکرد خام تمرکز دارد و می‌توان آن را مطابق نیازهای خاص شما سفارشی کرد. vLLM برای مدیریت کلاسترها به‌خوبی با Ray کار می‌کند و وقتی به عملکرد بالا و سازگاری نیاز دارید، انتخابی عالی است.</p> <p><strong>llama.cpp</strong> سادگی و قابلیت حمل را در اولویت قرار می‌دهد. پیاده‌سازی سرور آن سبک است و می‌تواند روی طیف گسترده‌ای از سخت‌افزارها، از سرورهای قدرتمند تا لپ‌تاپ‌های مصرفی و حتی برخی دستگاه‌های همراه رده‌بالا اجرا شود. با وابستگی‌های حداقلی و هسته‌ی ساده‌ی C/C++، استقرار آن در محیط‌هایی که نصب چارچوب‌های پایتونی دشوار است آسان است. سرور، ضمن حفظ ردپای بسیار کوچک‌تر منابع نسبت به راهکارهای دیگر، یک API سازگار با OpenAI ارائه می‌دهد.</p> <!--[1--><h2 class="relative group"><a id="شروع-کار" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#شروع-کار"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>شروع کار</span></h2><!--]--><!----> <p>بیایید نحوه‌ی استفاده از این چارچوب‌ها برای استقرار مدل‌های زبانی بزرگ را بررسی کنیم و از نصب و راه‌اندازی پایه شروع کنیم.</p> <!--[2--><h3 class="relative group"><a id="نصب-و-راهاندازی-پایه" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#نصب-و-راهاندازی-پایه"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>نصب و راه‌اندازی پایه</span></h3><!--]--><!----> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><!--]--></div> <div class="language-select"><!---->&lt;hfoption value="tgi" label="TGI"> <p>نصب و استفاده از TGI آسان است و یکپارچه‌سازی عمیقی با اکوسیستم هاگینگ‌فِیس دارد.</p> <p>ابتدا سرور TGI را با Docker اجرا کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-sh "><!---->docker run --gpus all \
--shm-size 1g \
-p 8080:80 \
-v ~/.cache/huggingface:/data \
ghcr.io/huggingface/text-generation-inference:latest \
--model-id HuggingFaceTB/SmolLM2-360M-Instruct<!----></pre></div><!----></div> <p>سپس با استفاده از InferenceClient هاگینگ‌فِیس با آن تعامل کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient
<span class="hljs-comment"># Initialize client pointing to TGI endpoint</span>
client = InferenceClient(
model=<span class="hljs-string">&quot;http://localhost:8080&quot;</span>, <span class="hljs-comment"># URL to the TGI server</span>
)
<span class="hljs-comment"># Text generation</span>
response = client.text_generation(
<span class="hljs-string">&quot;Tell me a story&quot;</span>,
max_new_tokens=<span class="hljs-number">100</span>,
temperature=<span class="hljs-number">0.7</span>,
top_p=<span class="hljs-number">0.95</span>,
details=<span class="hljs-literal">True</span>,
stop_sequences=[],
)
<span class="hljs-built_in">print</span>(response.generated_text)
<span class="hljs-comment"># For chat format</span>
response = client.chat_completion(
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Tell me a story&quot;</span>},
],
max_tokens=<span class="hljs-number">100</span>,
temperature=<span class="hljs-number">0.7</span>,
top_p=<span class="hljs-number">0.95</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!----></pre></div><!----></div> <p>یا می‌توانید از کلاینت OpenAI استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI
<span class="hljs-comment"># Initialize client pointing to TGI endpoint</span>
client = OpenAI(
base_url=<span class="hljs-string">&quot;http://localhost:8080/v1&quot;</span>, <span class="hljs-comment"># Make sure to include /v1</span>
api_key=<span class="hljs-string">&quot;not-needed&quot;</span>, <span class="hljs-comment"># TGI doesn&#x27;t require an API key by default</span>
)
<span class="hljs-comment"># Chat completion</span>
response = client.chat.completions.create(
model=<span class="hljs-string">&quot;HuggingFaceTB/SmolLM2-360M-Instruct&quot;</span>,
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Tell me a story&quot;</span>},
],
max_tokens=<span class="hljs-number">100</span>,
temperature=<span class="hljs-number">0.7</span>,
top_p=<span class="hljs-number">0.95</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="llama.cpp" label="llama.cpp"> <p>llama.cpp نصب و استفاده‌ی آسانی دارد، به وابستگی‌های کمی نیاز دارد و از استنتاج روی CPU و GPU پشتیبانی می‌کند.</p> <p>ابتدا llama.cpp را نصب و بسازید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-sh "><!----><span class="hljs-comment"># Clone the repository</span>
git <span class="hljs-built_in">clone</span> https://github.com/ggerganov/llama.cpp
<span class="hljs-built_in">cd</span> llama.cpp
<span class="hljs-comment"># Build the project</span>
make
<span class="hljs-comment"># Download the SmolLM2-1.7B-Instruct-GGUF model</span>
curl -L -O https://huggingface.co/HuggingFaceTB/SmolLM2-1.7B-Instruct-GGUF/resolve/main/smollm2-1.7b-instruct.Q4_K_M.gguf<!----></pre></div><!----></div> <p>سپس سرور را اجرا کنید (با سازگاری با API شرکت OpenAI):</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-sh "><!----><span class="hljs-comment"># Start the server</span>
./server \
-m smollm2-1.7b-instruct.Q4_K_M.gguf \
--host 0.0.0.0 \
--port 8080 \
-c 4096 \
--n-gpu-layers 0 <span class="hljs-comment"># Set to a higher number to use GPU</span><!----></pre></div><!----></div> <p>با استفاده از InferenceClient هاگینگ‌فِیس با سرور تعامل کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient
<span class="hljs-comment"># Initialize client pointing to llama.cpp server</span>
client = InferenceClient(
model=<span class="hljs-string">&quot;http://localhost:8080/v1&quot;</span>, <span class="hljs-comment"># URL to the llama.cpp server</span>
token=<span class="hljs-string">&quot;sk-no-key-required&quot;</span>, <span class="hljs-comment"># llama.cpp server requires this placeholder</span>
)
<span class="hljs-comment"># Text generation</span>
response = client.text_generation(
<span class="hljs-string">&quot;Tell me a story&quot;</span>,
max_new_tokens=<span class="hljs-number">100</span>,
temperature=<span class="hljs-number">0.7</span>,
top_p=<span class="hljs-number">0.95</span>,
details=<span class="hljs-literal">True</span>,
)
<span class="hljs-built_in">print</span>(response.generated_text)
<span class="hljs-comment"># For chat format</span>
response = client.chat_completion(
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Tell me a story&quot;</span>},
],
max_tokens=<span class="hljs-number">100</span>,
temperature=<span class="hljs-number">0.7</span>,
top_p=<span class="hljs-number">0.95</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!----></pre></div><!----></div> <p>یا می‌توانید از کلاینت OpenAI استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI
<span class="hljs-comment"># Initialize client pointing to llama.cpp server</span>
client = OpenAI(
base_url=<span class="hljs-string">&quot;http://localhost:8080/v1&quot;</span>,
api_key=<span class="hljs-string">&quot;sk-no-key-required&quot;</span>, <span class="hljs-comment"># llama.cpp server requires this placeholder</span>
)
<span class="hljs-comment"># Chat completion</span>
response = client.chat.completions.create(
model=<span class="hljs-string">&quot;smollm2-1.7b-instruct&quot;</span>, <span class="hljs-comment"># Model identifier can be anything as server only loads one model</span>
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Tell me a story&quot;</span>},
],
max_tokens=<span class="hljs-number">100</span>,
temperature=<span class="hljs-number">0.7</span>,
top_p=<span class="hljs-number">0.95</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="vllm" label="vLLM"> <p>vLLM نصب و استفاده‌ی آسانی دارد و هم با API شرکت OpenAI سازگار است و هم رابط بومی پایتون دارد.</p> <p>ابتدا سرور سازگار با OpenAI vLLM را اجرا کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-sh "><!---->python -m vllm.entrypoints.openai.api_server \
--model HuggingFaceTB/SmolLM2-360M-Instruct \
--host 0.0.0.0 \
--port 8000<!----></pre></div><!----></div> <p>سپس با استفاده از InferenceClient هاگینگ‌فِیس با آن تعامل کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient
<span class="hljs-comment"># Initialize client pointing to vLLM endpoint</span>
client = InferenceClient(
model=<span class="hljs-string">&quot;http://localhost:8000/v1&quot;</span>, <span class="hljs-comment"># URL to the vLLM server</span>
)
<span class="hljs-comment"># Text generation</span>
response = client.text_generation(
<span class="hljs-string">&quot;Tell me a story&quot;</span>,
max_new_tokens=<span class="hljs-number">100</span>,
temperature=<span class="hljs-number">0.7</span>,
top_p=<span class="hljs-number">0.95</span>,
details=<span class="hljs-literal">True</span>,
)
<span class="hljs-built_in">print</span>(response.generated_text)
<span class="hljs-comment"># For chat format</span>
response = client.chat_completion(
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Tell me a story&quot;</span>},
],
max_tokens=<span class="hljs-number">100</span>,
temperature=<span class="hljs-number">0.7</span>,
top_p=<span class="hljs-number">0.95</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!----></pre></div><!----></div> <p>یا می‌توانید از کلاینت OpenAI استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI
<span class="hljs-comment"># Initialize client pointing to vLLM endpoint</span>
client = OpenAI(
base_url=<span class="hljs-string">&quot;http://localhost:8000/v1&quot;</span>,
api_key=<span class="hljs-string">&quot;not-needed&quot;</span>, <span class="hljs-comment"># vLLM doesn&#x27;t require an API key by default</span>
)
<span class="hljs-comment"># Chat completion</span>
response = client.chat.completions.create(
model=<span class="hljs-string">&quot;HuggingFaceTB/SmolLM2-360M-Instruct&quot;</span>,
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a helpful assistant.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Tell me a story&quot;</span>},
],
max_tokens=<span class="hljs-number">100</span>,
temperature=<span class="hljs-number">0.7</span>,
top_p=<span class="hljs-number">0.95</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!----></pre></div><!----></div> &lt;/hfoption><!----></div><!----> <!--[2--><h3 class="relative group"><a id="تولید-پایهی-متن" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#تولید-پایهی-متن"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>تولید پایه‌ی متن</span></h3><!--]--><!----> <p>بیایید نمونه‌هایی از تولید متن با این چارچوب‌ها را بررسی کنیم:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><!--]--></div> <div class="language-select"><!---->&lt;hfoption value="tgi" label="TGI"> <p>ابتدا TGI را با پارامترهای پیشرفته مستقر کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-sh "><!---->docker run --gpus all \
--shm-size 1g \
-p 8080:80 \
-v ~/.cache/huggingface:/data \
ghcr.io/huggingface/text-generation-inference:latest \
--model-id HuggingFaceTB/SmolLM2-360M-Instruct \
--max-total-tokens 4096 \
--max-input-length 3072 \
--max-batch-total-tokens 8192 \
--waiting-served-ratio 1.2<!----></pre></div><!----></div> <p>برای تولید انعطاف‌پذیر متن از InferenceClient استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient
client = InferenceClient(model=<span class="hljs-string">&quot;http://localhost:8080&quot;</span>)
<span class="hljs-comment"># Advanced parameters example</span>
response = client.chat_completion(
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a creative storyteller.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Write a creative story&quot;</span>},
],
temperature=<span class="hljs-number">0.8</span>,
max_tokens=<span class="hljs-number">200</span>,
top_p=<span class="hljs-number">0.95</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)
<span class="hljs-comment"># Raw text generation</span>
response = client.text_generation(
<span class="hljs-string">&quot;Write a creative story about space exploration&quot;</span>,
max_new_tokens=<span class="hljs-number">200</span>,
temperature=<span class="hljs-number">0.8</span>,
top_p=<span class="hljs-number">0.95</span>,
repetition_penalty=<span class="hljs-number">1.1</span>,
do_sample=<span class="hljs-literal">True</span>,
details=<span class="hljs-literal">True</span>,
)
<span class="hljs-built_in">print</span>(response.generated_text)<!----></pre></div><!----></div> <p>یا از کلاینت OpenAI استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI
client = OpenAI(base_url=<span class="hljs-string">&quot;http://localhost:8080/v1&quot;</span>, api_key=<span class="hljs-string">&quot;not-needed&quot;</span>)
<span class="hljs-comment"># Advanced parameters example</span>
response = client.chat.completions.create(
model=<span class="hljs-string">&quot;HuggingFaceTB/SmolLM2-360M-Instruct&quot;</span>,
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a creative storyteller.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Write a creative story&quot;</span>},
],
temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Higher for more creativity</span>
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="llama.cpp" label="llama.cpp"> <p>در llama.cpp هنگام اجرای سرور می‌توانید پارامترهای پیشرفته را تنظیم کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-sh "><!---->./server \
-m smollm2-1.7b-instruct.Q4_K_M.gguf \
--host 0.0.0.0 \
--port 8080 \
-c 4096 \ <span class="hljs-comment"># Context size</span>
--threads 8 \ <span class="hljs-comment"># CPU threads to use</span>
--batch-size 512 \ <span class="hljs-comment"># Batch size for prompt evaluation</span>
--n-gpu-layers 0 <span class="hljs-comment"># GPU layers (0 = CPU only)</span><!----></pre></div><!----></div> <p>از InferenceClient استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient
client = InferenceClient(model=<span class="hljs-string">&quot;http://localhost:8080/v1&quot;</span>, token=<span class="hljs-string">&quot;sk-no-key-required&quot;</span>)
<span class="hljs-comment"># Advanced parameters example</span>
response = client.chat_completion(
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a creative storyteller.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Write a creative story&quot;</span>},
],
temperature=<span class="hljs-number">0.8</span>,
max_tokens=<span class="hljs-number">200</span>,
top_p=<span class="hljs-number">0.95</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)
<span class="hljs-comment"># For direct text generation</span>
response = client.text_generation(
<span class="hljs-string">&quot;Write a creative story about space exploration&quot;</span>,
max_new_tokens=<span class="hljs-number">200</span>,
temperature=<span class="hljs-number">0.8</span>,
top_p=<span class="hljs-number">0.95</span>,
repetition_penalty=<span class="hljs-number">1.1</span>,
details=<span class="hljs-literal">True</span>,
)
<span class="hljs-built_in">print</span>(response.generated_text)<!----></pre></div><!----></div> <p>یا برای تولید، با کنترل پارامترهای نمونه‌برداری، از کلاینت OpenAI استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI
client = OpenAI(base_url=<span class="hljs-string">&quot;http://localhost:8080/v1&quot;</span>, api_key=<span class="hljs-string">&quot;sk-no-key-required&quot;</span>)
<span class="hljs-comment"># Advanced parameters example</span>
response = client.chat.completions.create(
model=<span class="hljs-string">&quot;smollm2-1.7b-instruct&quot;</span>,
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a creative storyteller.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Write a creative story&quot;</span>},
],
temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Higher for more creativity</span>
top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Nucleus sampling probability</span>
frequency_penalty=<span class="hljs-number">0.5</span>, <span class="hljs-comment"># Reduce repetition of frequent tokens</span>
presence_penalty=<span class="hljs-number">0.5</span>, <span class="hljs-comment"># Reduce repetition by penalizing tokens already present</span>
max_tokens=<span class="hljs-number">200</span>, <span class="hljs-comment"># Maximum generation length</span>
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!----></pre></div><!----></div> <p>همچنین می‌توانید برای کنترل بیشتر از کتابخانه‌ی بومی llama.cpp استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># Using llama-cpp-python package for direct model access</span>
<span class="hljs-keyword">from</span> llama_cpp <span class="hljs-keyword">import</span> Llama
<span class="hljs-comment"># Load the model</span>
llm = Llama(
model_path=<span class="hljs-string">&quot;smollm2-1.7b-instruct.Q4_K_M.gguf&quot;</span>,
n_ctx=<span class="hljs-number">4096</span>, <span class="hljs-comment"># Context window size</span>
n_threads=<span class="hljs-number">8</span>, <span class="hljs-comment"># CPU threads</span>
n_gpu_layers=<span class="hljs-number">0</span>, <span class="hljs-comment"># GPU layers (0 = CPU only)</span>
)
<span class="hljs-comment"># Format prompt according to the model&#x27;s expected format</span>
prompt = <span class="hljs-string">&quot;&quot;&quot;&lt;|im_start|&gt;system
You are a creative storyteller.
&lt;|im_end|&gt;
&lt;|im_start|&gt;user
Write a creative story
&lt;|im_end|&gt;
&lt;|im_start|&gt;assistant
&quot;&quot;&quot;</span>
<span class="hljs-comment"># Generate response with precise parameter control</span>
output = llm(
prompt,
max_tokens=<span class="hljs-number">200</span>,
temperature=<span class="hljs-number">0.8</span>,
top_p=<span class="hljs-number">0.95</span>,
frequency_penalty=<span class="hljs-number">0.5</span>,
presence_penalty=<span class="hljs-number">0.5</span>,
stop=[<span class="hljs-string">&quot;&lt;|im_end|&gt;&quot;</span>],
)
<span class="hljs-built_in">print</span>(output[<span class="hljs-string">&quot;choices&quot;</span>][<span class="hljs-number">0</span>][<span class="hljs-string">&quot;text&quot;</span>])<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="vllm" label="vLLM"> <p>برای استفاده‌ی پیشرفته از vLLM می‌توانید از InferenceClient استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> huggingface_hub <span class="hljs-keyword">import</span> InferenceClient
client = InferenceClient(model=<span class="hljs-string">&quot;http://localhost:8000/v1&quot;</span>)
<span class="hljs-comment"># Advanced parameters example</span>
response = client.chat_completion(
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a creative storyteller.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Write a creative story&quot;</span>},
],
temperature=<span class="hljs-number">0.8</span>,
max_tokens=<span class="hljs-number">200</span>,
top_p=<span class="hljs-number">0.95</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)
<span class="hljs-comment"># For direct text generation</span>
response = client.text_generation(
<span class="hljs-string">&quot;Write a creative story about space exploration&quot;</span>,
max_new_tokens=<span class="hljs-number">200</span>,
temperature=<span class="hljs-number">0.8</span>,
top_p=<span class="hljs-number">0.95</span>,
details=<span class="hljs-literal">True</span>,
)
<span class="hljs-built_in">print</span>(response.generated_text)<!----></pre></div><!----></div> <p>همچنین می‌توانید از کلاینت OpenAI استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> openai <span class="hljs-keyword">import</span> OpenAI
client = OpenAI(base_url=<span class="hljs-string">&quot;http://localhost:8000/v1&quot;</span>, api_key=<span class="hljs-string">&quot;not-needed&quot;</span>)
<span class="hljs-comment"># Advanced parameters example</span>
response = client.chat.completions.create(
model=<span class="hljs-string">&quot;HuggingFaceTB/SmolLM2-360M-Instruct&quot;</span>,
messages=[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a creative storyteller.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Write a creative story&quot;</span>},
],
temperature=<span class="hljs-number">0.8</span>,
top_p=<span class="hljs-number">0.95</span>,
max_tokens=<span class="hljs-number">200</span>,
)
<span class="hljs-built_in">print</span>(response.choices[<span class="hljs-number">0</span>].message.content)<!----></pre></div><!----></div> <p>vLLM همچنین یک رابط بومی پایتون با کنترل دقیق ارائه می‌دهد:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> vllm <span class="hljs-keyword">import</span> LLM, SamplingParams
<span class="hljs-comment"># Initialize the model with advanced parameters</span>
llm = LLM(
model=<span class="hljs-string">&quot;HuggingFaceTB/SmolLM2-360M-Instruct&quot;</span>,
gpu_memory_utilization=<span class="hljs-number">0.85</span>,
max_num_batched_tokens=<span class="hljs-number">8192</span>,
max_num_seqs=<span class="hljs-number">256</span>,
block_size=<span class="hljs-number">16</span>,
)
<span class="hljs-comment"># Configure sampling parameters</span>
sampling_params = SamplingParams(
temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Higher for more creativity</span>
top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Consider top 95% probability mass</span>
max_tokens=<span class="hljs-number">100</span>, <span class="hljs-comment"># Maximum length</span>
presence_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Reduce repetition</span>
frequency_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Reduce repetition</span>
stop=[<span class="hljs-string">&quot;\n\n&quot;</span>, <span class="hljs-string">&quot;###&quot;</span>], <span class="hljs-comment"># Stop sequences</span>
)
<span class="hljs-comment"># Generate text</span>
prompt = <span class="hljs-string">&quot;Write a creative story&quot;</span>
outputs = llm.generate(prompt, sampling_params)
<span class="hljs-built_in">print</span>(outputs[<span class="hljs-number">0</span>].outputs[<span class="hljs-number">0</span>].text)
<span class="hljs-comment"># For chat-style interactions</span>
chat_prompt = [
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;You are a creative storyteller.&quot;</span>},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Write a creative story&quot;</span>},
]
formatted_prompt = llm.get_chat_template()(chat_prompt) <span class="hljs-comment"># Uses model&#x27;s chat template</span>
outputs = llm.generate(formatted_prompt, sampling_params)
<span class="hljs-built_in">print</span>(outputs[<span class="hljs-number">0</span>].outputs[<span class="hljs-number">0</span>].text)<!----></pre></div><!----></div> &lt;/hfoption><!----></div><!----> <!--[1--><h2 class="relative group"><a id="کنترل-پیشرفتهی-تولید" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#کنترل-پیشرفتهی-تولید"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>کنترل پیشرفته‌ی تولید</span></h2><!--]--><!----> <!--[2--><h3 class="relative group"><a id="انتخاب-توکن-و-نمونهبرداری" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#انتخاب-توکن-و-نمونهبرداری"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>انتخاب توکِن و نمونه‌برداری</span></h3><!--]--><!----> <p>فرایند تولید متن شامل انتخاب توکِن بعدی در هر مرحله است. این فرایند انتخاب را می‌توان با پارامترهای مختلف کنترل کرد:</p> <ol><li><strong>لوجیت‌های خام</strong>: احتمال‌های اولیه‌ی خروجی برای هر توکِن</li> <li><strong>دما</strong>: میزان تصادفی بودن انتخاب را کنترل می‌کند (مقدار بیشتر = خلاقیت بیشتر)</li> <li><strong>نمونه‌برداری Top-p (هسته‌ای)</strong>: توکِن‌های برتر تشکیل‌دهنده‌ی X٪ از جرم احتمال را فیلتر می‌کند</li> <li><strong>فیلتر Top-k</strong>: انتخاب را به k توکِن محتمل‌تر محدود می‌کند</li></ol> <p>در اینجا نحوه‌ی پیکربندی این پارامترها را می‌بینید:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><!--]--></div> <div class="language-select"><!---->&lt;hfoption value="tgi" label="TGI"> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->client.generate(
<span class="hljs-string">&quot;Write a creative story&quot;</span>,
temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Higher for more creativity</span>
top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Consider top 95% probability mass</span>
top_k=<span class="hljs-number">50</span>, <span class="hljs-comment"># Consider top 50 tokens</span>
max_new_tokens=<span class="hljs-number">100</span>, <span class="hljs-comment"># Maximum length</span>
repetition_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Reduce repetition</span>
)<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="llama.cpp" label="llama.cpp"> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># Via OpenAI API compatibility</span>
response = client.completions.create(
model=<span class="hljs-string">&quot;smollm2-1.7b-instruct&quot;</span>, <span class="hljs-comment"># Model name (can be any string for llama.cpp server)</span>
prompt=<span class="hljs-string">&quot;Write a creative story&quot;</span>,
temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Higher for more creativity</span>
top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Consider top 95% probability mass</span>
frequency_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Reduce repetition</span>
presence_penalty=<span class="hljs-number">0.1</span>, <span class="hljs-comment"># Reduce repetition</span>
max_tokens=<span class="hljs-number">100</span>, <span class="hljs-comment"># Maximum length</span>
)
<span class="hljs-comment"># Via llama-cpp-python direct access</span>
output = llm(
<span class="hljs-string">&quot;Write a creative story&quot;</span>,
temperature=<span class="hljs-number">0.8</span>,
top_p=<span class="hljs-number">0.95</span>,
top_k=<span class="hljs-number">50</span>,
max_tokens=<span class="hljs-number">100</span>,
repeat_penalty=<span class="hljs-number">1.1</span>,
)<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="vllm" label="vLLM"> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->params = SamplingParams(
temperature=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Higher for more creativity</span>
top_p=<span class="hljs-number">0.95</span>, <span class="hljs-comment"># Consider top 95% probability mass</span>
top_k=<span class="hljs-number">50</span>, <span class="hljs-comment"># Consider top 50 tokens</span>
max_tokens=<span class="hljs-number">100</span>, <span class="hljs-comment"># Maximum length</span>
presence_penalty=<span class="hljs-number">0.1</span>, <span class="hljs-comment"># Reduce repetition</span>
)
llm.generate(<span class="hljs-string">&quot;Write a creative story&quot;</span>, sampling_params=params)<!----></pre></div><!----></div> &lt;/hfoption><!----></div><!----> <!--[2--><h3 class="relative group"><a id="کنترل-تکرار" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#کنترل-تکرار"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>کنترل تکرار</span></h3><!--]--><!----> <p>هر دو چارچوب راه‌هایی برای جلوگیری از تولید متن تکراری ارائه می‌دهند:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><!--]--></div> <div class="language-select"><!---->&lt;hfoption value="tgi" label="TGI"> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->client.generate(
<span class="hljs-string">&quot;Write a varied text&quot;</span>,
repetition_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Penalize repeated tokens</span>
no_repeat_ngram_size=<span class="hljs-number">3</span>, <span class="hljs-comment"># Prevent 3-gram repetition</span>
)<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="llama.cpp" label="llama.cpp"> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># Via OpenAI API</span>
response = client.completions.create(
model=<span class="hljs-string">&quot;smollm2-1.7b-instruct&quot;</span>,
prompt=<span class="hljs-string">&quot;Write a varied text&quot;</span>,
frequency_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Penalize frequent tokens</span>
presence_penalty=<span class="hljs-number">0.8</span>, <span class="hljs-comment"># Penalize tokens already present</span>
)
<span class="hljs-comment"># Via direct library</span>
output = llm(
<span class="hljs-string">&quot;Write a varied text&quot;</span>,
repeat_penalty=<span class="hljs-number">1.1</span>, <span class="hljs-comment"># Penalize repeated tokens</span>
frequency_penalty=<span class="hljs-number">0.5</span>, <span class="hljs-comment"># Additional frequency penalty</span>
presence_penalty=<span class="hljs-number">0.5</span>, <span class="hljs-comment"># Additional presence penalty</span>
)<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="vllm" label="vLLM"> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->params = SamplingParams(
presence_penalty=<span class="hljs-number">0.1</span>, <span class="hljs-comment"># Penalize token presence</span>
frequency_penalty=<span class="hljs-number">0.1</span>, <span class="hljs-comment"># Penalize token frequency</span>
)<!----></pre></div><!----></div> &lt;/hfoption><!----></div><!----> <!--[2--><h3 class="relative group"><a id="کنترل-طول-و-رشتههای-توقف" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#کنترل-طول-و-رشتههای-توقف"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>کنترل طول و رشته‌های توقف</span></h3><!--]--><!----> <p>می‌توانید طول تولید را کنترل کنید و زمان توقف را مشخص کنید:</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><!--]--></div> <div class="language-select"><!---->&lt;hfoption value="tgi" label="TGI"> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->client.generate(
<span class="hljs-string">&quot;Generate a short paragraph&quot;</span>,
max_new_tokens=<span class="hljs-number">100</span>,
min_new_tokens=<span class="hljs-number">10</span>,
stop_sequences=[<span class="hljs-string">&quot;\n\n&quot;</span>, <span class="hljs-string">&quot;###&quot;</span>],
)<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="llama.cpp" label="llama.cpp"> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># Via OpenAI API</span>
response = client.completions.create(
model=<span class="hljs-string">&quot;smollm2-1.7b-instruct&quot;</span>,
prompt=<span class="hljs-string">&quot;Generate a short paragraph&quot;</span>,
max_tokens=<span class="hljs-number">100</span>,
stop=[<span class="hljs-string">&quot;\n\n&quot;</span>, <span class="hljs-string">&quot;###&quot;</span>],
)
<span class="hljs-comment"># Via direct library</span>
output = llm(<span class="hljs-string">&quot;Generate a short paragraph&quot;</span>, max_tokens=<span class="hljs-number">100</span>, stop=[<span class="hljs-string">&quot;\n\n&quot;</span>, <span class="hljs-string">&quot;###&quot;</span>])<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="vllm" label="vLLM"> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->params = SamplingParams(
max_tokens=<span class="hljs-number">100</span>,
min_tokens=<span class="hljs-number">10</span>,
stop=[<span class="hljs-string">&quot;###&quot;</span>, <span class="hljs-string">&quot;\n\n&quot;</span>],
ignore_eos=<span class="hljs-literal">False</span>,
skip_special_tokens=<span class="hljs-literal">True</span>,
)<!----></pre></div><!----></div> &lt;/hfoption><!----></div><!----> <!--[1--><h2 class="relative group"><a id="مدیریت-حافظه" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#مدیریت-حافظه"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>مدیریت حافظه</span></h2><!--]--><!----> <p>هر دو چارچوب تکنیک‌های پیشرفته‌ی مدیریت حافظه را برای استنتاج کارآمد پیاده‌سازی می‌کنند.</p> <div class="flex space-x-2 items-center my-1.5 mr-8 h-7 !pl-0 -mx-3 md:mx-0"><!--[--><!--]--></div> <div class="language-select"><!---->&lt;hfoption value="tgi" label="TGI"> <p>TGI از Flash Attention 2 و بتچ‌سازی پیوسته استفاده می‌کند:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-sh "><!----><span class="hljs-comment"># Docker deployment with memory optimization</span>
docker run --gpus all -p 8080:80 \
--shm-size 1g \
ghcr.io/huggingface/text-generation-inference:latest \
--model-id HuggingFaceTB/SmolLM2-1.7B-Instruct \
--max-batch-total-tokens 8192 \
--max-input-length 4096<!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="llama.cpp" label="llama.cpp"> <p>llama.cpp از کوانتیزه‌سازی و چیدمان بهینه‌ی حافظه استفاده می‌کند:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-sh "><!----><span class="hljs-comment"># Server with memory optimizations</span>
./server \
-m smollm2-1.7b-instruct.Q4_K_M.gguf \
--host 0.0.0.0 \
--port 8080 \
-c 2048 \ <span class="hljs-comment"># Context size</span>
--threads 4 \ <span class="hljs-comment"># CPU threads</span>
--n-gpu-layers 32 \ <span class="hljs-comment"># Use more GPU layers for larger models</span>
--mlock \ <span class="hljs-comment"># Lock memory to prevent swapping</span>
--cont-batching <span class="hljs-comment"># Enable continuous batching</span><!----></pre></div><!----></div> <p>برای مدل‌هایی که برای GPU شما بیش از حد بزرگ هستند، می‌توانید از تخلیه به CPU استفاده کنید:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-sh "><!---->./server \
-m smollm2-1.7b-instruct.Q4_K_M.gguf \
--n-gpu-layers 20 \ <span class="hljs-comment"># Keep first 20 layers on GPU</span>
--threads 8 <span class="hljs-comment"># Use more CPU threads for CPU layers</span><!----></pre></div><!----></div> &lt;/hfoption>
&lt;hfoption value="vllm" label="vLLM"> <p>vLLM برای مدیریت بهینه‌ی حافظه از PagedAttention استفاده می‌کند:</p> <div dir="ltr"><div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> vllm.engine.arg_utils <span class="hljs-keyword">import</span> AsyncEngineArgs
engine_args = AsyncEngineArgs(
model=<span class="hljs-string">&quot;HuggingFaceTB/SmolLM2-1.7B-Instruct&quot;</span>,
gpu_memory_utilization=<span class="hljs-number">0.85</span>,
max_num_batched_tokens=<span class="hljs-number">8192</span>,
block_size=<span class="hljs-number">16</span>,
)
llm = LLM(engine_args=engine_args)<!----></pre></div><!----></div> &lt;/hfoption><!----></div><!----> <!--[1--><h2 class="relative group"><a id="منابع" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#منابع"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>منابع</span></h2><!--]--><!----> <ul><li><a href="https://huggingface.co/docs/text-generation-inference" rel="nofollow">مستندات Text Generation Inference</a></li> <li><a href="https://github.com/huggingface/text-generation-inference" rel="nofollow">مخزن GitHub مربوط به TGI</a></li> <li><a href="https://vllm.readthedocs.io/" rel="nofollow">مستندات vLLM</a></li> <li><a href="https://github.com/vllm-project/vllm" rel="nofollow">مخزن GitHub مربوط به vLLM</a></li> <li><a href="https://arxiv.org/abs/2309.06180" rel="nofollow">مقاله‌ی PagedAttention</a></li> <li><a href="https://github.com/ggerganov/llama.cpp" rel="nofollow">مخزن GitHub مربوط به llama.cpp</a></li> <li><a href="https://github.com/abetlen/llama-cpp-python" rel="nofollow">مخزن llama-cpp-python</a></li></ul></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/fa/chapter2/8.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_pfm8ny = {
base: "/docs/course/pr_1364/fa",
assets: "/docs/course/pr_1364/fa"
};
const element = document.currentScript.parentElement;
Promise.all([
import("/docs/course/pr_1364/fa/_app/immutable/entry/start.CqZXmSrv.js"),
import("/docs/course/pr_1364/fa/_app/immutable/entry/app.B4UvgX5H.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 43],
data: [null,null],
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
117 kB
·
Xet hash:
960a0b2b96883ebf892679911e652c3da9ff4728d46eb0e5a12a67aa72a0d651

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.