Buckets:

download
raw
62.9 kB
<meta charset="utf-8" /><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;تمرین عملی: GRPO با Unsloth&quot;,&quot;local&quot;:&quot;تمرین-عملی-grpo-با-unsloth&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;نصب وابستگی‌ها&quot;,&quot;local&quot;:&quot;نصب-وابستگیها&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;راه‌اندازی Unsloth&quot;,&quot;local&quot;:&quot;راهاندازی-unsloth&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;آماده‌سازی داده‌ها&quot;,&quot;local&quot;:&quot;آمادهسازی-دادهها&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;تعریف توابع پاداش&quot;,&quot;local&quot;:&quot;تعریف-توابع-پاداش&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;آموزش با GRPO&quot;,&quot;local&quot;:&quot;آموزش-با-grpo&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;آزمایش مدل&quot;,&quot;local&quot;:&quot;آزمایش-مدل&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;ذخیرهٔ مدل&quot;,&quot;local&quot;:&quot;ذخیره-مدل&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;ارسال به Hugging Face Hub&quot;,&quot;local&quot;:&quot;ارسال-به-hugging-face-hub&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;جمع‌بندی&quot;,&quot;local&quot;:&quot;جمعبندی&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/>
<link href="/docs/course/pr_1364/fa/_app/immutable/entry/start.CqZXmSrv.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/oyrVqMhC.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/B3UUDp28.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/entry/app.B4UvgX5H.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/BioDZtGL.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/NIl2lQjl.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/DN6uzRQe.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/DsnmJJEf.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/nodes/0.DcFj1eGj.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/u5kFQ1Zb.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/nodes/34.CG3PPT_f.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/BjCHz3cj.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/B8VBSgkq.js" rel="modulepreload">
<link href="/docs/course/pr_1364/fa/_app/immutable/chunks/DFIrGYWg.js" rel="modulepreload">
<!--d727is--><meta name="hf:doc:metadata" content="{&quot;title&quot;:&quot;تمرین عملی: GRPO با Unsloth&quot;,&quot;local&quot;:&quot;تمرین-عملی-grpo-با-unsloth&quot;,&quot;sections&quot;:[{&quot;title&quot;:&quot;نصب وابستگی‌ها&quot;,&quot;local&quot;:&quot;نصب-وابستگیها&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;راه‌اندازی Unsloth&quot;,&quot;local&quot;:&quot;راهاندازی-unsloth&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;آماده‌سازی داده‌ها&quot;,&quot;local&quot;:&quot;آمادهسازی-دادهها&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;تعریف توابع پاداش&quot;,&quot;local&quot;:&quot;تعریف-توابع-پاداش&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;آموزش با GRPO&quot;,&quot;local&quot;:&quot;آموزش-با-grpo&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;آزمایش مدل&quot;,&quot;local&quot;:&quot;آزمایش-مدل&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;ذخیرهٔ مدل&quot;,&quot;local&quot;:&quot;ذخیره-مدل&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;ارسال به Hugging Face Hub&quot;,&quot;local&quot;:&quot;ارسال-به-hugging-face-hub&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2},{&quot;title&quot;:&quot;جمع‌بندی&quot;,&quot;local&quot;:&quot;جمعبندی&quot;,&quot;sections&quot;:[],&quot;depth&quot;:2}],&quot;depth&quot;:1}"/><!---->
<link href="/docs/course/pr_1364/fa/_app/immutable/assets/0.tn0RQdqM.css" rel="modulepreload"> <!--[--><!--[0--><!--[--><!--[0--><!--[--><p></p> <div class="flex space-x-1 absolute z-10 right-0 top-0"><a href="https://discuss.huggingface.co/t/chapter-2-questions" target="_blank"><img alt="Ask a Question" class="!m-0" src="https://img.shields.io/badge/Ask%20a%20question-ffcb4c.svg?logo=data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHZpZXdCb3g9IjAgLTEgMTA0IDEwNiI+PGRlZnM+PHN0eWxlPi5jbHMtMXtmaWxsOiMyMzFmMjA7fS5jbHMtMntmaWxsOiNmZmY5YWU7fS5jbHMtM3tmaWxsOiMwMGFlZWY7fS5jbHMtNHtmaWxsOiMwMGE5NGY7fS5jbHMtNXtmaWxsOiNmMTVkMjI7fS5jbHMtNntmaWxsOiNlMzFiMjM7fTwvc3R5bGU+PC9kZWZzPjx0aXRsZT5EaXNjb3Vyc2VfbG9nbzwvdGl0bGU+PGcgaWQ9IkxheWVyXzIiPjxnIGlkPSJMYXllcl8zIj48cGF0aCBjbGFzcz0iY2xzLTEiIGQ9Ik01MS44NywwQzIzLjcxLDAsMCwyMi44MywwLDUxYzAsLjkxLDAsNTIuODEsMCw1Mi44MWw1MS44Ni0uMDVjMjguMTYsMCw1MS0yMy43MSw1MS01MS44N1M4MCwwLDUxLjg3LDBaIi8+PHBhdGggY2xhc3M9ImNscy0yIiBkPSJNNTIuMzcsMTkuNzRBMzEuNjIsMzEuNjIsMCwwLDAsMjQuNTgsNjYuNDFsLTUuNzIsMTguNEwzOS40LDgwLjE3YTMxLjYxLDMxLjYxLDAsMSwwLDEzLTYwLjQzWiIvPjxwYXRoIGNsYXNzPSJjbHMtMyIgZD0iTTc3LjQ1LDMyLjEyYTMxLjYsMzEuNiwwLDAsMS0zOC4wNSw0OEwxOC44Niw4NC44MmwyMC45MS0yLjQ3QTMxLjYsMzEuNiwwLDAsMCw3Ny40NSwzMi4xMloiLz48cGF0aCBjbGFzcz0iY2xzLTQiIGQ9Ik03MS42MywyNi4yOUEzMS42LDMxLjYsMCwwLDEsMzguOCw3OEwxOC44Niw4NC44MiwzOS40LDgwLjE3QTMxLjYsMzEuNiwwLDAsMCw3MS42MywyNi4yOVoiLz48cGF0aCBjbGFzcz0iY2xzLTUiIGQ9Ik0yNi40Nyw2Ny4xMWEzMS42MSwzMS42MSwwLDAsMSw1MS0zNUEzMS42MSwzMS42MSwwLDAsMCwyNC41OCw2Ni40MWwtNS43MiwxOC40WiIvPjxwYXRoIGNsYXNzPSJjbHMtNiIgZD0iTTI0LjU4LDY2LjQxQTMxLjYxLDMxLjYxLDAsMCwxLDcxLjYzLDI2LjI5YTMxLjYxLDMxLjYxLDAsMCwwLTQ5LDM5LjYzbC0zLjc2LDE4LjlaIi8+PC9nPjwvZz48L3N2Zz4="/></a><!----> <!--[0--><a href="https://colab.research.google.com/github/unslothai/notebooks/blob/main/nb/HuggingFace%20Course-Gemma3_(1B)-GRPO.ipynb" target="_blank"><img alt="Open In Colab" class="!m-0" src="https://colab.research.google.com/assets/colab-badge.svg"/></a><!--]--> <!--[-1--><!--]--></div><!----> <div dir="rtl"><div class="items-center shrink-0 min-w-[100px] max-sm:min-w-[50px] justify-end ml-auto flex" style="float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"><div class="inline-flex rounded-md max-sm:rounded-sm"><button class="inline-flex items-center gap-1 h-7 max-sm:h-7 px-2 max-sm:px-1.5 text-sm font-medium text-gray-800 border border-r-0 rounded-l-md max-sm:rounded-l-sm border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-live="polite"><span class="inline-flex items-center justify-center rounded-md p-0.5 max-sm:p-0 hover:text-gray-800 dark:hover:text-gray-200"><svg class="sm:size-3.5 size-3" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----></span> <span>Copy page</span></button> <button class="inline-flex items-center justify-center w-6 max-sm:w-5 h-7 max-sm:h-7 disabled:pointer-events-none text-sm text-gray-500 hover:text-gray-700 dark:hover:text-white rounded-r-md max-sm:rounded-r-sm border border-l transition border-gray-200 bg-white hover:shadow-inner dark:border-gray-850 dark:bg-gray-950 dark:text-gray-200 dark:hover:bg-gray-800" aria-haspopup="menu" aria-expanded="false" aria-label="Open copy menu"><svg class="transition-transform text-gray-400 overflow-visible sm:size-3.5 size-3 rotate-0" width="1em" height="1em" viewBox="0 0 12 7" fill="none" xmlns="http://www.w3.org/2000/svg"><path d="M1 1L6 6L11 1" stroke="currentColor"></path></svg><!----></button></div> <!--[-1--><!--]--></div><!----> <!--[0--><h1 class="relative group"><a id="تمرین-عملی-grpo-با-unsloth" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#تمرین-عملی-grpo-با-unsloth"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>تمرین عملی: GRPO با Unsloth</span></h1><!--]--><!----> <p>در این تمرین، با استفاده از Unsloth یک مدل را با GRPO (بهینه‌سازی خط‌مشی نسبی گروهی) ریزتنظیم می‌کنید تا توانایی‌های استدلالی آن بهبود یابد. GRPO را در <a href="/course/chapter3/3">فصل ۳</a> بررسی کردیم.</p> <p>Unsloth کتابخانه‌ای است که ریزتنظیم LLM را شتاب می‌دهد و آموزش مدل‌ها را با سرعت بیشتر و منابع محاسباتی کمتر ممکن می‌کند. Unsloth به TRL متصل می‌شود؛ بنابراین بر مطالب بخش‌های قبلی تکیه می‌کنیم و آن‌ها را با ویژگی‌های Unsloth سازگار می‌سازیم.</p> <blockquote class="tip"><p>این تمرین را می‌توان روی GPU رایگان T4 در Google Colab اجرا کرد. برای بهترین تجربه، همراه با نوت‌بوک لینک‌شده در بالا پیش بروید و خودتان آن را امتحان کنید.</p></blockquote> <!--[1--><h2 class="relative group"><a id="نصب-وابستگیها" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#نصب-وابستگیها"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>نصب وابستگی‌ها</span></h2><!--]--><!----> <p>ابتدا کتابخانه‌های لازم را نصب می‌کنیم. برای ریزتنظیم شتاب‌یافته به Unsloth و برای استنتاج سریع به vLLM نیاز داریم.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-bash "><!---->pip install unsloth vllm
pip install --upgrade pillow<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="راهاندازی-unsloth" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#راهاندازی-unsloth"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>راه‌اندازی Unsloth</span></h2><!--]--><!----> <p>Unsloth کلاسی (<code>FastLanguageModel</code>) ارائه می‌دهد که transformers را با بهینه‌سازی‌های Unsloth یکپارچه می‌کند. بیایید آن را وارد کنیم:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> unsloth <span class="hljs-keyword">import</span> FastLanguageModel<!----></pre></div><!----> <p>حالا مدل Gemma 3 1B Instruct گوگل را بارگذاری و برای ریزتنظیم پیکربندی می‌کنیم:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> unsloth <span class="hljs-keyword">import</span> FastLanguageModel
<span class="hljs-keyword">import</span> torch
max_seq_length = <span class="hljs-number">1024</span> <span class="hljs-comment"># Can increase for longer reasoning traces</span>
lora_rank = <span class="hljs-number">32</span> <span class="hljs-comment"># Larger rank = smarter, but slower</span>
model, tokenizer = FastLanguageModel.from_pretrained(
model_name=<span class="hljs-string">&quot;google/gemma-3-1b-it&quot;</span>,
max_seq_length=max_seq_length,
load_in_4bit=<span class="hljs-literal">True</span>, <span class="hljs-comment"># False for LoRA 16bit</span>
fast_inference=<span class="hljs-literal">True</span>, <span class="hljs-comment"># Enable vLLM fast inference</span>
max_lora_rank=lora_rank,
gpu_memory_utilization=<span class="hljs-number">0.6</span>, <span class="hljs-comment"># Reduce if out of memory</span>
)
model = FastLanguageModel.get_peft_model(
model,
r=lora_rank, <span class="hljs-comment"># Choose any number &gt; 0 ! Suggested 8, 16, 32, 64, 128</span>
target_modules=[
<span class="hljs-string">&quot;q_proj&quot;</span>,
<span class="hljs-string">&quot;k_proj&quot;</span>,
<span class="hljs-string">&quot;v_proj&quot;</span>,
<span class="hljs-string">&quot;o_proj&quot;</span>,
<span class="hljs-string">&quot;gate_proj&quot;</span>,
<span class="hljs-string">&quot;up_proj&quot;</span>,
<span class="hljs-string">&quot;down_proj&quot;</span>,
], <span class="hljs-comment"># Remove QKVO if out of memory</span>
lora_alpha=lora_rank,
use_gradient_checkpointing=<span class="hljs-string">&quot;unsloth&quot;</span>, <span class="hljs-comment"># Enable long context finetuning</span>
random_state=<span class="hljs-number">3407</span>,
)<!----></pre></div><!----> <p>این کد مدل را برای صرفه‌جویی در حافظه با کوانتیزه‌سازی ۴ بیتی بارگذاری می‌کند و برای ریزتنظیم کارآمد، LoRA (سازگارسازی کم‌رتبه) را اعمال می‌کند. پارامتر <code>target_modules</code> لایه‌هایی از مدل را که باید ریزتنظیم شوند مشخص می‌کند و <code>use_gradient_checkpointing</code> آموزش با زمینه‌های طولانی‌تر را فعال می‌سازد.</p> <blockquote class="tip"><p>در این فصل جزئیات LoRA را بررسی نمی‌کنیم، اما می‌توانید در <a href="/course/chapter11/3">فصل ۱۱</a> بیشتر دربارهٔ آن بیاموزید.</p></blockquote> <!--[1--><h2 class="relative group"><a id="آمادهسازی-دادهها" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#آمادهسازی-دادهها"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>آماده‌سازی داده‌ها</span></h2><!--]--><!----> <p>برای این تمرین از مجموعه‌دادهٔ GSM8K استفاده می‌کنیم که شامل مسائل ریاضی مقطع ابتدایی است. داده‌ها را طوری قالب‌بندی می‌کنیم که مدل پیش از ارائهٔ پاسخ، استدلال خود را نشان دهد.</p> <p>ابتدا قالب پرامپت‌ها و پاسخ‌ها را تعریف می‌کنیم:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># Define the system prompt that instructs the model to use a specific format</span>
SYSTEM_PROMPT = <span class="hljs-string">&quot;&quot;&quot;
Respond in the following format:
&lt;reasoning&gt;
...
&lt;/reasoning&gt;
&lt;answer&gt;
...
&lt;/answer&gt;
&quot;&quot;&quot;</span>
XML_COT_FORMAT = <span class="hljs-string">&quot;&quot;&quot;\
&lt;reasoning&gt;
{reasoning}
&lt;/reasoning&gt;
&lt;answer&gt;
{answer}
&lt;/answer&gt;
&quot;&quot;&quot;</span><!----></pre></div><!----> <p>حالا مجموعه‌داده را آماده می‌کنیم:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">import</span> re
<span class="hljs-keyword">from</span> datasets <span class="hljs-keyword">import</span> load_dataset, Dataset
<span class="hljs-comment"># Helper functions to extract answers from different formats</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">extract_xml_answer</span>(<span class="hljs-params">text: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span>:
answer = text.split(<span class="hljs-string">&quot;&lt;answer&gt;&quot;</span>)[-<span class="hljs-number">1</span>]
answer = answer.split(<span class="hljs-string">&quot;&lt;/answer&gt;&quot;</span>)[<span class="hljs-number">0</span>]
<span class="hljs-keyword">return</span> answer.strip()
<span class="hljs-keyword">def</span> <span class="hljs-title function_">extract_hash_answer</span>(<span class="hljs-params">text: <span class="hljs-built_in">str</span></span>) -&gt; <span class="hljs-built_in">str</span> | <span class="hljs-literal">None</span>:
<span class="hljs-keyword">if</span> <span class="hljs-string">&quot;####&quot;</span> <span class="hljs-keyword">not</span> <span class="hljs-keyword">in</span> text:
<span class="hljs-keyword">return</span> <span class="hljs-literal">None</span>
<span class="hljs-keyword">return</span> text.split(<span class="hljs-string">&quot;####&quot;</span>)[<span class="hljs-number">1</span>].strip()
<span class="hljs-comment"># Function to prepare the GSM8K dataset</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_gsm8k_questions</span>(<span class="hljs-params">split=<span class="hljs-string">&quot;train&quot;</span></span>) -&gt; Dataset:
data = load_dataset(<span class="hljs-string">&quot;openai/gsm8k&quot;</span>, <span class="hljs-string">&quot;main&quot;</span>)[split]
data = data.<span class="hljs-built_in">map</span>(
<span class="hljs-keyword">lambda</span> x: {
<span class="hljs-string">&quot;prompt&quot;</span>: [
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: SYSTEM_PROMPT},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: x[<span class="hljs-string">&quot;question&quot;</span>]},
],
<span class="hljs-string">&quot;answer&quot;</span>: extract_hash_answer(x[<span class="hljs-string">&quot;answer&quot;</span>]),
}
)
<span class="hljs-keyword">return</span> data
dataset = get_gsm8k_questions()<!----></pre></div><!----> <p>مجموعه‌داده با استخراج پاسخ از داده‌ها و قالب‌بندی آن به‌صورت رشته آماده می‌شود.</p> <!--[1--><h2 class="relative group"><a id="تعریف-توابع-پاداش" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#تعریف-توابع-پاداش"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>تعریف توابع پاداش</span></h2><!--]--><!----> <p>همان‌طور که در <a href="/course/chapter13/4">یک صفحه‌ی پیشین</a> گفتیم، GRPO می‌تواند با تکیه بر معیارهای قابل‌راستی‌آزمایی مانند طول و قالب‌بندی، از توابع پاداش برای هدایت یادگیری مدل استفاده کند.</p> <p>در این تمرین چند تابع پاداش تعریف می‌کنیم که جنبه‌های مختلف استدلال خوب را تقویت می‌کنند. برای نمونه، به مدل برای ارائهٔ پاسخ به‌صورت عدد صحیح و پیروی از قالب دقیق پاداش می‌دهیم.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># Reward function that checks if the answer is correct</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">correctness_reward_func</span>(<span class="hljs-params">prompts, completions, answer, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
q = prompts[<span class="hljs-number">0</span>][-<span class="hljs-number">1</span>][<span class="hljs-string">&quot;content&quot;</span>]
extracted_responses = [extract_xml_answer(r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
<span class="hljs-built_in">print</span>(
<span class="hljs-string">&quot;-&quot;</span> * <span class="hljs-number">20</span>,
<span class="hljs-string">f&quot;Question:\n<span class="hljs-subst">{q}</span>&quot;</span>,
<span class="hljs-string">f&quot;\nAnswer:\n<span class="hljs-subst">{answer[<span class="hljs-number">0</span>]}</span>&quot;</span>,
<span class="hljs-string">f&quot;\nResponse:\n<span class="hljs-subst">{responses[<span class="hljs-number">0</span>]}</span>&quot;</span>,
<span class="hljs-string">f&quot;\nExtracted:\n<span class="hljs-subst">{extracted_responses[<span class="hljs-number">0</span>]}</span>&quot;</span>,
)
<span class="hljs-keyword">return</span> [<span class="hljs-number">2.0</span> <span class="hljs-keyword">if</span> r == a <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> r, a <span class="hljs-keyword">in</span> <span class="hljs-built_in">zip</span>(extracted_responses, answer)]
<span class="hljs-comment"># Reward function that checks if the answer is an integer</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">int_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
extracted_responses = [extract_xml_answer(r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
<span class="hljs-keyword">return</span> [<span class="hljs-number">0.5</span> <span class="hljs-keyword">if</span> r.isdigit() <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> extracted_responses]
<span class="hljs-comment"># Reward function that checks if the completion follows the strict format</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">strict_format_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
pattern = <span class="hljs-string">r&quot;^&lt;reasoning&gt;\n.*?\n&lt;/reasoning&gt;\n&lt;answer&gt;\n.*?\n&lt;/answer&gt;\n$&quot;</span>
responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
matches = [re.<span class="hljs-keyword">match</span>(pattern, r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
<span class="hljs-keyword">return</span> [<span class="hljs-number">0.5</span> <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">in</span> matches]
<span class="hljs-comment"># Reward function that checks if the completion follows a more relaxed format</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">soft_format_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
pattern = <span class="hljs-string">r&quot;&lt;reasoning&gt;.*?&lt;/reasoning&gt;\s*&lt;answer&gt;.*?&lt;/answer&gt;&quot;</span>
responses = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
matches = [re.<span class="hljs-keyword">match</span>(pattern, r) <span class="hljs-keyword">for</span> r <span class="hljs-keyword">in</span> responses]
<span class="hljs-keyword">return</span> [<span class="hljs-number">0.5</span> <span class="hljs-keyword">if</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0.0</span> <span class="hljs-keyword">for</span> <span class="hljs-keyword">match</span> <span class="hljs-keyword">in</span> matches]
<span class="hljs-comment"># Reward function that counts XML tags and penalizes extra content</span>
<span class="hljs-keyword">def</span> <span class="hljs-title function_">count_xml</span>(<span class="hljs-params">text</span>) -&gt; <span class="hljs-built_in">float</span>:
count = <span class="hljs-number">0.0</span>
<span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;&lt;reasoning&gt;\n&quot;</span>) == <span class="hljs-number">1</span>:
count += <span class="hljs-number">0.125</span>
<span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;\n&lt;/reasoning&gt;\n&quot;</span>) == <span class="hljs-number">1</span>:
count += <span class="hljs-number">0.125</span>
<span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;\n&lt;answer&gt;\n&quot;</span>) == <span class="hljs-number">1</span>:
count += <span class="hljs-number">0.125</span>
count -= <span class="hljs-built_in">len</span>(text.split(<span class="hljs-string">&quot;\n&lt;/answer&gt;\n&quot;</span>)[-<span class="hljs-number">1</span>]) * <span class="hljs-number">0.001</span>
<span class="hljs-keyword">if</span> text.count(<span class="hljs-string">&quot;\n&lt;/answer&gt;&quot;</span>) == <span class="hljs-number">1</span>:
count += <span class="hljs-number">0.125</span>
count -= (<span class="hljs-built_in">len</span>(text.split(<span class="hljs-string">&quot;\n&lt;/answer&gt;&quot;</span>)[-<span class="hljs-number">1</span>]) - <span class="hljs-number">1</span>) * <span class="hljs-number">0.001</span>
<span class="hljs-keyword">return</span> count
<span class="hljs-keyword">def</span> <span class="hljs-title function_">xmlcount_reward_func</span>(<span class="hljs-params">completions, **kwargs</span>) -&gt; <span class="hljs-built_in">list</span>[<span class="hljs-built_in">float</span>]:
contents = [completion[<span class="hljs-number">0</span>][<span class="hljs-string">&quot;content&quot;</span>] <span class="hljs-keyword">for</span> completion <span class="hljs-keyword">in</span> completions]
<span class="hljs-keyword">return</span> [count_xml(c) <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> contents]<!----></pre></div><!----> <p>این توابع پاداش هدف‌های متفاوتی دارند:</p> <table><thead><tr><th>تابع پاداش</th><th>هدف</th></tr></thead><tbody><tr><td><code>correctness_reward_func</code></td><td>وقتی پاسخ مدل با پاسخ درست مطابقت دارد، به مدل پاداش می‌دهد</td></tr><tr><td><code>int_reward_func</code></td><td>برای ارائه‌ی یک پاسخ عددی به مدل پاداش می‌دهد</td></tr><tr><td><code>strict_format_reward_func</code> و <code>soft_format_reward_func</code></td><td>برای رعایت قالب مشخص‌شده به مدل پاداش می‌دهد</td></tr><tr><td><code>xmlcount_reward_func</code></td><td>استفاده‌ی درست از برچسب‌های XML را پاداش می‌دهد و محتوای اضافی پس از برچسب‌های بسته‌شدن را جریمه می‌کند</td></tr></tbody></table> <!--[1--><h2 class="relative group"><a id="آموزش-با-grpo" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#آموزش-با-grpo"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>آموزش با GRPO</span></h2><!--]--><!----> <p>حالا آموزش‌دهندهٔ GRPO را با مدل، توکن‌ساز و توابع پاداش خود راه‌اندازی می‌کنیم. این بخش همان رویکرد <a href="/course/chapter12/5">تمرین قبلی</a> را دنبال می‌کند.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> trl <span class="hljs-keyword">import</span> GRPOConfig, GRPOTrainer
max_prompt_length = <span class="hljs-number">256</span>
training_args = GRPOConfig(
learning_rate=<span class="hljs-number">5e-6</span>,
adam_beta1=<span class="hljs-number">0.9</span>,
adam_beta2=<span class="hljs-number">0.99</span>,
weight_decay=<span class="hljs-number">0.1</span>,
warmup_ratio=<span class="hljs-number">0.1</span>,
lr_scheduler_type=<span class="hljs-string">&quot;cosine&quot;</span>,
optim=<span class="hljs-string">&quot;paged_adamw_8bit&quot;</span>,
logging_steps=<span class="hljs-number">1</span>,
per_device_train_batch_size=<span class="hljs-number">1</span>,
gradient_accumulation_steps=<span class="hljs-number">1</span>, <span class="hljs-comment"># Increase to 4 for smoother training</span>
num_generations=<span class="hljs-number">6</span>, <span class="hljs-comment"># Decrease if out of memory</span>
max_prompt_length=max_prompt_length,
max_completion_length=max_seq_length - max_prompt_length,
<span class="hljs-comment"># num_train_epochs = 1, # Set to 1 for a full training run</span>
max_steps=<span class="hljs-number">250</span>,
save_steps=<span class="hljs-number">250</span>,
max_grad_norm=<span class="hljs-number">0.1</span>,
report_to=<span class="hljs-string">&quot;none&quot;</span>, <span class="hljs-comment"># Can use Weights &amp; Biases</span>
output_dir=<span class="hljs-string">&quot;outputs&quot;</span>,
)
trainer = GRPOTrainer(
model=model,
processing_class=tokenizer,
reward_funcs=[
xmlcount_reward_func,
soft_format_reward_func,
strict_format_reward_func,
int_reward_func,
correctness_reward_func,
],
args=training_args,
train_dataset=dataset,
)<!----></pre></div><!----> <p><code>GRPOConfig</code> ابرپارامترهای مختلف آموزش را تنظیم می‌کند:</p> <ul><li><code>use_vllm</code>: استنتاج سریع با vLLM را فعال می‌کند.</li> <li><code>learning_rate</code>: سرعت یادگیری مدل را کنترل می‌کند.</li> <li><code>num_generations</code>: تعداد تکمیل‌هایی که برای هر پرامپت تولید می‌شوند.</li> <li><code>max_steps</code>: تعداد کل گام‌های آموزشی که باید انجام شوند.</li></ul> <p>حالا آموزش را شروع می‌کنیم:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->trainer.train()<!----></pre></div><!----> <blockquote class="warning"><p>آموزش ممکن است مدتی طول بکشد. شاید افزایش پاداش‌ها را بلافاصله نبینید؛ ممکن است ۱۵۰ تا ۲۰۰ گام طول بکشد تا بهبودها ظاهر شوند. صبور باشید!</p></blockquote> <!--[1--><h2 class="relative group"><a id="آزمایش-مدل" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#آزمایش-مدل"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>آزمایش مدل</span></h2><!--]--><!----> <p>پس از آموزش، مدل را آزمایش می‌کنیم تا عملکرد آن را ببینیم. ابتدا وزن‌های LoRA را ذخیره می‌کنیم:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->model.save_lora(<span class="hljs-string">&quot;grpo_saved_lora&quot;</span>)<!----></pre></div><!----> <p>حالا مدل را با یک پرسش جدید آزمایش می‌کنیم:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-keyword">from</span> vllm <span class="hljs-keyword">import</span> SamplingParams
text = tokenizer.apply_chat_template(
[
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;system&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: SYSTEM_PROMPT},
{<span class="hljs-string">&quot;role&quot;</span>: <span class="hljs-string">&quot;user&quot;</span>, <span class="hljs-string">&quot;content&quot;</span>: <span class="hljs-string">&quot;Calculate pi.&quot;</span>},
],
tokenize=<span class="hljs-literal">False</span>,
add_generation_prompt=<span class="hljs-literal">True</span>,
)
sampling_params = SamplingParams(
temperature=<span class="hljs-number">0.8</span>,
top_p=<span class="hljs-number">0.95</span>,
max_tokens=<span class="hljs-number">1024</span>,
)
output = (
model.fast_generate(
text,
sampling_params=sampling_params,
lora_request=model.load_lora(<span class="hljs-string">&quot;grpo_saved_lora&quot;</span>),
)[<span class="hljs-number">0</span>]
.outputs[<span class="hljs-number">0</span>]
.text
)
<span class="hljs-built_in">print</span>(output)<!----></pre></div><!----> <p>باید ببینید که مدل اکنون از قالب مشخص‌شده پیروی می‌کند و پیش از ارائهٔ پاسخ، استدلال خود را نشان می‌دهد.</p> <!--[1--><h2 class="relative group"><a id="ذخیره-مدل" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ذخیره-مدل"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>ذخیرهٔ مدل</span></h2><!--]--><!----> <p>Unsloth چند گزینه برای ذخیرهٔ مدل ریزتنظیم‌شدهٔ شما ارائه می‌دهد، اما بر رایج‌ترین گزینه تمرکز می‌کنیم.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># Save to 16-bit precision</span>
model.save_pretrained_merged(<span class="hljs-string">&quot;model&quot;</span>, tokenizer, save_method=<span class="hljs-string">&quot;merged_16bit&quot;</span>)<!----></pre></div><!----> <!--[1--><h2 class="relative group"><a id="ارسال-به-hugging-face-hub" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#ارسال-به-hugging-face-hub"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>ارسال به Hugging Face Hub</span></h2><!--]--><!----> <p>مدل را با استفاده از متد <code>push_to_hub_merged</code> به Hugging Face Hub ارسال می‌کنیم. این متد اجازه می‌دهد مدل را در چند قالب کوانتیزه‌سازی ارسال کنیم.</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!----><span class="hljs-comment"># Push to Hugging Face Hub (requires a token)</span>
model.push_to_hub_merged(
<span class="hljs-string">&quot;your-username/model-name&quot;</span>, tokenizer, save_method=<span class="hljs-string">&quot;merged_16bit&quot;</span>, token=<span class="hljs-string">&quot;your-token&quot;</span>
)<!----></pre></div><!----> <p>Unsloth همچنین ذخیره‌سازی در قالب GGUF را برای استفاده با llama.cpp پشتیبانی می‌کند:</p> <div class="code-block relative "><div class="absolute top-2.5 right-4"><button class="inline-flex items-center relative text-sm focus:text-green-500 cursor-pointer focus:outline-none transition duration-200 ease-in-out opacity-0 mx-0.5 text-gray-600 " title="code excerpt" type="button"><svg xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M28,10V28H10V10H28m0-2H10a2,2,0,0,0-2,2V28a2,2,0,0,0,2,2H28a2,2,0,0,0,2-2V10a2,2,0,0,0-2-2Z" transform="translate(0)"></path><path d="M4,18H2V4A2,2,0,0,1,4,2H18V4H4Z" transform="translate(0)"></path><rect fill="none" width="32" height="32"></rect></svg><!----> <div class=" absolute pointer-events-none transition-opacity bg-black text-white py-1 px-2 leading-tight rounded font-normal shadow left-1/2 top-full transform -translate-x-1/2 translate-y-2 opacity-0 "><div class="absolute bottom-full left-1/2 transform -translate-x-1/2 w-0 h-0 border-black border-4 border-t-0" style="border-left-color: transparent; border-right-color: transparent;"></div> Copied</div><!----></button><!----></div> <pre class="language-python "><!---->model.push_to_hub_gguf(
<span class="hljs-string">&quot;your-username/model-name&quot;</span>,
tokenizer,
quantization_method=[<span class="hljs-string">&quot;q4_k_m&quot;</span>, <span class="hljs-string">&quot;q8_0&quot;</span>, <span class="hljs-string">&quot;q5_k_m&quot;</span>],
token=<span class="hljs-string">&quot;your-token&quot;</span>,
)<!----></pre></div><!----> <p>فایل‌های GGUF را می‌توان با llama.cpp یا سامانه‌های مبتنی بر رابط کاربری مانند Jan و Open WebUI استفاده کرد.</p> <!--[1--><h2 class="relative group"><a id="جمعبندی" class="header-link block pr-1.5 text-lg no-hover:hidden with-hover:absolute with-hover:p-1.5 with-hover:opacity-0 with-hover:group-hover:opacity-100 with-hover:right-full" href="#جمعبندی"><span><svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" aria-hidden="true" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 256 256"><path d="M167.594 88.393a8.001 8.001 0 0 1 0 11.314l-67.882 67.882a8 8 0 1 1-11.314-11.315l67.882-67.881a8.003 8.003 0 0 1 11.314 0zm-28.287 84.86l-28.284 28.284a40 40 0 0 1-56.567-56.567l28.284-28.284a8 8 0 0 0-11.315-11.315l-28.284 28.284a56 56 0 0 0 79.196 79.197l28.285-28.285a8 8 0 1 0-11.315-11.314zM212.852 43.14a56.002 56.002 0 0 0-79.196 0l-28.284 28.284a8 8 0 1 0 11.314 11.314l28.284-28.284a40 40 0 0 1 56.568 56.567l-28.285 28.285a8 8 0 0 0 11.315 11.314l28.284-28.284a56.065 56.065 0 0 0 0-79.196z" fill="currentColor"></path></svg><!----></span></a> <span>جمع‌بندی</span></h2><!--]--><!----> <p>در این تمرین یاد گرفتید چگونه:</p> <ol><li>Unsloth را برای ریزتنظیم شتاب‌یافته راه‌اندازی کنید.</li> <li>داده‌ها را برای آموزش GRPO آماده کنید.</li> <li>توابع پاداش سفارشی برای هدایت یادگیری مدل تعریف کنید.</li> <li>مدلی را با استفاده از GRPO آموزش دهید.</li> <li>مدل ریزتنظیم‌شده را آزمایش کنید.</li> <li>مدل را در قالب‌های مختلف ذخیره کنید.</li></ol> <p>GRPO روشی قدرتمند برای هم‌راستا کردن مدل‌های زبانی با رفتارهای مشخص است و Unsloth آن را حتی روی سخت‌افزار محدود نیز در دسترس می‌کند. با ترکیب چند تابع پاداش، می‌توانید مدل را به پیروی از قالبی مشخص هدایت کنید و هم‌زمان توانایی‌های استدلالی آن را بهبود دهید.</p> <p>برای اطلاعات و منابع بیشتر، این موارد را ببینید:</p> <ul><li><a href="https://docs.unsloth.ai/" rel="nofollow">مستندات Unsloth</a></li> <li><a href="https://discord.gg/unsloth" rel="nofollow">دیسکورد Unsloth</a></li> <li><a href="https://github.com/unslothai/unsloth" rel="nofollow">گیت‌هاب Unsloth</a></li></ul></div> <a class="!text-gray-400 !no-underline text-sm flex items-center not-prose mt-4" href="https://github.com/huggingface/course/blob/main/chapters/fa/chapter12/6.mdx" target="_blank"><svg class="mr-1" xmlns="http://www.w3.org/2000/svg" aria-hidden="true" fill="currentColor" focusable="false" role="img" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 32 32"><path d="M31,16l-7,7l-1.41-1.41L28.17,16l-5.58-5.59L24,9l7,7z"></path><path d="M1,16l7-7l1.41,1.41L3.83,16l5.58,5.59L8,23l-7-7z"></path><path d="M12.419,25.484L17.639,6.552l1.932,0.518L14.351,26.002z"></path></svg><!----> <span><span class="underline">Update</span> on GitHub</span></a><!----> <p></p><!--]--><!----><!--]--><!--]--><!--]--> <!--[-1--><!--]--><!--]-->
<script>
{
__sveltekit_pfm8ny = {
base: "/docs/course/pr_1364/fa",
assets: "/docs/course/pr_1364/fa"
};
const element = document.currentScript.parentElement;
Promise.all([
import("/docs/course/pr_1364/fa/_app/immutable/entry/start.CqZXmSrv.js"),
import("/docs/course/pr_1364/fa/_app/immutable/entry/app.B4UvgX5H.js")
]).then(([kit, app]) => {
kit.start(app, element, {
node_ids: [0, 34],
data: [null,null],
form: null,
error: null
});
});
}
</script>

Xet Storage Details

Size:
62.9 kB
·
Xet hash:
d4ef08edd4bf40ecd9228e7ec144805afde39ced3c9ed96d96e032f6e04a8b72

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.