Buckets:
| import"../chunks/DsnmJJEf.js";import{i as v,h as f,C as b,H as _,D as m,E as k,s as x,a as n}from"../chunks/DBLdnrwR.js";import{p as C,o as N,s as e,f as D,a as J,b as G,c as M,d as w,n as y,r as T}from"../chunks/Bb-LL0eD.js";import{E as o}from"../chunks/CbWRAJMO.js";const Z='{"title":"Doc","local":"lighteval.tasks.requests.Doc","sections":[],"depth":1}';var B=w('<meta name="hf:doc:metadata"/>'),R=w(`<p></p> <!> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Dataclass representing a single evaluation sample for a benchmark.</p> <p>This class encapsulates all the information needed to evaluate a model on a single | |
| task instance. It contains the input query, expected outputs, metadata, and | |
| configuration parameters for different types of evaluation tasks.</p> <p><strong>Required Fields:</strong></p> <ul><li><code>query</code>: The input prompt or question</li> <li><code>choices</code>: Available answer choices (for multiple choice tasks)</li> <li><code>gold_index</code>: Index(es) of the correct answer(s)</li></ul> <p><strong>Optional Fields:</strong></p> <ul><li><code>instruction</code>: System prompt, task specific. Will be appended to model specific system prompt.</li> <li><code>images</code>: Visual inputs for multimodal tasks.</li></ul> <p>Methods: | |
| get_golds(): | |
| Returns the correct answer(s) as strings based on gold_index. | |
| Handles both single and multiple correct answers.</p> <p>Usage Examples:</p> <p><strong>Multiple Choice Question:</strong></p> <!> <p><strong>Generative Task:</strong></p> <!> <p><strong>Few-shot Learning:</strong></p> <!> <p><strong>Multimodal Task:</strong></p> <!> <div class="docstring border-l-2 border-t-2 pl-4 pt-3.5 border-gray-100 rounded-tl-xl mb-6 mt-8"><!> <p>Return gold targets extracted from the target dict</p></div></div> <!> <p></p>`,1);function z(I,U){C(U,!1),N(()=>{new URLSearchParams(window.location.search).get("fw")}),v();var l=R();f("16f7qus",s=>{var t=B();x(t,"content",Z),J(s,t)});var r=e(D(l),2);b(r,{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"});var i=e(r,2);_(i,{title:"Doc",local:"lighteval.tasks.requests.Doc",headingTag:"h1"});var a=e(i,2),c=M(a);m(c,{name:"class lighteval.tasks.requests.Doc",anchor:"lighteval.tasks.requests.Doc",source:"https://github.com/huggingface/lighteval/blob/vr_1445/src/lighteval/tasks/requests.py#L44",parameters:[{name:"query",val:": str"},{name:"choices",val:": list"},{name:"gold_index",val:": typing.Union[int, list[int]]"},{name:"instruction",val:": str | None = None"},{name:"images",val:": list['Image'] | None = None"},{name:"specific",val:": dict | None = None"},{name:"unconditioned_query",val:": str | None = None"},{name:"original_query",val:": str | None = None"},{name:"id",val:": str = ''"},{name:"task_name",val:": str = ''"},{name:"fewshot_samples",val:": list = <factory>"},{name:"sampling_methods",val:": list = <factory>"},{name:"fewshot_sorting_class",val:": str | None = None"},{name:"generation_size",val:": int | None = None"},{name:"stop_sequences",val:": list[str] | None = None"},{name:"use_logits",val:": bool = False"},{name:"num_samples",val:": int = 1"},{name:"generation_grammar",val:": None = None"}],parametersDescription:[{anchor:"lighteval.tasks.requests.Doc.query",description:`<strong>query</strong> (str) — | |
| The main query, prompt, or question to be sent to the model.`,name:"query"},{anchor:"lighteval.tasks.requests.Doc.choices",description:`<strong>choices</strong> (list[str]) — | |
| List of possible answer choices for the query. | |
| For multiple choice tasks, this contains all options (A, B, C, D, etc.). | |
| For generative tasks, this may be empty or contain reference answers.`,name:"choices"},{anchor:"lighteval.tasks.requests.Doc.gold_index",description:`<strong>gold_index</strong> (Union[int, list[int]]) — | |
| Index or indices of the correct answer(s) in the choices list. | |
| For single correct answers,(e.g., 0 for first choice). | |
| For multiple correct answers, use a list (e.g., [0, 2] for first and third).`,name:"gold_index"},{anchor:"lighteval.tasks.requests.Doc.instruction",description:`<strong>instruction</strong> (str | None) — | |
| System prompt or task-specific instructions to guide the model. | |
| This is typically prepended to the query to set context or behavior.`,name:"instruction"},{anchor:"lighteval.tasks.requests.Doc.images",description:`<strong>images</strong> (list[“Image”] | None) — | |
| List of PIL Image objects for multimodal tasks.`,name:"images"},{anchor:"lighteval.tasks.requests.Doc.specific",description:`<strong>specific</strong> (dict | None) — | |
| Task-specific information or metadata. | |
| Can contain any additional data needed for evaluation.`,name:"specific"},{anchor:"lighteval.tasks.requests.Doc.unconditioned_query",description:`<strong>unconditioned_query</strong> (Optional[str]) — | |
| Query without task-specific context for PMI normalization. | |
| Used to calculate: log P(choice | Query) - log P(choice | Unconditioned Query).`,name:"unconditioned_query"},{anchor:"lighteval.tasks.requests.Doc.original_query",description:`<strong>original_query</strong> (str | None) — | |
| The query before any preprocessing or modification.`,name:"original_query"},{anchor:"lighteval.tasks.requests.Doc.#",description:"<strong>#</strong> Set by task parameters —",name:"#"},{anchor:"lighteval.tasks.requests.Doc.id",description:`<strong>id</strong> (str) — | |
| Unique identifier for this evaluation instance. | |
| Set by the task and not the user.`,name:"id"},{anchor:"lighteval.tasks.requests.Doc.task_name",description:`<strong>task_name</strong> (str) — | |
| Name of the task or benchmark this Doc belongs to.`,name:"task_name"},{anchor:"lighteval.tasks.requests.Doc.##",description:"<strong>##</strong> Few-shot Learning Parameters —",name:"##"},{anchor:"lighteval.tasks.requests.Doc.fewshot_samples",description:`<strong>fewshot_samples</strong> (list) — | |
| List of Doc objects representing few-shot examples. | |
| These examples are prepended to the main query to provide context.`,name:"fewshot_samples"},{anchor:"lighteval.tasks.requests.Doc.sampling_methods",description:`<strong>sampling_methods</strong> (list[SamplingMethod]) — | |
| List of sampling methods to use for this instance. | |
| Options: GENERATIVE, LOGPROBS, PERPLEXITY.`,name:"sampling_methods"},{anchor:"lighteval.tasks.requests.Doc.fewshot_sorting_class",description:`<strong>fewshot_sorting_class</strong> (Optional[str]) — | |
| Class label for balanced few-shot example selection. | |
| Used to ensure diverse representation in few-shot examples.`,name:"fewshot_sorting_class"},{anchor:"lighteval.tasks.requests.Doc.##",description:"<strong>##</strong> Generation Control Parameters —",name:"##"},{anchor:"lighteval.tasks.requests.Doc.generation_size",description:`<strong>generation_size</strong> (int | None) — | |
| Maximum number of tokens to generate for this instance.`,name:"generation_size"},{anchor:"lighteval.tasks.requests.Doc.stop_sequences",description:`<strong>stop_sequences</strong> (list[str] | None) — | |
| List of strings that should stop generation when encountered. | |
| <strong>Used for</strong>: Controlled generation, preventing unwanted continuations.`,name:"stop_sequences"},{anchor:"lighteval.tasks.requests.Doc.use_logits",description:`<strong>use_logits</strong> (bool) — | |
| Whether to return logits (raw model outputs) in addition to text. | |
| <strong>Used for</strong>: Probability analysis, confidence scoring, detailed evaluation.`,name:"use_logits"},{anchor:"lighteval.tasks.requests.Doc.num_samples",description:`<strong>num_samples</strong> (int) — | |
| Number of different samples to generate for this instance. | |
| <strong>Used for</strong>: Diversity analysis, uncertainty estimation, ensemble methods.`,name:"num_samples"},{anchor:"lighteval.tasks.requests.Doc.generation_grammar",description:`<strong>generation_grammar</strong> (None) — | |
| Grammar constraints for generation (currently not implemented). | |
| <strong>Reserved for</strong>: Future structured generation features.`,name:"generation_grammar"}]});var p=e(c,20);o(p,{anchor:"lighteval.tasks.requests.Doc.example",children:(s,t)=>{n(s,{code:"ZG9jJTIwJTNEJTIwRG9jKCUwQSUyMCUyMCUyMCUyMHF1ZXJ5JTNEJTIyV2hhdCUyMGlzJTIwdGhlJTIwY2FwaXRhbCUyMG9mJTIwRnJhbmNlJTNGJTIyJTJDJTBBJTIwJTIwJTIwJTIwY2hvaWNlcyUzRCU1QiUyMkxvbmRvbiUyMiUyQyUyMCUyMlBhcmlzJTIyJTJDJTIwJTIyQmVybGluJTIyJTJDJTIwJTIyTWFkcmlkJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDElMkMlMjAlMjAlMjMlMjBQYXJpcyUyMGlzJTIwdGhlJTIwY29ycmVjdCUyMGFuc3dlciUwQSUyMCUyMCUyMCUyMGluc3RydWN0aW9uJTNEJTIyQW5zd2VyJTIwdGhlJTIwZm9sbG93aW5nJTIwZ2VvZ3JhcGh5JTIwcXVlc3Rpb24lM0ElMjIlMkMlMEEp",highlighted:`doc = Doc( | |
| query=<span class="hljs-string">"What is the capital of France?"</span>, | |
| choices=[<span class="hljs-string">"London"</span>, <span class="hljs-string">"Paris"</span>, <span class="hljs-string">"Berlin"</span>, <span class="hljs-string">"Madrid"</span>], | |
| gold_index=<span class="hljs-number">1</span>, <span class="hljs-comment"># Paris is the correct answer</span> | |
| instruction=<span class="hljs-string">"Answer the following geography question:"</span>, | |
| )`,lang:"python",wrap:!1})},$$slots:{default:!0}});var h=e(p,4);o(h,{anchor:"lighteval.tasks.requests.Doc.example-2",children:(s,t)=>{n(s,{code:"ZG9jJTIwJTNEJTIwRG9jKCUwQSUyMCUyMCUyMCUyMHF1ZXJ5JTNEJTIyV3JpdGUlMjBhJTIwc2hvcnQlMjBzdG9yeSUyMGFib3V0JTIwYSUyMHJvYm90LiUyMiUyQyUwQSUyMCUyMCUyMCUyMGNob2ljZXMlM0QlNUIlNUQlMkMlMjAlMjAlMjMlMjBObyUyMHByZWRlZmluZWQlMjBjaG9pY2VzJTIwZm9yJTIwZ2VuZXJhdGl2ZSUyMHRhc2tzJTBBJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDAlMkMlMjAlMjAlMjMlMjBOb3QlMjB1c2VkJTIwZm9yJTIwZ2VuZXJhdGl2ZSUyMHRhc2tzJTBBJTIwJTIwJTIwJTIwZ2VuZXJhdGlvbl9zaXplJTNEMTAwJTJDJTBBJTIwJTIwJTIwJTIwc3RvcF9zZXF1ZW5jZXMlM0QlNUIlMjIlMEElMEFFbmQlMjIlNUQlMkMlMEEp",highlighted:`doc = Doc( | |
| query=<span class="hljs-string">"Write a short story about a robot."</span>, | |
| choices=[], <span class="hljs-comment"># No predefined choices for generative tasks</span> | |
| gold_index=<span class="hljs-number">0</span>, <span class="hljs-comment"># Not used for generative tasks</span> | |
| generation_size=<span class="hljs-number">100</span>, | |
| stop_sequences=[<span class="hljs-string">" | |
| End"</span>], | |
| )`,lang:"python",wrap:!1})},$$slots:{default:!0}});var g=e(h,4);o(g,{anchor:"lighteval.tasks.requests.Doc.example-3",children:(s,t)=>{n(s,{code:"ZG9jJTIwJTNEJTIwRG9jKCUwQSUyMCUyMCUyMCUyMHF1ZXJ5JTNEJTIyVHJhbnNsYXRlJTIwJ0hlbGxvJTIwd29ybGQnJTIwdG8lMjBTcGFuaXNoLiUyMiUyQyUwQSUyMCUyMCUyMCUyMGNob2ljZXMlM0QlNUIlMjJIb2xhJTIwbXVuZG8lMjIlMkMlMjAlMjJCb25qb3VyJTIwbW9uZGUlMjIlMkMlMjAlMjJDaWFvJTIwbW9uZG8lMjIlNUQlMkMlMEElMjAlMjAlMjAlMjBnb2xkX2luZGV4JTNEMCUyQyUwQSUyMCUyMCUyMCUyMGZld3Nob3Rfc2FtcGxlcyUzRCU1QiUwQSUyMCUyMCUyMCUyMCUyMCUyMCUyMCUyMERvYyhxdWVyeSUzRCUyMlRyYW5zbGF0ZSUyMCdHb29kJTIwbW9ybmluZyclMjB0byUyMFNwYW5pc2guJTIyJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwY2hvaWNlcyUzRCU1QiUyMkJ1ZW5vcyUyMGQlQzMlQURhcyUyMiUyQyUyMCUyMkJvbmpvdXIlMjIlMkMlMjAlMjJCdW9uZ2lvcm5vJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDApJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwRG9jKHF1ZXJ5JTNEJTIyVHJhbnNsYXRlJTIwJ1RoYW5rJTIweW91JyUyMHRvJTIwU3BhbmlzaC4lMjIlMkMlMEElMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjAlMjBjaG9pY2VzJTNEJTVCJTIyR3JhY2lhcyUyMiUyQyUyMCUyMk1lcmNpJTIyJTJDJTIwJTIyR3JhemllJTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDApJTBBJTIwJTIwJTIwJTIwJTVEJTJDJTBBKQ==",highlighted:`doc = Doc( | |
| query=<span class="hljs-string">"Translate 'Hello world' to Spanish."</span>, | |
| choices=[<span class="hljs-string">"Hola mundo"</span>, <span class="hljs-string">"Bonjour monde"</span>, <span class="hljs-string">"Ciao mondo"</span>], | |
| gold_index=<span class="hljs-number">0</span>, | |
| fewshot_samples=[ | |
| Doc(query=<span class="hljs-string">"Translate 'Good morning' to Spanish."</span>, | |
| choices=[<span class="hljs-string">"Buenos días"</span>, <span class="hljs-string">"Bonjour"</span>, <span class="hljs-string">"Buongiorno"</span>], | |
| gold_index=<span class="hljs-number">0</span>), | |
| Doc(query=<span class="hljs-string">"Translate 'Thank you' to Spanish."</span>, | |
| choices=[<span class="hljs-string">"Gracias"</span>, <span class="hljs-string">"Merci"</span>, <span class="hljs-string">"Grazie"</span>], | |
| gold_index=<span class="hljs-number">0</span>) | |
| ], | |
| )`,lang:"python",wrap:!1})},$$slots:{default:!0}});var d=e(g,4);o(d,{anchor:"lighteval.tasks.requests.Doc.example-4",children:(s,t)=>{n(s,{code:"ZG9jJTIwJTNEJTIwRG9jKCUwQSUyMCUyMCUyMCUyMHF1ZXJ5JTNEJTIyV2hhdCUyMGlzJTIwc2hvd24lMjBpbiUyMHRoaXMlMjBpbWFnZSUzRiUyMiUyQyUwQSUyMCUyMCUyMCUyMGNob2ljZXMlM0QlNUIlMjJBJTIwY2F0JTIyJTVEJTJDJTBBJTIwJTIwJTIwJTIwZ29sZF9pbmRleCUzRDAlMkMlMEElMjAlMjAlMjAlMjBpbWFnZXMlM0QlNUJwaWxfaW1hZ2UlNUQlMkMlMjAlMjAlMjMlMjBQSUwlMjBJbWFnZSUyMG9iamVjdCUwQSk=",highlighted:`doc = Doc( | |
| query=<span class="hljs-string">"What is shown in this image?"</span>, | |
| choices=[<span class="hljs-string">"A cat"</span>], | |
| gold_index=<span class="hljs-number">0</span>, | |
| images=[pil_image], <span class="hljs-comment"># PIL Image object</span> | |
| )`,lang:"python",wrap:!1})},$$slots:{default:!0}});var u=e(d,2),j=M(u);m(j,{name:"get_golds",anchor:"lighteval.tasks.requests.Doc.get_golds",source:"https://github.com/huggingface/lighteval/blob/vr_1445/src/lighteval/tasks/requests.py#L217",parameters:[]}),y(2),T(u),T(a);var q=e(a,2);k(q,{source:"https://github.com/huggingface/lighteval/blob/main/docs/source/package_reference/doc.mdx"}),y(2),J(I,l),G()}export{z as component}; | |
Xet Storage Details
- Size:
- 13.9 kB
- Xet hash:
- 59601978d1e40d0fc5156bc506e5e0a161bb826415ca696944f5062d44eda420
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.