traj-mc / code /eval /lm_tasks /bbh_llada /logical_deduction_five_objects.yaml

Publish code/eval

76d1db9 verified 2 months ago

1.76 kB

dataset_name: logical_deduction_five_objects
description: 'A logical deduction task which requires deducing the order of a sequence
  of objects.
  '
doc_to_text: 'Q: {{input}}
  A:'
fewshot_config:
  sampler: first_n
  samples:
  - input: 'The following paragraphs each describe a set of three objects arranged
      in a fixed order. The statements are logically consistent within each paragraph.
      In a golf tournament, there were three golfers: Amy, Eli, and Eve. Eve finished
      above Amy. Eli finished below Amy.
      Options:
      (A) Amy finished last
      (B) Eli finished last
      (C) Eve finished last'
    target: (B)
  - input: 'The following paragraphs each describe a set of three objects arranged
      in a fixed order. The statements are logically consistent within each paragraph.
      On a shelf, there are three books: a white book, a green book, and an orange
      book. The green book is to the right of the white book. The orange book is the
      rightmost.
      Options:
      (A) The white book is the leftmost
      (B) The green book is the leftmost
      (C) The orange book is the leftmost'
    target: (A)
  - input: 'The following paragraphs each describe a set of three objects arranged
      in a fixed order. The statements are logically consistent within each paragraph.
      On a shelf, there are three books: a red book, a gray book, and a white book.
      The white book is to the left of the gray book. The red book is the second from
      the left.
      Options:
      (A) The red book is the leftmost
      (B) The gray book is the leftmost
      (C) The white book is the leftmost'
    target: (C)
include: _bbh_llada_template_yaml
task: bbh_llada_logical_deduction_five_objects

	dataset_name: logical_deduction_five_objects
	description: 'A logical deduction task which requires deducing the order of a sequence
	of objects.


	'
	doc_to_text: 'Q: {{input}}

	A:'
	fewshot_config:
	sampler: first_n
	samples:
	- input: 'The following paragraphs each describe a set of three objects arranged
	in a fixed order. The statements are logically consistent within each paragraph.
	In a golf tournament, there were three golfers: Amy, Eli, and Eve. Eve finished
	above Amy. Eli finished below Amy.

	Options:

	(A) Amy finished last

	(B) Eli finished last

	(C) Eve finished last'
	target: (B)
	- input: 'The following paragraphs each describe a set of three objects arranged
	in a fixed order. The statements are logically consistent within each paragraph.
	On a shelf, there are three books: a white book, a green book, and an orange
	book. The green book is to the right of the white book. The orange book is the
	rightmost.

	Options:

	(A) The white book is the leftmost

	(B) The green book is the leftmost

	(C) The orange book is the leftmost'
	target: (A)
	- input: 'The following paragraphs each describe a set of three objects arranged
	in a fixed order. The statements are logically consistent within each paragraph.
	On a shelf, there are three books: a red book, a gray book, and a white book.
	The white book is to the left of the gray book. The red book is the second from
	the left.

	Options:

	(A) The red book is the leftmost

	(B) The gray book is the leftmost

	(C) The white book is the leftmost'
	target: (C)
	include: _bbh_llada_template_yaml
	task: bbh_llada_logical_deduction_five_objects