File size: 7,498 Bytes
caea08d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 | """System prompt + per-turn user-prompt builder for the JewelryShop env.
Logic mirrors `inference.py`'s `build_user_prompt` so that whatever the model
saw during inference evaluation it also sees during training. Kept as a plain
sync function (no asyncio) so it composes cleanly with TRL's rollout_func.
"""
from __future__ import annotations
import math
import textwrap
from typing import List
SYSTEM_PROMPT = textwrap.dedent(
"""
You are an expert agent running a jewelry shop. The episode runs in 3 phases
and may loop back to MARKET if the warehouse runs out of gold. The episode
reward is the SUM of per-step partial rewards across the whole episode and
is bounded in [0, 1]. Each task weights the phases differently:
- market_timing -> phase 1 = 0.6, phase 2 = 0.2, phase 3 = 0.2
- demand_crafter -> phase 1 = 0.2, phase 2 = 0.6, phase 3 = 0.2
- profit_negotiator -> phase 1 = 0.2, phase 2 = 0.2, phase 3 = 0.6
## Phase 1: MARKET (buy / wait)
Two modes:
- synthetic mode: gold price moves randomly each WAIT step within a round cap.
- real mode: gold price comes from a live source (yfinance: GC=F),
no round cap; WAIT just refreshes the live quote.
Coordination from the warehouse:
- inventory_urgent=True / cannot_wait=True means you MUST buy now;
WAIT will be blocked. Submit "buy X.XX" with an affordable troy-oz qty.
Behavior:
- If you can wait, observe the price trend in gold_price_history before buying.
- Reserve cash for labor (ring=$200, necklace=$300, bracelet=$100).
- Respond: "buy X.XX" (troy oz of gold) or "wait".
## Phase 2: WAREHOUSE (choose product)
You see two demand fields:
- demand : the TRUE per-product demand for THIS episode (ground truth).
- demand_forecast : a NOISY signal you can also lean on for planning.
Products: ring (1oz + $200), necklace (2oz + $300), bracelet (0.5oz + $100).
If you don't have enough gold to craft your choice, the env may BOUNCE you back
to MARKET to buy more (up to max_market_reentries times). After max bounces or
when truly broke, the customer leaves and the episode ends.
Respond: "ring", "necklace", or "bracelet".
## Phase 3: SHOWROOM (negotiate)
The customer makes an offer; if you counter, they raise it ~5% per round,
up to 5 rounds. After 5 rounds with no acceptance, the customer leaves
(no phase-3 reward). Reject also gives 0 phase-3 reward.
Respond: "I accept" or a counter like "How about $X?". NEVER explicitly reject.
CRITICAL: Respond with ONLY the action value. No explanations.
"""
).strip()
def build_user_prompt(step: int, obs, last_reward: float, history: List[str]) -> str:
"""Format a single observation into a user prompt the LLM sees this turn.
Mirrors inference.py:build_user_prompt so the model sees the same input shape
during training and at evaluation time.
"""
history_block = "\n".join(history[-4:]) if history else "None"
if obs.phase == "market":
prices = getattr(obs, "gold_price_history", []) or []
trend = ""
if len(prices) >= 2:
if prices[-1] < prices[-2]:
trend = "FALLING (might keep dropping, consider waiting)"
else:
trend = "RISING (buy now before it gets more expensive)"
if getattr(obs, "cannot_wait", False):
trend = (
"URGENT: inventory needs gold now — you cannot wait; buy at the current "
"live quote with an affordable gold_qty (troy oz)."
)
max_rounds = getattr(obs, "max_market_rounds", None)
rounds_left = (max_rounds - getattr(obs, "market_round", 0)) if max_rounds else None
reserve = 300.0
gold_price = getattr(obs, "gold_price", 0) or 0
cash = getattr(obs, "cash", 0) or 0
if gold_price > 0:
raw_qty = (cash - reserve) / gold_price
suggested_qty = max(math.floor(raw_qty * 100) / 100, 0.01)
else:
suggested_qty = 1.0
rl = "unlimited" if rounds_left is None else str(rounds_left)
phase_hint = (
f"Price: ${gold_price}/oz ({getattr(obs, 'gold_price_source', '') or 'n/a'}). "
f"Price history: {prices}. Trend: {trend}. "
f"Rounds / waits so far: {getattr(obs, 'market_round', 0)}; cap: {rl}. "
f"Gold on hand: {getattr(obs, 'gold_oz', 0)} troy oz "
f"(~{getattr(obs, 'gold_grams', 0):.2f} g). "
f"If buying, suggested qty: {suggested_qty} oz (reserves $300 for labor). "
f"Respond: 'buy {suggested_qty}' or 'wait'"
)
elif obs.phase == "warehouse":
demand = getattr(obs, "demand", {}) or {}
forecast = getattr(obs, "demand_forecast", {}) or {}
best_product = max(demand, key=demand.get) if demand else "ring"
phase_hint = (
f"Demand (episode): ring={demand.get('ring', 0):.0%}, "
f"necklace={demand.get('necklace', 0):.0%}, "
f"bracelet={demand.get('bracelet', 0):.0%}. "
f"Forecast (noisy): ring={forecast.get('ring', 0):.0%}, "
f"necklace={forecast.get('necklace', 0):.0%}, "
f"bracelet={forecast.get('bracelet', 0):.0%}. "
f"Highest demand: {best_product}. "
f"You have {getattr(obs, 'gold_oz', 0)}oz gold and "
f"${getattr(obs, 'cash', 0)} cash. "
f"Respond with EXACTLY: {best_product}"
)
elif obs.phase == "showroom":
cost_basis = getattr(obs, "cost_basis", 0) or 0
current_offer = getattr(obs, "current_offer", 0) or 0
negotiation_round = getattr(obs, "negotiation_round", 0) or 0
margin = ""
if current_offer and cost_basis > 0:
margin_pct = ((current_offer - cost_basis) / cost_basis) * 100
margin = f"Margin: {margin_pct:+.1f}%. "
should_accept = negotiation_round >= 4 or (
current_offer and cost_basis > 0 and current_offer > cost_basis * 1.3
)
if should_accept:
phase_hint = (
f"Cost: ${cost_basis}. Offer: ${current_offer}. {margin}"
f"Round {negotiation_round}/5. "
f"Respond with EXACTLY: I accept"
)
else:
counter_msgs = [
"I need a better price for this quality piece",
"That's too low, this craftsmanship deserves more",
f"How about ${round(cost_basis * 1.4, 2)}?",
f"I can't go below ${round(cost_basis * 1.3, 2)}",
]
msg = counter_msgs[min(negotiation_round, len(counter_msgs) - 1)]
phase_hint = (
f"Cost: ${cost_basis}. Offer: ${current_offer}. {margin}"
f"Round {negotiation_round}/5. "
f"DO NOT ACCEPT. Counter-offer. "
f"Respond with EXACTLY: {msg}"
)
else:
phase_hint = ""
return textwrap.dedent(
f"""
Step: {step} | Phase: {obs.phase} | Last reward: {last_reward:.2f}
Cash: ${getattr(obs, 'cash', 0)} | Gold: {getattr(obs, 'gold_oz', 0)}oz | Rings: {getattr(obs, 'inventory', {})}
Gold Price: ${getattr(obs, 'gold_price', 0)}/oz
Env Message: {getattr(obs, 'message', '')}
{phase_hint}
History: {history_block}
"""
).strip()
|