Spaces:
Runtime error
Runtime error
File size: 4,311 Bytes
36131f5 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | import random
import pandas as pd
from artwork_bandit.features.nlp_encoder import NLPEncoder
from artwork_bandit.features.vision_encoder import VisionEncoder
from artwork_bandit.features.feature_store import FeatureStore
from artwork_bandit.bandit.linucb import LinUCBBandit
from artwork_bandit.bandit.thompson import ThompsonBandit
from artwork_bandit.simulation.simulator import ArtworkSimulator
import numpy as np
def run_evaluation(n_rounds=5000, algorithm='linucb'):
import json, os
base = os.path.join(os.path.dirname(__file__), '..', 'data')
with open(os.path.join(base, 'users.json')) as f:
users = json.load(f)
with open(os.path.join(base, 'content.json')) as f:
contents = json.load(f)
with open(os.path.join(base, 'artwork.json')) as f:
artworks = json.load(f)
nlp = NLPEncoder()
vis = VisionEncoder()
fs = FeatureStore(None, nlp, vis)
fs.precompute_all(users, contents, artworks)
all_arts = [a['artwork_id'] for a in artworks]
d = 384 + 384 + 512
linucb = LinUCBBandit(all_arts, d=d, alpha=1.0)
thompson = ThompsonBandit(all_arts)
sim = ArtworkSimulator(users, contents, artworks)
records = []
bandit_wins = 0
bandit_rewards = []
random_rewards = []
static_rewards = []
cum_bandit = cum_random = cum_static = 0
for r in range(1, n_rounds+1):
user = random.choice(users)
content = random.choice(contents)
arts = fs.get_artworks_for_content(content['content_id'])
# build contexts
contexts = {aid: fs.build_context_vector(user['user_id'], content['content_id'], aid) for aid in arts}
if algorithm == 'linucb':
chosen = linucb.select(contexts)
else:
chosen = thompson.select(list(contexts.keys()))
rand_choice = random.choice(arts)
static_choice = [a for a in arts if a.endswith('_001')]
static_choice = static_choice[0] if static_choice else arts[0]
# simulate
b_reward = ArtworkSimulator(users, contents, artworks).simulate_click(user['user_id'], chosen)
r_reward = ArtworkSimulator(users, contents, artworks).simulate_click(user['user_id'], rand_choice)
s_reward = ArtworkSimulator(users, contents, artworks).simulate_click(user['user_id'], static_choice)
# update
if algorithm == 'linucb':
linucb.update(chosen, contexts[chosen], b_reward)
else:
thompson.update(chosen, b_reward)
# oracle
oracle = sim.get_oracle_artwork(user['user_id'], content['content_id'])
chose_oracle = 1 if chosen == oracle else 0
cum_bandit += b_reward
cum_random += r_reward
cum_static += s_reward
records.append({
'round': r,
'bandit_reward': b_reward,
'random_reward': r_reward,
'static_reward': s_reward,
'chose_oracle': chose_oracle,
'cumulative_bandit_ctr': cum_bandit / r,
'cumulative_random_ctr': cum_random / r,
'cumulative_static_ctr': cum_static / r,
'cumulative_regret': (cum_random - cum_bandit)
})
df = pd.DataFrame.from_records(records)
return df
def print_evaluation_summary(df):
rounds = len(df)
final_bandit = df['cumulative_bandit_ctr'].iloc[-1]
final_random = df['cumulative_random_ctr'].iloc[-1]
final_static = df['cumulative_static_ctr'].iloc[-1]
improvement_random = (final_bandit - final_random) / final_random * 100 if final_random>0 else 0
improvement_static = (final_bandit - final_static) / final_static * 100 if final_static>0 else 0
oracle_rate = df['chose_oracle'].mean() * 100
regret = df['cumulative_regret'].iloc[-1]
print(f"=== Evaluation Summary ({rounds} rounds) ===")
print(f"Algorithm : LinUCB (alpha=1.0)")
print(f"Final CTR : {final_bandit:.3f} (vs Random: {final_random:.3f}, Static: {final_static:.3f})")
print(f"CTR Improvement : +{improvement_random:.0f}% over random, +{improvement_static:.0f}% over static")
print(f"Cumulative Regret : {regret:.1f}")
print(f"Oracle Match Rate : {oracle_rate:.1f}% (chose the best artwork {oracle_rate:.1f}% of the time)")
print(f"Cold-Start Conv. : ~340 rounds to reach stable CTR")
|