File size: 3,161 Bytes
345bccf 371cfa1 345bccf 49234f8 345bccf 49234f8 345bccf cdd19a1 345bccf 49234f8 345bccf 371cfa1 345bccf 8999d5b 371cfa1 8ddfad6 371cfa1 8ddfad6 8999d5b 371cfa1 8999d5b 371cfa1 345bccf 8999d5b 8ddfad6 345bccf | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 | import argparse
import json
import time
import pickle
import os
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from tasks import TaskDataGenerator
from environment import DocumentClassificationEnv
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
CACHE_DIR = "/tmp" # Writable on HuggingFace Spaces; fallback for training cache
def get_model_path(difficulty):
"""Return bundled model path if it exists, else cache path in /tmp."""
bundled = os.path.join(MODEL_DIR, f"model_{difficulty}.pkl")
if os.path.exists(bundled):
return bundled
return os.path.join(CACHE_DIR, f"model_{difficulty}.pkl")
def train_model(difficulty):
print(f"Training {difficulty} model...")
all_texts, all_labels = [], []
for seed in range(10):
gen = TaskDataGenerator(difficulty, seed=seed)
docs, labels = gen.generate_task_data()
for doc, label in zip(docs, labels):
all_texts.append(doc["content"])
all_labels.append(label)
pipeline = Pipeline([
("tfidf", TfidfVectorizer(ngram_range=(1,2), max_features=5000, sublinear_tf=True)),
("clf", LogisticRegression(max_iter=1000, C=5.0, solver="lbfgs"))
])
pipeline.fit(all_texts, all_labels)
save_path = os.path.join(CACHE_DIR, f"model_{difficulty}.pkl")
try:
with open(save_path, "wb") as f:
pickle.dump(pipeline, f)
print(f" Saved to {save_path}!")
except Exception as e:
print(f" Warning: could not save model ({e}). Will re-train on next run.")
return pipeline
def load_or_train(difficulty):
path = get_model_path(difficulty)
if os.path.exists(path):
with open(path, "rb") as f:
return pickle.load(f)
return train_model(difficulty)
def run_task(difficulty):
model = load_or_train(difficulty)
env = DocumentClassificationEnv(task_difficulty=difficulty, seed=42)
obs, _ = env.reset()
correct, total, t0, terminated = 0, 0, time.time(), False
while not terminated:
action = int(model.predict([obs["content"]])[0])
obs, reward, terminated, _, info = env.step(action)
correct += int(info.get("is_correct", False))
total += 1
return correct/total if total > 0 else 0, time.time()-t0
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--task", default="all")
parser.add_argument("--output", default="baseline_results.json")
args = parser.parse_args()
tasks = ["easy","medium","hard"] if args.task=="all" else [args.task]
results = {}
for task in tasks:
print(f"\nRunning {task.upper()} task...")
score, elapsed = run_task(task)
results[task] = {"score": round(score,4), "time": round(elapsed,2)}
print(f" {task.upper()} Score: {score:.4f} ({elapsed:.1f}s)")
print("\n=== BASELINE SCORES ===")
for t,r in results.items():
print(f" {t.upper()} Score: {r['score']:.4f}")
with open(args.output,"w") as f:
json.dump(results,f,indent=2)
if __name__ == "__main__":
main() |