File size: 2,687 Bytes
345bccf
 
 
 
 
 
 
 
371cfa1
345bccf
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d188498
345bccf
 
 
 
 
 
 
 
 
 
 
 
 
371cfa1
 
345bccf
 
 
 
 
 
 
 
 
 
8999d5b
 
371cfa1
 
 
8ddfad6
371cfa1
8ddfad6
 
8999d5b
371cfa1
 
8999d5b
371cfa1
 
 
 
345bccf
8999d5b
8ddfad6
345bccf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
import argparse
import json
import time
import pickle
import os
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from tasks import TaskDataGenerator
from environment import DocumentClassificationEnv

MODEL_DIR = os.path.dirname(os.path.abspath(__file__))

def get_model_path(difficulty):
    return os.path.join(MODEL_DIR, f"model_{difficulty}.pkl")

def train_model(difficulty):
    print(f"Training {difficulty} model...")
    all_texts, all_labels = [], []
    for seed in range(10):
        gen = TaskDataGenerator(difficulty, seed=seed)
        docs, labels = gen.generate_task_data()
        for doc, label in zip(docs, labels):
            all_texts.append(doc["content"])
            all_labels.append(label)
    pipeline = Pipeline([
        ("tfidf", TfidfVectorizer(ngram_range=(1,2), max_features=5000, sublinear_tf=True)),
        ("clf", LogisticRegression(max_iter=1000, C=5.0, solver="lbfgs"))
    ])
    pipeline.fit(all_texts, all_labels)
    with open(get_model_path(difficulty), "wb") as f:
        pickle.dump(pipeline, f)
    print(f"  Saved!")
    return pipeline

def load_or_train(difficulty):
    path = get_model_path(difficulty)
    if os.path.exists(path):
        with open(path, "rb") as f:
            return pickle.load(f)
    return train_model(difficulty)

def run_task(difficulty):
    model = load_or_train(difficulty)
    env = DocumentClassificationEnv(task_difficulty=difficulty, seed=42)
    obs, _ = env.reset()
    correct, total, t0, terminated = 0, 0, time.time(), False
    while not terminated:
        action = int(model.predict([obs["content"]])[0])
        obs, reward, terminated, _, info = env.step(action)
        correct += int(info.get("is_correct", False))
        total += 1
    return correct/total if total > 0 else 0, time.time()-t0

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--task", default="all")
    parser.add_argument("--output", default="baseline_results.json")
    args = parser.parse_args()
    tasks = ["easy","medium","hard"] if args.task=="all" else [args.task]
    results = {}
    for task in tasks:
        print(f"\nRunning {task.upper()} task...")
        score, elapsed = run_task(task)
        results[task] = {"score": round(score,4), "time": round(elapsed,2)}
        print(f"  {task.upper()} Score: {score:.4f} ({elapsed:.1f}s)")
    print("\n=== BASELINE SCORES ===")
    for t,r in results.items():
        print(f"  {t.upper()} Score: {r['score']:.4f}")
    with open(args.output,"w") as f:
        json.dump(results,f,indent=2)

if __name__ == "__main__":
    main()