document-classification-env / baseline_inference.py
TanujInsane's picture
Update baseline_inference.py
d188498 verified
Raw History Blame Contribute Delete
2.69 kB
import argparse
import json
import time
import pickle
import os
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from tasks import TaskDataGenerator
from environment import DocumentClassificationEnv
MODEL_DIR = os.path.dirname(os.path.abspath(__file__))
def get_model_path(difficulty):
return os.path.join(MODEL_DIR, f"model_{difficulty}.pkl")
def train_model(difficulty):
print(f"Training {difficulty} model...")
all_texts, all_labels = [], []
for seed in range(10):
gen = TaskDataGenerator(difficulty, seed=seed)
docs, labels = gen.generate_task_data()
for doc, label in zip(docs, labels):
all_texts.append(doc["content"])
all_labels.append(label)
pipeline = Pipeline([
("tfidf", TfidfVectorizer(ngram_range=(1,2), max_features=5000, sublinear_tf=True)),
("clf", LogisticRegression(max_iter=1000, C=5.0, solver="lbfgs"))
])
pipeline.fit(all_texts, all_labels)
with open(get_model_path(difficulty), "wb") as f:
pickle.dump(pipeline, f)
print(f" Saved!")
return pipeline
def load_or_train(difficulty):
path = get_model_path(difficulty)
if os.path.exists(path):
with open(path, "rb") as f:
return pickle.load(f)
return train_model(difficulty)
def run_task(difficulty):
model = load_or_train(difficulty)
env = DocumentClassificationEnv(task_difficulty=difficulty, seed=42)
obs, _ = env.reset()
correct, total, t0, terminated = 0, 0, time.time(), False
while not terminated:
action = int(model.predict([obs["content"]])[0])
obs, reward, terminated, _, info = env.step(action)
correct += int(info.get("is_correct", False))
total += 1
return correct/total if total > 0 else 0, time.time()-t0
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--task", default="all")
parser.add_argument("--output", default="baseline_results.json")
args = parser.parse_args()
tasks = ["easy","medium","hard"] if args.task=="all" else [args.task]
results = {}
for task in tasks:
print(f"\nRunning {task.upper()} task...")
score, elapsed = run_task(task)
results[task] = {"score": round(score,4), "time": round(elapsed,2)}
print(f" {task.upper()} Score: {score:.4f} ({elapsed:.1f}s)")
print("\n=== BASELINE SCORES ===")
for t,r in results.items():
print(f" {t.upper()} Score: {r['score']:.4f}")
with open(args.output,"w") as f:
json.dump(results,f,indent=2)
if __name__ == "__main__":
main()