akash Cursor commited on
Commit
1c8e925
Β·
1 Parent(s): fa81b74

Fix IMDb load by downloading namespaced parquet shards.

Browse files

Avoid the Hub rewrite of stanfordnlp/imdb to the invalid hf://datasets/imdb URI, and stop reporting a missing model as a successful training run.

Co-authored-by: Cursor <cursoragent@cursor.com>

Files changed (4) hide show
  1. README.md +1 -0
  2. app.py +18 -27
  3. requirements.txt +1 -0
  4. train.py +20 -3
README.md CHANGED
@@ -9,6 +9,7 @@ python_version: '3.13'
9
  app_file: app.py
10
  pinned: false
11
  license: apache-2.0
 
12
  ---
13
 
14
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
9
  app_file: app.py
10
  pinned: false
11
  license: apache-2.0
12
+ # Rebuild after IMDb parquet loader fix.
13
  ---
14
 
15
  Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
app.py CHANGED
@@ -85,26 +85,24 @@ def run_training():
85
  if not os.path.exists("train.py"):
86
  return "❌ Training script not found! The file `train.py` is missing from this Space. Please add it and try again."
87
 
88
- # Run the training script with full output capture
 
89
  result = subprocess.run(
90
- [sys.executable, "train.py"],
91
  capture_output=True,
92
  text=True,
93
- timeout=600,
94
- env=os.environ.copy(),
95
  )
96
 
97
- # Combine stdout and stderr for full output
98
- full_output = result.stdout + result.stderr
99
-
100
- # Truncate if too long
101
- if len(full_output) > 3000:
102
- full_output = full_output[:3000] + "\n... (output truncated)"
103
 
104
- if result.returncode == 0:
105
- if os.path.exists("./my_finetuned_model"):
106
- load_status = load_model()
107
- return f"""βœ… Training completed successfully!
108
 
109
  Full Output:
110
  {full_output}
@@ -113,23 +111,16 @@ Model saved to: ./my_finetuned_model
113
  Status: {load_status}
114
 
115
  Try the Predict tab now with your custom model!"""
116
- else:
117
- return f"""⚠️ Training completed but model not found!
118
 
119
- Output:
120
- {full_output}
121
 
122
- The model should be at ./my_finetuned_model but it's missing."""
123
- else:
124
- return f"""❌ Training failed! (Exit code: {result.returncode})
125
-
126
- Full Error Output:
127
  {full_output}
128
 
129
- Troubleshooting tips:
130
- 1. Check if torch is installed: pip list | grep torch
131
- 2. Check if datasets is installed: pip list | grep datasets
132
- 3. Try running training from terminal: python train.py"""
133
 
134
  except subprocess.TimeoutExpired:
135
  return """⏰ Training timed out after 10 minutes!
 
85
  if not os.path.exists("train.py"):
86
  return "❌ Training script not found! The file `train.py` is missing from this Space. Please add it and try again."
87
 
88
+ env = os.environ.copy()
89
+ env["PYTHONUNBUFFERED"] = "1"
90
  result = subprocess.run(
91
+ [sys.executable, "-u", "train.py"],
92
  capture_output=True,
93
  text=True,
94
+ timeout=1200,
95
+ env=env,
96
  )
97
 
98
+ full_output = (result.stdout or "") + (result.stderr or "")
99
+ if len(full_output) > 4000:
100
+ full_output = full_output[:4000] + "\n... (output truncated)"
 
 
 
101
 
102
+ model_exists = os.path.exists("./my_finetuned_model")
103
+ if result.returncode == 0 and model_exists:
104
+ load_status = load_model()
105
+ return f"""βœ… Training completed successfully!
106
 
107
  Full Output:
108
  {full_output}
 
111
  Status: {load_status}
112
 
113
  Try the Predict tab now with your custom model!"""
 
 
114
 
115
+ return f"""❌ Training failed. (Exit code: {result.returncode})
116
+ Model folder present: {model_exists}
117
 
118
+ Output:
 
 
 
 
119
  {full_output}
120
 
121
+ If this output still mentions `load_dataset("imdb")` or does not include
122
+ `Train script version: 2026-09-07-parquet-imdb`, factory-reboot the Space
123
+ so it picks up the latest `train.py`."""
 
124
 
125
  except subprocess.TimeoutExpired:
126
  return """⏰ Training timed out after 10 minutes!
requirements.txt CHANGED
@@ -1,6 +1,7 @@
1
  gradio
2
  transformers
3
  datasets>=3.0.0
 
4
  accelerate
5
  torch
6
  huggingface_hub
 
1
  gradio
2
  transformers
3
  datasets>=3.0.0
4
+ pyarrow
5
  accelerate
6
  torch
7
  huggingface_hub
train.py CHANGED
@@ -9,7 +9,7 @@ import sys
9
  import numpy as np
10
  import torch
11
  from datasets import load_dataset
12
- from huggingface_hub import login
13
  from sklearn.metrics import accuracy_score
14
  from transformers import (
15
  AutoModelForSequenceClassification,
@@ -18,11 +18,29 @@ from transformers import (
18
  TrainingArguments,
19
  )
20
 
 
21
  print(f"βœ… PyTorch version: {torch.__version__}")
22
  print(f"βœ… Transformers version: {__import__('transformers').__version__}")
23
  print(f"βœ… Datasets version: {__import__('datasets').__version__}")
24
 
25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  def compute_metrics(eval_pred):
27
  logits, labels = eval_pred
28
  predictions = np.argmax(logits, axis=-1)
@@ -39,8 +57,7 @@ def setup_training():
39
  # 1. Load Dataset
40
  print("\nπŸ“Š Step 1: Loading dataset...")
41
  try:
42
- # Hub now requires namespace/name. "imdb" is stanfordnlp/imdb.
43
- dataset = load_dataset("stanfordnlp/imdb")
44
  print(f"βœ… Loaded {len(dataset['train'])} training samples")
45
  print(f"βœ… Loaded {len(dataset['test'])} test samples")
46
 
 
9
  import numpy as np
10
  import torch
11
  from datasets import load_dataset
12
+ from huggingface_hub import hf_hub_download, login
13
  from sklearn.metrics import accuracy_score
14
  from transformers import (
15
  AutoModelForSequenceClassification,
 
18
  TrainingArguments,
19
  )
20
 
21
+ print("πŸ› οΈ Train script version: 2026-09-07-parquet-imdb")
22
  print(f"βœ… PyTorch version: {torch.__version__}")
23
  print(f"βœ… Transformers version: {__import__('transformers').__version__}")
24
  print(f"βœ… Datasets version: {__import__('datasets').__version__}")
25
 
26
 
27
+ def load_imdb_dataset():
28
+ """Load IMDb without the legacy Hub id `imdb`.
29
+
30
+ datasets 2.x + recent huggingface_hub rewrites stanfordnlp/imdb to
31
+ hf://datasets/imdb@<sha>/README.md, which is rejected. Download the
32
+ parquet shards by namespaced repo id instead.
33
+ """
34
+ data_files = {}
35
+ for split in ("train", "test"):
36
+ data_files[split] = hf_hub_download(
37
+ repo_id="stanfordnlp/imdb",
38
+ filename=f"plain_text/{split}-00000-of-00001.parquet",
39
+ repo_type="dataset",
40
+ )
41
+ return load_dataset("parquet", data_files=data_files)
42
+
43
+
44
  def compute_metrics(eval_pred):
45
  logits, labels = eval_pred
46
  predictions = np.argmax(logits, axis=-1)
 
57
  # 1. Load Dataset
58
  print("\nπŸ“Š Step 1: Loading dataset...")
59
  try:
60
+ dataset = load_imdb_dataset()
 
61
  print(f"βœ… Loaded {len(dataset['train'])} training samples")
62
  print(f"βœ… Loaded {len(dataset['test'])} test samples")
63