renpas22 commited on
Commit ·
487225b
1
Parent(s): 83e0535
Auto-download repository in HF Jobs environment
Browse files- train_qwen_cot_dual.py +19 -5
train_qwen_cot_dual.py
CHANGED
|
@@ -23,15 +23,29 @@ import torch
|
|
| 23 |
from typing import Dict, List, Optional, Any
|
| 24 |
import logging
|
| 25 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
# Add repository root and src to Python path
|
| 27 |
-
repo_root = Path(__file__).parent.absolute()
|
| 28 |
sys.path.insert(0, str(repo_root))
|
| 29 |
sys.path.insert(0, str(repo_root / "src"))
|
| 30 |
|
| 31 |
-
from reasoning.step_level_cot import StepLevelCoTTrainer
|
| 32 |
-
from reasoning.flux_reason_dataset import FLUXReasonDataset, create_flux_reason_dataloader
|
| 33 |
-
from reasoning.finevision_dataset import FineVisionDataset, create_finevision_dataloader
|
| 34 |
-
from reasoning.step_data import ReasoningChain
|
| 35 |
from utils.add_special_tokens import add_special_tokens_to_model
|
| 36 |
|
| 37 |
# Set up logging
|
|
|
|
| 23 |
from typing import Dict, List, Optional, Any
|
| 24 |
import logging
|
| 25 |
|
| 26 |
+
# Clone repository if running in HF Jobs environment
|
| 27 |
+
if str(Path(__file__).parent) == "/tmp":
|
| 28 |
+
from huggingface_hub import snapshot_download
|
| 29 |
+
print("Running in HF Jobs - downloading repository...")
|
| 30 |
+
repo_path = snapshot_download(
|
| 31 |
+
repo_id="Mulebot/dei-model",
|
| 32 |
+
repo_type="model",
|
| 33 |
+
local_dir="/tmp/dei-model",
|
| 34 |
+
token=os.environ.get("HF_TOKEN")
|
| 35 |
+
)
|
| 36 |
+
repo_root = Path(repo_path)
|
| 37 |
+
print(f"Repository downloaded to: {repo_root}")
|
| 38 |
+
else:
|
| 39 |
+
repo_root = Path(__file__).parent.absolute()
|
| 40 |
+
|
| 41 |
# Add repository root and src to Python path
|
|
|
|
| 42 |
sys.path.insert(0, str(repo_root))
|
| 43 |
sys.path.insert(0, str(repo_root / "src"))
|
| 44 |
|
| 45 |
+
from src.reasoning.step_level_cot import StepLevelCoTTrainer
|
| 46 |
+
from src.reasoning.flux_reason_dataset import FLUXReasonDataset, create_flux_reason_dataloader
|
| 47 |
+
from src.reasoning.finevision_dataset import FineVisionDataset, create_finevision_dataloader
|
| 48 |
+
from src.reasoning.step_data import ReasoningChain
|
| 49 |
from utils.add_special_tokens import add_special_tokens_to_model
|
| 50 |
|
| 51 |
# Set up logging
|