renpas22 commited on
Commit
487225b
·
1 Parent(s): 83e0535

Auto-download repository in HF Jobs environment

Browse files
Files changed (1) hide show
  1. train_qwen_cot_dual.py +19 -5
train_qwen_cot_dual.py CHANGED
@@ -23,15 +23,29 @@ import torch
23
  from typing import Dict, List, Optional, Any
24
  import logging
25
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
26
  # Add repository root and src to Python path
27
- repo_root = Path(__file__).parent.absolute()
28
  sys.path.insert(0, str(repo_root))
29
  sys.path.insert(0, str(repo_root / "src"))
30
 
31
- from reasoning.step_level_cot import StepLevelCoTTrainer
32
- from reasoning.flux_reason_dataset import FLUXReasonDataset, create_flux_reason_dataloader
33
- from reasoning.finevision_dataset import FineVisionDataset, create_finevision_dataloader
34
- from reasoning.step_data import ReasoningChain
35
  from utils.add_special_tokens import add_special_tokens_to_model
36
 
37
  # Set up logging
 
23
  from typing import Dict, List, Optional, Any
24
  import logging
25
 
26
+ # Clone repository if running in HF Jobs environment
27
+ if str(Path(__file__).parent) == "/tmp":
28
+ from huggingface_hub import snapshot_download
29
+ print("Running in HF Jobs - downloading repository...")
30
+ repo_path = snapshot_download(
31
+ repo_id="Mulebot/dei-model",
32
+ repo_type="model",
33
+ local_dir="/tmp/dei-model",
34
+ token=os.environ.get("HF_TOKEN")
35
+ )
36
+ repo_root = Path(repo_path)
37
+ print(f"Repository downloaded to: {repo_root}")
38
+ else:
39
+ repo_root = Path(__file__).parent.absolute()
40
+
41
  # Add repository root and src to Python path
 
42
  sys.path.insert(0, str(repo_root))
43
  sys.path.insert(0, str(repo_root / "src"))
44
 
45
+ from src.reasoning.step_level_cot import StepLevelCoTTrainer
46
+ from src.reasoning.flux_reason_dataset import FLUXReasonDataset, create_flux_reason_dataloader
47
+ from src.reasoning.finevision_dataset import FineVisionDataset, create_finevision_dataloader
48
+ from src.reasoning.step_data import ReasoningChain
49
  from utils.add_special_tokens import add_special_tokens_to_model
50
 
51
  # Set up logging