| |
| """Setup pipeline for product design year classification""" |
|
|
| import argparse |
| from pathlib import Path |
| import subprocess |
| import sys |
| import os |
|
|
|
|
| def run_step(script_name, args=[]): |
| """Run a script step""" |
| |
| script_dir = Path(__file__).parent |
| script_path = script_dir / script_name |
|
|
| cmd = [sys.executable, str(script_path)] + args |
| result = subprocess.run(cmd, capture_output=True, text=True) |
| if result.returncode != 0: |
| print(f"Error in {script_name}:") |
| print(result.stderr) |
| print(result.stdout) |
| return False |
| print(result.stdout) |
| return True |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser(description='Setup data pipeline for product year classification') |
| parser.add_argument('--skip-validation', action='store_true', help='Skip URL validation') |
| parser.add_argument('--xlsx-path', default='../data/metadata/fetch_ALL.xlsx', help='Path to XLSX file') |
| parser.add_argument('--output-dir', default='../data', help='Output directory') |
| args = parser.parse_args() |
|
|
| print("=== Product Design Year Classification Pipeline ===\n") |
|
|
| |
| print("Step 1: Processing product data...") |
| if not run_step('process_xlsx.py'): |
| return |
|
|
| |
| if not args.skip_validation: |
| print("\nStep 2: Validating image URLs...") |
| if not run_step('validate_urls.py'): |
| print("URL validation failed, continuing anyway...") |
|
|
| |
| print("\nStep 3: Creating train/val/test splits...") |
| if not run_step('create_splits.py'): |
| return |
|
|
| |
| print("\nStep 4: Analyzing dataset...") |
| if not run_step('analyze_dataset.py'): |
| print("Analysis failed, but pipeline complete") |
|
|
| print("\n=== Pipeline Complete! ===") |
| print(f"Your data is ready in {args.output_dir}/") |
| print("\nNext steps:") |
| print("1. Review the analysis in data/analysis/") |
| print("2. Check data/metadata/processing_issues.txt for any problems") |
| print("3. Start training with: python scripts/train.py --model_name efficientnet-b2") |
|
|
|
|
| if __name__ == "__main__": |
| main() |
