ComputerAssignments/CA6_systematic_generalization/visualizations/analysis/comprehensive_analysis.json

{
  "timestamp": "2025-10-10T08:40:46.713170",
  "experiment": {
    "name": "Systematic Generalization Analysis",
    "version": "1.0.0",
    "description": "Comprehensive analysis of systematic generalization approaches"
  },
  "models": {
    "neural": {
      "architectures": [
        "MLP",
        "Modular",
        "Attention",
        "Graph",
        "Hierarchical",
        "Meta-Learning"
      ],
      "best_performer": "Hierarchical",
      "avg_parameters": 175000,
      "random_split_accuracy": 0.91,
      "systematic_split_accuracy": 0.75
    },
    "symbolic": {
      "systems": [
        "Production Rules",
        "Program Synthesis",
        "Grammar-based",
        "Logic Programming"
      ],
      "perfect_systematic_generalization": true,
      "requires_manual_rules": true,
      "random_split_accuracy": 0.95,
      "systematic_split_accuracy": 0.95
    },
    "neurosymbolic": {
      "approaches": [
        "NSR",
        "Neural Module Network",
        "Hybrid Reasoning"
      ],
      "best_performer": "Neural-Symbolic Recursive Machine",
      "combines_strengths": true,
      "random_split_accuracy": 0.91,
      "systematic_split_accuracy": 0.83
    }
  },
  "datasets": {
    "scan": {
      "size": 19076,
      "vocab_size": 85,
      "avg_length": 8.5
    },
    "arithmetic": {
      "size": 36190,
      "vocab_size": 156,
      "avg_length": 12.3
    },
    "visual_reasoning": {
      "size": 15603,
      "vocab_size": 234,
      "avg_length": 15.7
    },
    "logic": {
      "size": 22801,
      "vocab_size": 178,
      "avg_length": 10.2
    },
    "spatial": {
      "size": 18209,
      "vocab_size": 145,
      "avg_length": 11.8
    }
  },
  "key_findings": {
    "systematic_generalization_gap": {
      "mlp": 0.46,
      "modular": 0.23,
      "attention": 0.18,
      "nsr": 0.08
    },
    "complexity_generalization": {
      "mlp_degradation": 0.7,
      "nsr_degradation": 0.24
    },
    "length_generalization": {
      "mlp_at_2x": 0.38,
      "nsr_at_2x": 0.73
    }
  },
  "recommendations": [
    "Use NeuroSymbolic approaches for systematic generalization tasks",
    "Implement modular architectures when possible",
    "Test on systematic splits, not just random splits",
    "Monitor generalization gap during training",
    "Consider curriculum learning for complex compositions"
  ]
}