| import json |
| import matplotlib.pyplot as plt |
| import seaborn as sns |
| from pathlib import Path |
| import pandas as pd |
|
|
|
|
| def analyze_dataset(splits_dir): |
| """Create visualizations and analysis of the dataset""" |
|
|
| |
| splits = {} |
| for split_name in ['train', 'val', 'test']: |
| with open(Path(splits_dir) / f'{split_name}.json', 'r', encoding='utf-8') as f: |
| splits[split_name] = json.load(f) |
|
|
| |
| analysis_dir = Path(splits_dir).parent / 'analysis' |
| analysis_dir.mkdir(exist_ok=True) |
|
|
| |
| plt.figure(figsize=(12, 6)) |
|
|
| decade_data = [] |
| for split_name, split_data in splits.items(): |
| for entry in split_data: |
| decade_data.append({ |
| 'split': split_name, |
| 'decade': entry['decade'], |
| 'classification': entry['classification'] |
| }) |
|
|
| df = pd.DataFrame(decade_data) |
|
|
| |
| plt.subplot(1, 2, 1) |
| decade_counts = df.groupby(['split', 'decade']).size().unstack(fill_value=0) |
| decade_counts.plot(kind='bar', ax=plt.gca()) |
| plt.title('Images per Decade by Split') |
| plt.xlabel('Split') |
| plt.ylabel('Number of Images') |
| plt.xticks(rotation=0) |
| plt.legend(title='Decade', bbox_to_anchor=(1.05, 1), loc='upper left') |
|
|
| |
| plt.subplot(1, 2, 2) |
| top_classifications = df['classification'].value_counts().head(10).index |
| df_top = df[df['classification'].isin(top_classifications)] |
| class_counts = df_top.groupby(['split', 'classification']).size().unstack(fill_value=0) |
| class_counts.plot(kind='bar', ax=plt.gca()) |
| plt.title('Top 10 Product Classifications by Split') |
| plt.xlabel('Split') |
| plt.ylabel('Number of Images') |
| plt.xticks(rotation=0) |
| plt.legend(title='Classification', bbox_to_anchor=(1.05, 1), loc='upper left') |
|
|
| plt.tight_layout() |
| plt.savefig(analysis_dir / 'dataset_distribution.png', dpi=300, bbox_inches='tight') |
| plt.close() |
|
|
| |
| plt.figure(figsize=(10, 6)) |
|
|
| all_data = [] |
| for split_data in splits.values(): |
| all_data.extend(split_data) |
|
|
| |
| products_by_decade = {} |
| for entry in all_data: |
| decade = entry['decade'] |
| product_id = entry['product_id'] |
| if decade not in products_by_decade: |
| products_by_decade[decade] = set() |
| products_by_decade[decade].add(product_id) |
|
|
| decades = sorted(products_by_decade.keys()) |
| product_counts = [len(products_by_decade[d]) for d in decades] |
|
|
| plt.bar(decades, product_counts) |
| plt.title('Number of Unique Products by Decade') |
| plt.xlabel('Decade') |
| plt.ylabel('Number of Products') |
| plt.xticks(rotation=45) |
|
|
| |
| for i, (decade, count) in enumerate(zip(decades, product_counts)): |
| plt.text(i, count + 1, str(count), ha='center', va='bottom') |
|
|
| plt.tight_layout() |
| plt.savefig(analysis_dir / 'products_per_decade.png', dpi=300) |
| plt.close() |
|
|
| |
| report = [] |
| report.append("=== Dataset Analysis Report ===\n") |
|
|
| |
| total_images = sum(len(split_data) for split_data in splits.values()) |
| unique_products = len(set(entry['product_id'] for split_data in splits.values() for entry in split_data)) |
|
|
| report.append(f"Total Images: {total_images}") |
| report.append(f"Unique Products: {unique_products}") |
| report.append(f"Average Images per Product: {total_images / unique_products:.2f}\n") |
|
|
| |
| report.append("Decade Coverage:") |
| for decade in sorted(products_by_decade.keys()): |
| report.append(f" {decade}: {len(products_by_decade[decade])} products") |
|
|
| |
| all_classifications = [entry['classification'] for split_data in splits.values() for entry in split_data] |
| unique_classifications = len(set(all_classifications)) |
| report.append(f"\nUnique Classifications: {unique_classifications}") |
|
|
| |
| all_countries = [entry['country'] for split_data in splits.values() for entry in split_data] |
| unique_countries = len(set(all_countries)) |
| report.append(f"Unique Countries: {unique_countries}") |
|
|
| |
| with open(analysis_dir / 'dataset_report.txt', 'w', encoding='utf-8') as f: |
| f.write('\n'.join(report)) |
|
|
| print('\n'.join(report)) |
| print(f"\nAnalysis saved to {analysis_dir}/") |
|
|
|
|
| if __name__ == "__main__": |
|
|
| project_root = Path(__file__).parent.parent |
| |
| analyze_dataset(str(project_root / 'data/splits')) |
