Bundle OpenModelLab package in Space

openmodellab/benchmark.py

@@ -0,0 +1,35 @@
+from datetime import datetime, UTC
+
+from openmodellab.genome.inspectors.hardware import inspect_hardware
+from openmodellab.genome.inspectors.latency import inspect_latency
+from openmodellab.genome.inspectors.memory import inspect_memory
+from openmodellab.genome.inspectors.throughput import inspect_throughput
+from openmodellab.genome.inspectors.batch import inspect_batch_scaling
+
+
+def analyze_benchmark(model, tokenizer):
+    return {
+        "report": {
+            "tool": "OpenModelLab",
+            "tool_version": "0.1.0-alpha",
+            "schema_version": "1.0",
+            "generated_at": datetime.now(UTC).isoformat(),
+        },
+        "hardware": inspect_hardware(),
+        "latency": inspect_latency(
+            model,
+            tokenizer
+        ),
+        "memory": inspect_memory(
+            model,
+            tokenizer
+        ),
+        "throughput": inspect_throughput(
+            model,
+            tokenizer
+        ),
+        "batch_scaling": inspect_batch_scaling(
+            model,
+            tokenizer
+        )
+    }

openmodellab/cli/main.py

@@ -0,0 +1,189 @@
+import argparse
+
+from openmodellab.comparison.compare import generate_comparison
+from openmodellab.comparison.table import print_table
+
+from openmodellab.genome.model_loader import load_model
+from openmodellab.genome.analyzer import analyze_model
+
+from openmodellab.benchmark import analyze_benchmark
+
+from openmodellab.reporting.json_writer import save_json
+from openmodellab.reporting.benchmark_writer import save_benchmark
+from openmodellab.reporting.csv_writer import save_csv
+from openmodellab.reporting.html_writer import save_html
+
+from openmodellab.visualization.charts import (
+    save_latency_chart,
+    save_throughput_chart
+)
+
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        prog="openmodellab",
+        description="OpenModelLab - Model Genome Generator"
+    )
+
+    subparsers = parser.add_subparsers(
+        dest="command"
+    )
+
+
+    # Genome command
+    genome = subparsers.add_parser(
+        "genome",
+        help="Generate a model genome report"
+    )
+
+    genome.add_argument(
+        "--model",
+        required=True,
+        help="Hugging Face model name"
+    )
+
+
+    # Compare command
+    compare = subparsers.add_parser(
+        "compare",
+        help="Compare model reports"
+    )
+
+    compare.add_argument(
+        "--reports",
+        required=True,
+        help="Directory containing genome and benchmark reports"
+    )
+
+    compare.add_argument(
+        "--format",
+        choices=[
+            "table",
+            "csv",
+            "html",
+            "charts"
+        ],
+        default="table",
+        help="Output format"
+    )
+
+
+    args = parser.parse_args()
+
+
+    if args.command == "genome":
+
+        print("=" * 60)
+        print("OpenModelLab Genome")
+        print("=" * 60)
+
+        model, tokenizer = load_model(
+            args.model
+        )
+
+
+        genome_report = analyze_model(
+            args.model,
+            model,
+            tokenizer
+        )
+
+        genome_file = save_json(
+            genome_report,
+            args.model
+        )
+
+
+        benchmark_report = analyze_benchmark(
+            model,
+            tokenizer
+        )
+
+        benchmark_file = save_benchmark(
+            benchmark_report,
+            args.model
+        )
+
+
+        print()
+        print("Genome report saved:")
+        print(genome_file)
+
+        print()
+        print("Benchmark report saved:")
+        print(benchmark_file)
+
+
+
+    elif args.command == "compare":
+
+        print("=" * 60)
+        print("OpenModelLab Model Comparison")
+        print("=" * 60)
+
+
+        results = generate_comparison(
+            args.reports
+        )
+
+
+        if args.format == "table":
+
+            print_table(
+                results
+            )
+
+
+        elif args.format == "csv":
+
+            outfile = save_csv(
+                results,
+                "comparison_report.csv"
+            )
+
+            print()
+            print("CSV report saved:")
+            print(outfile)
+
+
+
+        elif args.format == "html":
+
+            outfile = save_html(
+                results,
+                "comparison_report.html"
+            )
+
+            print()
+            print("HTML report saved:")
+            print(outfile)
+
+
+
+        elif args.format == "charts":
+
+            latency_file = "latency_comparison.png"
+            throughput_file = "throughput_comparison.png"
+
+
+            save_latency_chart(
+                results,
+                latency_file
+            )
+
+            save_throughput_chart(
+                results,
+                throughput_file
+            )
+
+
+            print()
+            print("Charts generated:")
+            print(latency_file)
+            print(throughput_file)
+
+
+
+if __name__ == "__main__":
+    main()

openmodellab/comparison/compare.py

@@ -0,0 +1,106 @@
+import json
+import os
+
+
+def load_reports(report_dir):
+
+    models = {}
+
+    for file in os.listdir(report_dir):
+
+        path = os.path.join(
+            report_dir,
+            file
+        )
+
+        if file.endswith("_genome.json"):
+
+            name = file.replace(
+                "_genome.json",
+                ""
+            )
+
+            models.setdefault(name, {})
+            
+            with open(path, "r") as f:
+                models[name]["genome"] = json.load(f)
+
+
+        elif file.endswith("_benchmark.json"):
+
+            name = file.replace(
+                "_benchmark.json",
+                ""
+            )
+
+            models.setdefault(name, {})
+
+            with open(path, "r") as f:
+                models[name]["benchmark"] = json.load(f)
+
+    return models
+
+
+
+def generate_comparison(report_dir):
+
+    reports = load_reports(report_dir)
+
+    table = []
+
+
+    for name, data in reports.items():
+
+        genome = data.get(
+            "genome",
+            {}
+        )
+
+        benchmark = data.get(
+            "benchmark",
+            {}
+        )
+
+
+        table.append({
+
+            "model": name,
+
+            "parameters": genome.get(
+                "model",
+                {}
+            ).get(
+                "parameter_count"
+            ),
+
+            "layers": genome.get(
+                "model",
+                {}
+            ).get(
+                "num_hidden_layers"
+            ),
+
+            "hidden_size": genome.get(
+                "model",
+                {}
+            ).get(
+                "hidden_size"
+            ),
+
+            "latency_ms": benchmark.get(
+                "latency",
+                {}
+            ).get(
+                "average_ms"
+            ),
+
+            "throughput": benchmark.get(
+                "throughput",
+                {}
+            ).get(
+                "samples_per_second"
+            )
+        })
+
+
+    return table

openmodellab/comparison/table.py

@@ -0,0 +1,37 @@
+from tabulate import tabulate
+
+
+def print_table(results):
+
+    rows = []
+
+    for item in results:
+
+        rows.append([
+            item["model"],
+            round(
+                item["parameters"] / 1_000_000,
+                2
+            ) if item["parameters"] else None,
+            item["layers"],
+            item["hidden_size"],
+            item["latency_ms"],
+            item["throughput"]
+        ])
+
+    print()
+
+    print(
+        tabulate(
+            rows,
+            headers=[
+                "Model",
+                "Params(M)",
+                "Layers",
+                "Hidden",
+                "Latency(ms)",
+                "Throughput"
+            ],
+            tablefmt="grid"
+        )
+    )

openmodellab/genome/analyzer.py

@@ -0,0 +1,54 @@
+from huggingface_hub import model_info
+
+from openmodellab.genome.inspectors.tokenizer import inspect_tokenizer
+
+from openmodellab.genome.inspectors.hardware import inspect_hardware
+
+from openmodellab.genome.inspectors.fingerprint import inspect_fingerprint
+
+from datetime import datetime, UTC
+
+from openmodellab.genome.inspectors.precision import inspect_precision
+
+from openmodellab.genome.inspectors.execution import inspect_execution
+
+
+def count_parameters(model):
+    return sum(p.numel() for p in model.parameters())
+
+
+def inspect_model(model_name, model):
+    cfg = model.config
+    info = model_info(model_name)
+
+    return {
+        "model_name": model_name,
+        "architecture": model.__class__.__name__,
+        "model_type": getattr(cfg, "model_type", None),
+        "hidden_size": getattr(cfg, "hidden_size", None),
+        "num_hidden_layers": getattr(cfg, "num_hidden_layers", None),
+        "num_attention_heads": getattr(cfg, "num_attention_heads", None),
+        "intermediate_size": getattr(cfg, "intermediate_size", None),
+        "max_position_embeddings": getattr(cfg, "max_position_embeddings", None),
+        "parameter_count": count_parameters(model),
+        "dtype": str(next(model.parameters()).dtype),
+        "framework": "transformers",
+        "license": info.card_data.get("license") if info.card_data else None,
+    }
+
+
+def analyze_model(model_name, model, tokenizer):
+    return {
+         "report": {
+            "tool": "OpenModelLab",
+            "tool_version": "0.1.0-alpha",
+            "schema_version": "1.0",
+            "generated_at": datetime.now(UTC).isoformat(),
+        },   
+        "model": inspect_model(model_name, model),
+        "tokenizer": inspect_tokenizer(tokenizer),
+        "hardware": inspect_hardware(),
+        "fingerprint": inspect_fingerprint(model),
+        "precision": inspect_precision(model),
+        "execution": inspect_execution(model),
+    }

openmodellab/genome/inspectors/batch.py

@@ -0,0 +1,70 @@
+import time
+import torch
+
+
+def inspect_batch_scaling(
+    model,
+    tokenizer,
+    batch_sizes=None,
+):
+    if batch_sizes is None:
+        batch_sizes = [1, 2, 4, 8]
+
+    model.eval()
+
+    device = next(model.parameters()).device
+
+    results = []
+
+    for batch_size in batch_sizes:
+
+        texts = [
+            "OpenModelLab batch benchmark."
+            for _ in range(batch_size)
+        ]
+
+        inputs = tokenizer(
+            texts,
+            return_tensors="pt",
+            padding=True,
+            truncation=True,
+            max_length=128,
+        )
+
+        inputs = {
+            key: value.to(device)
+            for key, value in inputs.items()
+        }
+
+        with torch.no_grad():
+            _ = model(**inputs)
+
+        if str(device).startswith("cuda"):
+            torch.cuda.synchronize()
+
+        start = time.perf_counter()
+
+        with torch.no_grad():
+            _ = model(**inputs)
+
+        if str(device).startswith("cuda"):
+            torch.cuda.synchronize()
+
+        end = time.perf_counter()
+
+        latency_ms = (end - start) * 1000
+
+        results.append({
+            "batch_size": batch_size,
+            "latency_ms": round(latency_ms, 3),
+            "samples_per_second": round(
+                batch_size / (end - start),
+                3
+            )
+        })
+
+    return {
+        "device": str(device),
+        "batch_sizes": batch_sizes,
+        "results": results
+    }

openmodellab/genome/inspectors/execution.py

@@ -0,0 +1,28 @@
+import torch
+
+
+def inspect_execution(model, requested_device="auto"):
+
+    actual_device = str(
+        next(model.parameters()).device
+    )
+
+    cuda_available = torch.cuda.is_available()
+
+    fallback = (
+        cuda_available
+        and not actual_device.startswith("cuda")
+    )
+
+    gpu_name = None
+
+    if cuda_available:
+        gpu_name = torch.cuda.get_device_name(0)
+
+    return {
+        "requested_device": requested_device,
+        "actual_device": actual_device,
+        "cuda_available": cuda_available,
+        "gpu_name": gpu_name,
+        "fallback": fallback
+    }

openmodellab/genome/inspectors/fingerprint.py

@@ -0,0 +1,24 @@
+import hashlib
+import json
+import transformers
+
+
+def inspect_fingerprint(model):
+    cfg = model.config.to_dict()
+
+    config_json = json.dumps(cfg, sort_keys=True)
+
+    sha = hashlib.sha256(config_json.encode()).hexdigest()
+
+    signature = (
+        f"{cfg.get('model_type', 'unknown')}-"
+        f"{cfg.get('num_hidden_layers', '?')}L-"
+        f"{cfg.get('hidden_size', '?')}H-"
+        f"{cfg.get('num_attention_heads', '?')}A"
+    )
+
+    return {
+        "architecture_signature": signature,
+        "config_sha256": sha,
+        "transformers_version": transformers.__version__,
+    }

openmodellab/genome/inspectors/hardware.py

@@ -0,0 +1,27 @@
+import torch
+
+
+def inspect_hardware():
+    report = {
+        "torch_version": torch.__version__,
+        "cuda_available": torch.cuda.is_available(),
+        "cuda_version": torch.version.cuda,
+        "device_count": torch.cuda.device_count(),
+    }
+
+    if torch.cuda.is_available():
+        props = torch.cuda.get_device_properties(0)
+
+        free_mem, total_mem = torch.cuda.mem_get_info()
+
+        report.update({
+            "device": "cuda:0",
+            "gpu_name": props.name,
+            "gpu_total_memory_mb": round(total_mem / 1024 / 1024, 2),
+            "gpu_free_memory_mb": round(free_mem / 1024 / 1024, 2),
+            "gpu_compute_capability": f"{props.major}.{props.minor}",
+        })
+    else:
+        report["device"] = "cpu"
+
+    return report

openmodellab/genome/inspectors/latency.py

@@ -0,0 +1,78 @@
+import statistics
+import time
+
+import torch
+
+
+def inspect_latency(
+    model,
+    tokenizer,
+    text="OpenModelLab is benchmarking Hugging Face models.",
+    warmup_runs=5,
+    benchmark_runs=20,
+):
+    model.eval()
+
+    # Detect execution device
+    if hasattr(model, "hf_device_map"):
+        devices = [
+            d for d in model.hf_device_map.values()
+            if d != "cpu"
+        ]
+
+        if devices:
+            current_device = torch.device(devices[0])
+        else:
+            current_device = torch.device("cpu")
+    else:
+        current_device = next(model.parameters()).device
+
+    device = str(current_device)
+
+    inputs = tokenizer(
+        text,
+        return_tensors="pt",
+        truncation=True,
+        max_length=128,
+    )
+
+    inputs = {
+        key: value.to(current_device)
+        for key, value in inputs.items()
+    }
+
+    with torch.no_grad():
+        for _ in range(warmup_runs):
+            _ = model(**inputs)
+
+    if device.startswith("cuda"):
+        torch.cuda.synchronize()
+
+    timings = []
+
+    with torch.no_grad():
+        for _ in range(benchmark_runs):
+
+            if device.startswith("cuda"):
+                torch.cuda.synchronize()
+
+            start = time.perf_counter()
+
+            _ = model(**inputs)
+
+            if device.startswith("cuda"):
+                torch.cuda.synchronize()
+
+            end = time.perf_counter()
+
+            timings.append((end - start) * 1000)
+
+    return {
+        "device": device,
+        "warmup_runs": warmup_runs,
+        "benchmark_runs": benchmark_runs,
+        "average_ms": round(statistics.mean(timings), 3),
+        "min_ms": round(min(timings), 3),
+        "max_ms": round(max(timings), 3),
+        "std_ms": round(statistics.stdev(timings), 3),
+    }

openmodellab/genome/inspectors/memory.py

@@ -0,0 +1,56 @@
+import os
+import psutil
+import torch
+
+
+def get_process_memory_mb():
+    process = psutil.Process(os.getpid())
+    return process.memory_info().rss / (1024 ** 2)
+
+
+def inspect_memory(model, tokenizer):
+
+    model.eval()
+
+    device = str(next(model.parameters()).device)
+
+    memory = {
+        "device": device,
+        "ram_before_mb": round(get_process_memory_mb(), 2),
+    }
+
+    text = "OpenModelLab memory benchmark."
+
+    inputs = tokenizer(
+        text,
+        return_tensors="pt",
+        truncation=True,
+        max_length=128,
+    )
+
+    inputs = {
+        key: value.to(next(model.parameters()).device)
+        for key, value in inputs.items()
+    }
+
+    if device.startswith("cuda"):
+        torch.cuda.reset_peak_memory_stats()
+        torch.cuda.empty_cache()
+
+    with torch.no_grad():
+        _ = model(**inputs)
+
+    memory["ram_after_mb"] = round(get_process_memory_mb(), 2)
+
+    if device.startswith("cuda"):
+        memory["gpu_allocated_mb"] = round(
+            torch.cuda.memory_allocated() / (1024 ** 2),
+            2
+        )
+
+        memory["gpu_peak_allocated_mb"] = round(
+            torch.cuda.max_memory_allocated() / (1024 ** 2),
+            2
+        )
+
+    return memory

openmodellab/genome/inspectors/precision.py

@@ -0,0 +1,40 @@
+import torch
+
+
+def inspect_precision(model):
+
+    dtype = str(
+        next(model.parameters()).dtype
+    )
+
+    mapping = {
+        "torch.float32": {
+            "bits": 32,
+            "category": "FP32"
+        },
+        "torch.float16": {
+            "bits": 16,
+            "category": "FP16"
+        },
+        "torch.bfloat16": {
+            "bits": 16,
+            "category": "BF16"
+        },
+        "torch.int8": {
+            "bits": 8,
+            "category": "INT8"
+        }
+    }
+
+    info = mapping.get(
+        dtype,
+        {
+            "bits": None,
+            "category": "unknown"
+        }
+    )
+
+    return {
+        "dtype": dtype,
+        **info
+    }

openmodellab/genome/inspectors/throughput.py

@@ -0,0 +1,57 @@
+import time
+import torch
+
+
+def inspect_throughput(
+    model,
+    tokenizer,
+    text="OpenModelLab throughput benchmark.",
+    runs=50,
+):
+    model.eval()
+
+    device = next(model.parameters()).device
+    device_name = str(device)
+
+    inputs = tokenizer(
+        text,
+        return_tensors="pt",
+        truncation=True,
+        max_length=128,
+    )
+
+    inputs = {
+        key: value.to(device)
+        for key, value in inputs.items()
+    }
+
+    # Warmup
+    with torch.no_grad():
+        for _ in range(5):
+            _ = model(**inputs)
+
+    if device_name.startswith("cuda"):
+        torch.cuda.synchronize()
+
+    start = time.perf_counter()
+
+    with torch.no_grad():
+        for _ in range(runs):
+            _ = model(**inputs)
+
+    if device_name.startswith("cuda"):
+        torch.cuda.synchronize()
+
+    end = time.perf_counter()
+
+    total_time = end - start
+
+    return {
+        "device": device_name,
+        "runs": runs,
+        "total_time_sec": round(total_time, 4),
+        "samples_per_second": round(
+            runs / total_time,
+            3
+        ),
+    }

openmodellab/genome/inspectors/tokenizer.py

@@ -0,0 +1,10 @@
+def inspect_tokenizer(tokenizer):
+    return {
+        "vocab_size": tokenizer.vocab_size,
+        "model_max_length": tokenizer.model_max_length,
+        "is_fast": tokenizer.is_fast,
+        "padding_side": tokenizer.padding_side,
+        "truncation_side": tokenizer.truncation_side,
+        "special_tokens": tokenizer.special_tokens_map,
+        "all_special_tokens": tokenizer.all_special_tokens,
+    }

openmodellab/genome/model_loader.py

@@ -0,0 +1,49 @@
+from transformers import AutoModel, AutoTokenizer
+import torch
+
+
+def load_model(model_name: str, device: str = "auto"):
+
+    print(f"Loading model: {model_name}")
+
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+
+    if device == "auto":
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+
+    if device == "cuda":
+
+        try:
+            model = AutoModel.from_pretrained(
+                model_name
+            )
+
+            model = model.cuda()
+
+            print(
+                "Model loaded on CUDA:",
+                torch.cuda.get_device_name(0)
+            )
+
+        except Exception as e:
+
+            print("Standard CUDA loading failed.")
+            print(e)
+
+            print("Trying automatic device mapping...")
+
+            model = AutoModel.from_pretrained(
+                model_name,
+                device_map="auto"
+            )
+
+    else:
+        model = AutoModel.from_pretrained(
+            model_name
+        )
+
+        model = model.cpu()
+
+        print("Model loaded on CPU.")
+
+    return model, tokenizer

openmodellab/reporting/benchmark_writer.py

@@ -0,0 +1,18 @@
+from pathlib import Path
+import json
+
+
+def save_benchmark(report: dict, model_name: str):
+    """
+    Save benchmark report as benchmark.json.
+    """
+
+    outdir = Path("outputs") / model_name.replace("/", "_")
+    outdir.mkdir(parents=True, exist_ok=True)
+
+    outfile = outdir / "benchmark.json"
+
+    with open(outfile, "w", encoding="utf-8") as f:
+        json.dump(report, f, indent=2)
+
+    return str(outfile)

openmodellab/reporting/csv_writer.py

@@ -0,0 +1,34 @@
+import csv
+
+
+def save_csv(results, filename):
+
+    with open(
+        filename,
+        "w",
+        newline=""
+    ) as f:
+
+        writer = csv.writer(f)
+
+        writer.writerow([
+            "Model",
+            "Parameters",
+            "Layers",
+            "Hidden Size",
+            "Latency(ms)",
+            "Throughput"
+        ])
+
+        for item in results:
+
+            writer.writerow([
+                item["model"],
+                item["parameters"],
+                item["layers"],
+                item["hidden_size"],
+                item["latency_ms"],
+                item["throughput"]
+            ])
+
+    return filename

openmodellab/reporting/html_writer.py

@@ -0,0 +1,50 @@
+def save_html(results, filename):
+
+    rows = ""
+
+    for item in results:
+
+        rows += f"""
+        <tr>
+            <td>{item['model']}</td>
+            <td>{item['parameters']}</td>
+            <td>{item['layers']}</td>
+            <td>{item['hidden_size']}</td>
+            <td>{item['latency_ms']}</td>
+            <td>{item['throughput']}</td>
+        </tr>
+        """
+
+    html = f"""
+    <html>
+    <head>
+        <title>OpenModelLab Comparison</title>
+    </head>
+
+    <body>
+
+    <h1>OpenModelLab Model Comparison</h1>
+
+    <table border="1">
+
+    <tr>
+        <th>Model</th>
+        <th>Parameters</th>
+        <th>Layers</th>
+        <th>Hidden Size</th>
+        <th>Latency(ms)</th>
+        <th>Throughput</th>
+    </tr>
+
+    {rows}
+
+    </table>
+
+    </body>
+    </html>
+    """
+
+    with open(filename, "w") as f:
+        f.write(html)
+
+    return filename

openmodellab/reporting/json_writer.py

@@ -0,0 +1,16 @@
+import json
+from pathlib import Path
+
+
+def save_json(report: dict, model_name: str):
+    model_dir = model_name.replace("/", "_")
+
+    out_dir = Path("outputs") / model_dir
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    output_file = out_dir / "genome.json"
+
+    with open(output_file, "w", encoding="utf-8") as f:
+        json.dump(report, f, indent=2)
+
+    return output_file

openmodellab/visualization/charts.py

@@ -0,0 +1,90 @@
+import matplotlib.pyplot as plt
+
+
+def save_latency_chart(results, filename):
+
+    models = [
+        item["model"]
+        for item in results
+    ]
+
+    latency = [
+        item["latency_ms"]
+        for item in results
+    ]
+
+    plt.figure(figsize=(8, 5))
+
+    plt.bar(
+        models,
+        latency
+    )
+
+    plt.xlabel(
+        "Model"
+    )
+
+    plt.ylabel(
+        "Latency (ms)"
+    )
+
+    plt.title(
+        "Model Latency Comparison"
+    )
+
+    plt.xticks(
+        rotation=45
+    )
+
+    plt.tight_layout()
+
+    plt.savefig(
+        filename
+    )
+
+    plt.close()
+
+
+
+def save_throughput_chart(results, filename):
+
+    models = [
+        item["model"]
+        for item in results
+    ]
+
+    throughput = [
+        item["throughput"]
+        for item in results
+    ]
+
+    plt.figure(figsize=(8, 5))
+
+    plt.bar(
+        models,
+        throughput
+    )
+
+    plt.xlabel(
+        "Model"
+    )
+
+    plt.ylabel(
+        "Samples / Second"
+    )
+
+    plt.title(
+        "Model Throughput Comparison"
+    )
+
+    plt.xticks(
+        rotation=45
+    )
+
+    plt.tight_layout()
+
+    plt.savefig(
+        filename
+    )
+
+    plt.close()

requirements.txt

@@ -4,4 +4,3 @@ torch
 transformers
 sentencepiece
 safetensors
-git+https://github.com/ajaygovinds/OpenModelLab.git