Upload extraction model

README.md

@@ -0,0 +1,72 @@
+---
+license: apache-2.0
+language: en
+library_name: transformers
+pipeline_tag: token-classification
+base_model: distilbert-base-cased
+tags: [token-classification, document-understanding, procurement, india, gem, synthetic-data]
+---
+
+# GeM certificate field extractor
+
+Token-classification model that reads the OCR text of Indian business certificates submitted with
+Government e-Marketplace (GeM) bids and extracts the fields a compliance officer cross-checks:
+GSTIN, PAN, Udyam number, CIN, EPFO establishment code, DPIIT number, legal and trade name,
+activity, enterprise category, certified local-content % and validity-until date.
+
+Part of an automated bid-compliance platform; it replaces a regex parser in the document stage,
+and its output is cross-verified against government portal data by an auditable rule engine.
+
+## Intended use
+
+Input: plain text from a certificate (PDF text layer or OCR). Output: one value per field, with a
+confidence. Covered document types: GST registration certificate (REG-06), Udyam certificate,
+Make in India local-content self-certificate, PAN card, EPFO compliance certificate, DPIIT startup
+recognition, OEM authorization letter, CA turnover certificate, NSIC registration.
+
+It is an extraction aid, not a verifier: a correctly extracted GSTIN says nothing about whether
+that GSTIN is active — that comes from the GST portal.
+
+## Training data
+
+Generated by `ml/extraction/generate.py`: 54,000 train, 3,000 validation, 3,000 test examples.
+
+The generator deliberately includes look-alikes a regex gets wrong: the OEM's own GSTIN beside the
+bidder's in authorization letters, a "minimum local content of N%" threshold restated before the
+certified figure, validity *start* dates, PANs embedded in GSTINs, past-year Udyam classifications,
+and UDIN/FRN/ESIC numbers — plus OCR-style character confusions (O/0, S/5, I/1) and merged lines.
+
+**All training data is synthetic.** Identifiers follow real formats (GSTINs carry a valid check
+character) but correspond to no real entity. Accuracy on real, scanned certificates with layouts
+unlike the templates will be lower than the figures below; evaluate on your own documents first.
+
+## Evaluation (held-out synthetic test set)
+
+Entity-level, exact span match:
+
+| Field | Precision | Recall | F1 | Support |
+|---|---|---|---|---|
+| ACTIVITY | 100.0% | 100.0% | 100.0% | 902 |
+| CATEGORY | 100.0% | 100.0% | 100.0% | 902 |
+| CIN | 100.0% | 100.0% | 100.0% | 124 |
+| DPIIT | 100.0% | 100.0% | 100.0% | 312 |
+| EPFO | 100.0% | 100.0% | 100.0% | 311 |
+| GSTIN | 100.0% | 100.0% | 100.0% | 1174 |
+| LEGAL_NAME | 100.0% | 100.0% | 100.0% | 3000 |
+| LOCAL_CONTENT | 100.0% | 99.7% | 99.8% | 296 |
+| PAN | 100.0% | 100.0% | 100.0% | 593 |
+| TRADE_NAME | 100.0% | 100.0% | 100.0% | 296 |
+| UDYAM | 100.0% | 100.0% | 100.0% | 902 |
+| VALID_UNTIL | 100.0% | 100.0% | 100.0% | 789 |
+| **All (micro)** | 100.0% | 100.0% | **100.0%** | 9601 |
+
+## Usage
+
+```python
+from transformers import pipeline
+extract = pipeline("token-classification", model="HarshilDaGoat/gem-certificate-extractor", aggregation_strategy="simple")
+extract("Registration Number: 27AABCN1234A1Z5\nLegal Name of Business: Nova Electro Systems Private Limited")
+```
+
+For field values normalised exactly as the platform uses them (dates as ISO, local content as a
+number, one value per field), use `ml/extraction/infer.py` from the project repository.

config.json

@@ -0,0 +1,82 @@
+{
+  "activation": "gelu",
+  "architectures": [
+    "DistilBertForTokenClassification"
+  ],
+  "attention_dropout": 0.1,
+  "bos_token_id": null,
+  "dim": 768,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "hidden_dim": 3072,
+  "id2label": {
+    "0": "O",
+    "1": "B-GSTIN",
+    "2": "I-GSTIN",
+    "3": "B-PAN",
+    "4": "I-PAN",
+    "5": "B-UDYAM",
+    "6": "I-UDYAM",
+    "7": "B-CIN",
+    "8": "I-CIN",
+    "9": "B-EPFO",
+    "10": "I-EPFO",
+    "11": "B-DPIIT",
+    "12": "I-DPIIT",
+    "13": "B-TRADE_NAME",
+    "14": "I-TRADE_NAME",
+    "15": "B-LEGAL_NAME",
+    "16": "I-LEGAL_NAME",
+    "17": "B-ACTIVITY",
+    "18": "I-ACTIVITY",
+    "19": "B-CATEGORY",
+    "20": "I-CATEGORY",
+    "21": "B-LOCAL_CONTENT",
+    "22": "I-LOCAL_CONTENT",
+    "23": "B-VALID_UNTIL",
+    "24": "I-VALID_UNTIL"
+  },
+  "initializer_range": 0.02,
+  "label2id": {
+    "B-ACTIVITY": 17,
+    "B-CATEGORY": 19,
+    "B-CIN": 7,
+    "B-DPIIT": 11,
+    "B-EPFO": 9,
+    "B-GSTIN": 1,
+    "B-LEGAL_NAME": 15,
+    "B-LOCAL_CONTENT": 21,
+    "B-PAN": 3,
+    "B-TRADE_NAME": 13,
+    "B-UDYAM": 5,
+    "B-VALID_UNTIL": 23,
+    "I-ACTIVITY": 18,
+    "I-CATEGORY": 20,
+    "I-CIN": 8,
+    "I-DPIIT": 12,
+    "I-EPFO": 10,
+    "I-GSTIN": 2,
+    "I-LEGAL_NAME": 16,
+    "I-LOCAL_CONTENT": 22,
+    "I-PAN": 4,
+    "I-TRADE_NAME": 14,
+    "I-UDYAM": 6,
+    "I-VALID_UNTIL": 24,
+    "O": 0
+  },
+  "max_position_embeddings": 512,
+  "model_type": "distilbert",
+  "n_heads": 12,
+  "n_layers": 6,
+  "output_past": true,
+  "pad_token_id": 0,
+  "qa_dropout": 0.1,
+  "seq_classif_dropout": 0.2,
+  "sinusoidal_pos_embds": false,
+  "tie_weights_": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.17.0",
+  "use_cache": false,
+  "vocab_size": 28996
+}

field_eval.json

@@ -0,0 +1,68 @@
+{
+  "model": {
+    "overall_accuracy": 0.9999,
+    "overall_false_positive_rate": 0.0,
+    "per_field": {
+      "gstin": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 1174
+      },
+      "pan": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 593
+      },
+      "udyam_number": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 902
+      },
+      "cin": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 124
+      },
+      "establishment_code": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 311
+      },
+      "dpiit_number": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 312
+      },
+      "trade_name": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 296
+      },
+      "legal_name": {
+        "accuracy": 1.0,
+        "false_positive_rate": null,
+        "support": 3000
+      },
+      "activity": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 902
+      },
+      "enterprise_category": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 901
+      },
+      "local_content_pct": {
+        "accuracy": 0.9966,
+        "false_positive_rate": 0.0,
+        "support": 296
+      },
+      "valid_until": {
+        "accuracy": 1.0,
+        "false_positive_rate": 0.0,
+        "support": 784
+      }
+    }
+  }
+}

metrics.json

@@ -0,0 +1,95 @@
+{
+  "test": {
+    "micro": {
+      "precision": 1.0,
+      "recall": 0.9999,
+      "f1": 0.9999,
+      "support": 9601
+    },
+    "per_label": {
+      "ACTIVITY": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 902
+      },
+      "CATEGORY": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 902
+      },
+      "CIN": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 124
+      },
+      "DPIIT": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 312
+      },
+      "EPFO": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 311
+      },
+      "GSTIN": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 1174
+      },
+      "LEGAL_NAME": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 3000
+      },
+      "LOCAL_CONTENT": {
+        "precision": 1.0,
+        "recall": 0.9966,
+        "f1": 0.9983,
+        "support": 296
+      },
+      "PAN": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 593
+      },
+      "TRADE_NAME": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 296
+      },
+      "UDYAM": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 902
+      },
+      "VALID_UNTIL": {
+        "precision": 1.0,
+        "recall": 1.0,
+        "f1": 1.0,
+        "support": 789
+      }
+    }
+  },
+  "args": {
+    "data": "/content/repo/ml/data/extraction",
+    "out": "/content/repo/ml/models/extraction",
+    "base_model": "distilbert-base-cased",
+    "epochs": 3.0,
+    "batch_size": 16,
+    "lr": 5e-05,
+    "max_length": 512,
+    "limit": null,
+    "seed": 13
+  }
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:963dbdd68a515c36e88c86e77da10ee8f842709b748ded1ff9841294b52fbee1
+size 260852884

tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "backend": "tokenizers",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f98a77b89a5a75458f58e8ff6427bfd9d7e513bad0562966d01d1f0c5fbe3bd
+size 5265