| |
| """Harness: COBOL evidence fixtures → evidence with deterministic 1536-d vectors. |
| |
| Mirrors sql/003_evidence_seed_cobol.sql IDs/content; fills embedding (003 leaves NULL). |
| Reuse embed/fmt_vector from load_agent_registry (smoke hashed bag-of-tokens only). |
| """ |
| from __future__ import annotations |
|
|
| import re |
| import sys |
| from pathlib import Path |
|
|
| sys.path.insert(0, str(Path(__file__).resolve().parent)) |
| from load_agent_registry import embed, fmt_vector |
|
|
| try: |
| import psycopg2 |
| except ImportError: |
| psycopg2 = None |
|
|
| |
| FIXTURES = [ |
| { |
| "evidence_id": "ev_copybook_01", |
| "source_id": "src_copybook_acct", |
| "source_type": "copybook", |
| "document_id": "ACCTREC.cpy", |
| "location": "01-ACCOUNT-BALANCE", |
| "content": ( |
| "01 ACCOUNT-RECORD. 05 ACCOUNT-BALANCE PIC S9(9)V99. " |
| "05 PAYMENT-AMOUNT PIC S9(7)V99. 05 LAST-PAYMENT-DATE PIC X(8)." |
| ), |
| "content_hash": "seed_copybook_01", |
| "source_authority": 0.95, |
| "parent_evidence": ["ev_provenance_root"], |
| }, |
| { |
| "evidence_id": "ev_cobol_src_01", |
| "source_id": "src_cobol_pay", |
| "source_type": "cobol_source", |
| "document_id": "PAYROLL.cbl", |
| "location": "PROCEDURE DIVISION / CALC-BALANCE", |
| "content": ( |
| "COMPUTE ACCOUNT-BALANCE = ACCOUNT-BALANCE - PAYMENT-AMOUNT. " |
| "IF ACCOUNT-BALANCE < ZERO THEN PERFORM OVERDRAFT-CHECK. " |
| "MOVE PAYMENT-AMOUNT TO WS-LAST-PAYMENT." |
| ), |
| "content_hash": "seed_cobol_src_01", |
| "source_authority": 0.90, |
| "parent_evidence": ["ev_provenance_root"], |
| }, |
| { |
| "evidence_id": "ev_db2_schema_01", |
| "source_id": "src_db2_acct", |
| "source_type": "db2_schema", |
| "document_id": "ACCT.DDL", |
| "location": "TABLE ACCOUNT_BALANCES", |
| "content": ( |
| "CREATE TABLE ACCOUNT_BALANCES (ACCT_ID CHAR(12), BALANCE DECIMAL(11,2), " |
| "LAST_PAYMENT DECIMAL(9,2), UPDATED_TS TIMESTAMP)." |
| ), |
| "content_hash": "seed_db2_schema_01", |
| "source_authority": 0.85, |
| "parent_evidence": ["ev_provenance_root"], |
| }, |
| { |
| "evidence_id": "ev_quant_pricing_01", |
| "source_id": "src_quant_report", |
| "source_type": "quant_report", |
| "document_id": "PRICING_Q3.pdf", |
| "location": "section:monte_carlo", |
| "content": ( |
| "Monte Carlo pricing of exotic options. Implied volatility surface and " |
| "greek sensitivities. No COBOL or account balance fields." |
| ), |
| "content_hash": "seed_quant_pricing_01", |
| "source_authority": 0.80, |
| "parent_evidence": ["ev_provenance_root"], |
| }, |
| { |
| "evidence_id": "ev_lexical_lift_01", |
| "source_id": "src_glossary", |
| "source_type": "glossary", |
| "document_id": "LEGACY-GLOSSARY.txt", |
| "location": "entry:PAYMENT-BALANCE", |
| "content": ( |
| "cobol payment balance glossary entry: PAYMENT and BALANCE fields on " |
| "ACCOUNT records in legacy batch." |
| ), |
| "content_hash": "seed_lexical_lift_01", |
| "source_authority": 0.70, |
| "parent_evidence": ["ev_provenance_root"], |
| }, |
| { |
| "evidence_id": "ev_junk_low_auth", |
| "source_id": "src_web_scrape", |
| "source_type": "web_scrape", |
| "document_id": "random-blog.html", |
| "location": "body", |
| "content": ( |
| "Someone said COBOL payment balance might work like a spreadsheet. " |
| "Unverified forum post." |
| ), |
| "content_hash": "seed_junk_low_auth", |
| "source_authority": 0.10, |
| "parent_evidence": [], |
| }, |
| ] |
|
|
|
|
| def normalize_for_embed(text: str) -> str: |
| """Split identifiers on - _ / . so ACCOUNT_BALANCES → account balances tokens.""" |
| return re.sub(r"[-_./]+", " ", text.lower()) |
|
|
|
|
| def embed_text_of(f: dict) -> str: |
| """content + source_type + document_id + location for dense path.""" |
| parts = [ |
| f.get("source_type") or "", |
| f.get("document_id") or "", |
| f.get("location") or "", |
| f["content"], |
| ] |
| return " ".join(p for p in parts if p) |
|
|
|
|
| def main() -> int: |
| dsn = sys.argv[1] if len(sys.argv) > 1 else "dbname=david user=david host=127.0.0.1" |
| if psycopg2 is None: |
| print("psycopg2 missing", file=sys.stderr) |
| return 2 |
|
|
| conn = psycopg2.connect(dsn) |
| conn.autocommit = True |
| with conn.cursor() as cur: |
| for f in FIXTURES: |
| et = normalize_for_embed(embed_text_of(f)) |
| emb = fmt_vector(embed(et)) |
| cur.execute( |
| """ |
| INSERT INTO evidence ( |
| evidence_id, source_id, source_type, document_id, location, |
| content, content_hash, embedding, source_authority, parent_evidence |
| ) VALUES ( |
| %s, %s, %s, %s, %s, |
| %s, %s, %s::vector, %s, %s::text[] |
| ) |
| ON CONFLICT (evidence_id) DO UPDATE SET |
| source_id = EXCLUDED.source_id, |
| source_type = EXCLUDED.source_type, |
| document_id = EXCLUDED.document_id, |
| location = EXCLUDED.location, |
| content = EXCLUDED.content, |
| content_hash = EXCLUDED.content_hash, |
| embedding = EXCLUDED.embedding, |
| source_authority = EXCLUDED.source_authority, |
| parent_evidence = EXCLUDED.parent_evidence |
| """, |
| ( |
| f["evidence_id"], |
| f["source_id"], |
| f["source_type"], |
| f["document_id"], |
| f["location"], |
| f["content"], |
| f["content_hash"], |
| emb, |
| f["source_authority"], |
| f["parent_evidence"], |
| ), |
| ) |
| cur.execute( |
| "SELECT count(*) FILTER (WHERE embedding IS NOT NULL), count(*) FROM evidence" |
| ) |
| with_emb, total = cur.fetchone() |
| conn.close() |
| print(f"loaded_evidence={total} with_embedding={with_emb}") |
| return 0 |
|
|
|
|
| if __name__ == "__main__": |
| raise SystemExit(main()) |
|
|
