File size: 5,390 Bytes
e699b35
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
import os
import json
import csv
import time
import asyncio
from pydantic import BaseModel, Field
from google import genai
from google.genai import types

# Define the structured output schema for Gemini
class Vertex(BaseModel):
    id: str = Field(description="The unique identifier or name of the entity.")
    type: str = Field(description="Must be one of: Company, Executive, RiskFactor, Subsidiary")

class Edge(BaseModel):
    source: str = Field(description="The id of the source vertex.")
    target: str = Field(description="The id of the target vertex.")
    type: str = Field(description="Must be one of: EMPLOYS, FACES_RISK, OWNS, COMPETES_WITH")

class GraphExtraction(BaseModel):
    vertices: list[Vertex]
    edges: list[Edge]

# Setup Gemini Client (Async)
if "GEMINI_API_KEY" not in os.environ:
    print("ERROR: GEMINI_API_KEY environment variable not set.")
    exit(1)

client = genai.Client()

async def process_document(doc, sem):
    """
    Process a single document through Gemini 1.5 Flash to extract graph entities.
    Uses a semaphore to limit concurrent API calls and prevent 429 errors.
    """
    async with sem:
        text = doc.get("text", "")
        company = doc.get("company", "Unknown")
        
        # If the text is too large, we truncate it for extraction to save costs 
        # (For 100M tokens, we might process full texts, but let's limit to 15k chars per doc for the demo)
        text_chunk = text[:15000] 
        
        prompt = f"""
        Analyze the following SEC 10-K filing excerpt for {company}.
        Extract all relevant entities (Company, Executive, RiskFactor, Subsidiary) and their relationships (EMPLOYS, FACES_RISK, OWNS, COMPETES_WITH).
        
        Text excerpt:
        {text_chunk}
        """
        
        try:
            # Note: We use the async client `client.aio`
            response = await client.aio.models.generate_content(
                model='gemini-2.5-flash',
                contents=prompt,
                config=types.GenerateContentConfig(
                    response_mime_type="application/json",
                    response_schema=GraphExtraction,
                    temperature=0.1
                )
            )
            
            # The response text will be a JSON string conforming to the GraphExtraction schema
            return json.loads(response.text)
            
        except Exception as e:
            print(f"Error extracting graph for {company}: {e}")
            return None

async def main():
    input_file = "financial_corpus.jsonl"
    if not os.path.exists(input_file):
        print(f"ERROR: {input_file} not found. Run count_tokens.py first.")
        return

    # Prepare CSV writers
    vertices_file = open("vertices.csv", "w", newline="", encoding="utf-8")
    edges_file = open("edges.csv", "w", newline="", encoding="utf-8")
    
    v_writer = csv.writer(vertices_file)
    e_writer = csv.writer(edges_file)
    
    # Write headers
    v_writer.writerow(["v_id", "v_type"])
    e_writer.writerow(["source", "target", "e_type"])
    
    # Read corpus
    print("Loading corpus...")
    docs = []
    with open(input_file, 'r', encoding='utf-8') as f:
        for line in f:
            docs.append(json.loads(line))
            if len(docs) >= 50:
                break
            
    print(f"Loaded {len(docs)} documents for extraction.")
    
    # We use a Semaphore to allow max 5 concurrent requests to Gemini to respect standard tier rate limits
    sem = asyncio.Semaphore(5)
    
    # Batch processing
    BATCH_SIZE = 50
    total_vertices = 0
    total_edges = 0
    
    for i in range(0, len(docs), BATCH_SIZE):
        batch = docs[i:i+BATCH_SIZE]
        print(f"Processing batch {i//BATCH_SIZE + 1} ({len(batch)} docs)...")
        
        tasks = [process_document(doc, sem) for doc in batch]
        results = await asyncio.gather(*tasks)
        
        for result in results:
            if not result:
                continue
                
            # Write vertices
            for v in result.get("vertices", []):
                # Basic cleaning
                v_id = v.get("id", "").strip().replace("\n", " ")
                v_type = v.get("type", "Unknown")
                if v_id:
                    v_writer.writerow([v_id, v_type])
                    total_vertices += 1
                    
            # Write edges
            for e in result.get("edges", []):
                source = e.get("source", "").strip().replace("\n", " ")
                target = e.get("target", "").strip().replace("\n", " ")
                e_type = e.get("type", "Unknown")
                if source and target:
                    e_writer.writerow([source, target, e_type])
                    total_edges += 1
                    
        # Flush to disk periodically
        vertices_file.flush()
        edges_file.flush()
        
        # Sleep slightly between batches to cool down rate limits
        await asyncio.sleep(2)
        
    vertices_file.close()
    edges_file.close()
    
    print("\n--- GRAPH EXTRACTION COMPLETE ---")
    print(f"Total Vertices Extracted: {total_vertices:,}")
    print(f"Total Edges Extracted: {total_edges:,}")
    print("Saved to vertices.csv and edges.csv")
    print("Ready for TigerGraph Bulk Loading!")

if __name__ == "__main__":
    asyncio.run(main())