Spaces:
Sleeping
Sleeping
File size: 5,390 Bytes
e699b35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 | import os
import json
import csv
import time
import asyncio
from pydantic import BaseModel, Field
from google import genai
from google.genai import types
# Define the structured output schema for Gemini
class Vertex(BaseModel):
id: str = Field(description="The unique identifier or name of the entity.")
type: str = Field(description="Must be one of: Company, Executive, RiskFactor, Subsidiary")
class Edge(BaseModel):
source: str = Field(description="The id of the source vertex.")
target: str = Field(description="The id of the target vertex.")
type: str = Field(description="Must be one of: EMPLOYS, FACES_RISK, OWNS, COMPETES_WITH")
class GraphExtraction(BaseModel):
vertices: list[Vertex]
edges: list[Edge]
# Setup Gemini Client (Async)
if "GEMINI_API_KEY" not in os.environ:
print("ERROR: GEMINI_API_KEY environment variable not set.")
exit(1)
client = genai.Client()
async def process_document(doc, sem):
"""
Process a single document through Gemini 1.5 Flash to extract graph entities.
Uses a semaphore to limit concurrent API calls and prevent 429 errors.
"""
async with sem:
text = doc.get("text", "")
company = doc.get("company", "Unknown")
# If the text is too large, we truncate it for extraction to save costs
# (For 100M tokens, we might process full texts, but let's limit to 15k chars per doc for the demo)
text_chunk = text[:15000]
prompt = f"""
Analyze the following SEC 10-K filing excerpt for {company}.
Extract all relevant entities (Company, Executive, RiskFactor, Subsidiary) and their relationships (EMPLOYS, FACES_RISK, OWNS, COMPETES_WITH).
Text excerpt:
{text_chunk}
"""
try:
# Note: We use the async client `client.aio`
response = await client.aio.models.generate_content(
model='gemini-2.5-flash',
contents=prompt,
config=types.GenerateContentConfig(
response_mime_type="application/json",
response_schema=GraphExtraction,
temperature=0.1
)
)
# The response text will be a JSON string conforming to the GraphExtraction schema
return json.loads(response.text)
except Exception as e:
print(f"Error extracting graph for {company}: {e}")
return None
async def main():
input_file = "financial_corpus.jsonl"
if not os.path.exists(input_file):
print(f"ERROR: {input_file} not found. Run count_tokens.py first.")
return
# Prepare CSV writers
vertices_file = open("vertices.csv", "w", newline="", encoding="utf-8")
edges_file = open("edges.csv", "w", newline="", encoding="utf-8")
v_writer = csv.writer(vertices_file)
e_writer = csv.writer(edges_file)
# Write headers
v_writer.writerow(["v_id", "v_type"])
e_writer.writerow(["source", "target", "e_type"])
# Read corpus
print("Loading corpus...")
docs = []
with open(input_file, 'r', encoding='utf-8') as f:
for line in f:
docs.append(json.loads(line))
if len(docs) >= 50:
break
print(f"Loaded {len(docs)} documents for extraction.")
# We use a Semaphore to allow max 5 concurrent requests to Gemini to respect standard tier rate limits
sem = asyncio.Semaphore(5)
# Batch processing
BATCH_SIZE = 50
total_vertices = 0
total_edges = 0
for i in range(0, len(docs), BATCH_SIZE):
batch = docs[i:i+BATCH_SIZE]
print(f"Processing batch {i//BATCH_SIZE + 1} ({len(batch)} docs)...")
tasks = [process_document(doc, sem) for doc in batch]
results = await asyncio.gather(*tasks)
for result in results:
if not result:
continue
# Write vertices
for v in result.get("vertices", []):
# Basic cleaning
v_id = v.get("id", "").strip().replace("\n", " ")
v_type = v.get("type", "Unknown")
if v_id:
v_writer.writerow([v_id, v_type])
total_vertices += 1
# Write edges
for e in result.get("edges", []):
source = e.get("source", "").strip().replace("\n", " ")
target = e.get("target", "").strip().replace("\n", " ")
e_type = e.get("type", "Unknown")
if source and target:
e_writer.writerow([source, target, e_type])
total_edges += 1
# Flush to disk periodically
vertices_file.flush()
edges_file.flush()
# Sleep slightly between batches to cool down rate limits
await asyncio.sleep(2)
vertices_file.close()
edges_file.close()
print("\n--- GRAPH EXTRACTION COMPLETE ---")
print(f"Total Vertices Extracted: {total_vertices:,}")
print(f"Total Edges Extracted: {total_edges:,}")
print("Saved to vertices.csv and edges.csv")
print("Ready for TigerGraph Bulk Loading!")
if __name__ == "__main__":
asyncio.run(main())
|