Spaces:
Sleeping
Sleeping
File size: 4,952 Bytes
e699b35 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 | import os
import json
import time
from datasets import load_dataset
from google import genai
def main():
if "GEMINI_API_KEY" not in os.environ:
print("ERROR: GEMINI_API_KEY environment variable not set.")
print("Please set it before running this script.")
return
client = genai.Client()
TARGET_TOKENS = 105_000_000
current_tokens = 0
document_count = 0
output_file = "financial_corpus.jsonl"
print(f"Starting token counting using Gemini 'countTokens' API...")
print(f"Targeting {TARGET_TOKENS:,} tokens.")
# Resume Logic
if os.path.exists(output_file):
print(f"Found existing {output_file}. Resuming...")
with open(output_file, 'r', encoding='utf-8') as f:
for line in f:
if not line.strip(): continue
data = json.loads(line)
current_tokens += data.get("tokens", 0)
document_count += 1
print(f"Resumed from {document_count} documents with {current_tokens:,} tokens.")
if current_tokens >= TARGET_TOKENS:
print("Target already reached!")
return
start_time = time.time()
splits = ['001', '002', '003', '004', '005']
docs_to_skip = document_count
with open(output_file, 'a', encoding='utf-8') as f:
for split_id in splits:
if current_tokens >= TARGET_TOKENS:
break
print(f"Loading split {split_id}...")
try:
dataset = load_dataset('winterForestStump/10-K_sec_filings', split=split_id, streaming=True)
except Exception as e:
print(f"Error loading dataset split {split_id}: {e}")
continue
for row in dataset:
# Fast forward
if docs_to_skip > 0:
docs_to_skip -= 1
continue
if current_tokens >= TARGET_TOKENS:
break
company = row.get("company_name", "Unknown Company")
year = str(row.get("filing_date", "Unknown Year"))[:4] if row.get("filing_date") else "Unknown Year"
text_sections = []
if row.get("Business"): text_sections.append(str(row["Business"]))
if row.get("Risk Factors"): text_sections.append(str(row["Risk Factors"]))
for k, v in row.items():
if "Management" in k and "Discussion" in k and v:
text_sections.append(str(v))
combined_text = "\n\n".join(text_sections).strip()
if not combined_text:
continue
try:
response = client.models.count_tokens(
model='gemini-2.5-flash',
contents=combined_text
)
tokens = response.total_tokens
corpus_item = {
"company": company,
"year": year,
"text": combined_text,
"tokens": tokens
}
f.write(json.dumps(corpus_item) + "\n")
f.flush()
current_tokens += tokens
document_count += 1
if document_count % 10 == 0:
print(f"Processed {document_count} documents. Total Tokens: {current_tokens:,}")
except Exception as e:
print(f"API Error counting tokens for {company}: {e}")
time.sleep(1) # Backoff
elapsed = time.time() - start_time
print("\n--- TOKEN COUNTING COMPLETE ---")
print(f"Total Tokens: {current_tokens:,}")
print(f"Total Documents: {document_count}")
print(f"Time Taken for this run: {elapsed:.2f} seconds")
print(f"Corpus saved to: {output_file}")
report = f"""TigerGraph Hackathon Benchmark Report
-----------------------------------
Dataset: winterForestStump/10-K_sec_filings (SEC 10-K Filings)
Total Tokens: {current_tokens:,}
Total Documents: {document_count}
Tokenizer Documentation:
As per the hackathon requirements, the dataset size was measured using the official Gemini API (`countTokens` endpoint) via the `google-genai` Python SDK.
The underlying tokenizer utilized by the API is the official Gemini SentencePiece tokenizer, ensuring exact alignment with the model's actual token usage for processing.
"""
with open("benchmark_report.txt", "w") as f:
f.write(report)
print("benchmark_report.txt generated successfully!")
if __name__ == "__main__":
main()
|