Spaces:
Sleeping
Sleeping
Update parsing.py
Browse files- parsing.py +205 -142
parsing.py
CHANGED
|
@@ -1,142 +1,205 @@
|
|
| 1 |
-
from langchain_core.prompts import PromptTemplate
|
| 2 |
-
from langchain_google_genai import ChatGoogleGenerativeAI
|
| 3 |
-
import os
|
| 4 |
-
import json
|
| 5 |
-
import re
|
| 6 |
-
from dotenv import load_dotenv
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
| 19 |
-
|
| 20 |
-
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
| 32 |
-
|
| 33 |
-
|
| 34 |
-
|
| 35 |
-
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
**
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
- **
|
| 65 |
-
- **Wt. in gram
|
| 66 |
-
|
| 67 |
-
**
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
"
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
|
| 103 |
-
|
| 104 |
-
|
| 105 |
-
|
| 106 |
-
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
|
| 110 |
-
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
|
| 130 |
-
|
| 131 |
-
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from langchain_core.prompts import PromptTemplate
|
| 2 |
+
from langchain_google_genai import ChatGoogleGenerativeAI
|
| 3 |
+
import os
|
| 4 |
+
import json
|
| 5 |
+
import re
|
| 6 |
+
from dotenv import load_dotenv
|
| 7 |
+
import requests
|
| 8 |
+
import time
|
| 9 |
+
load_dotenv()
|
| 10 |
+
|
| 11 |
+
def get_llm():
|
| 12 |
+
"""Create and return a ChatGoogleGenerativeAI instance with proper authentication."""
|
| 13 |
+
google_api_key = os.getenv("GOOGLE_API_KEY")
|
| 14 |
+
if not google_api_key:
|
| 15 |
+
raise ValueError("GOOGLE_API_KEY environment variable is not set")
|
| 16 |
+
return ChatGoogleGenerativeAI(
|
| 17 |
+
model="gemini-2.0-flash",
|
| 18 |
+
temperature=0,
|
| 19 |
+
google_api_key=google_api_key
|
| 20 |
+
)
|
| 21 |
+
|
| 22 |
+
def extract_json_from_text(text):
|
| 23 |
+
"""Extract JSON from text that might contain explanation text or code blocks."""
|
| 24 |
+
# Look for text between curly braces
|
| 25 |
+
json_pattern = r'\{.*\}'
|
| 26 |
+
match = re.search(json_pattern, text, re.DOTALL)
|
| 27 |
+
|
| 28 |
+
if match:
|
| 29 |
+
potential_json = match.group(0)
|
| 30 |
+
try:
|
| 31 |
+
return json.loads(potential_json)
|
| 32 |
+
except json.JSONDecodeError:
|
| 33 |
+
pass
|
| 34 |
+
|
| 35 |
+
# Try to find JSON in markdown code blocks
|
| 36 |
+
code_block_pattern = r'```(?:json)?\s*([\s\S]*?)\s*```'
|
| 37 |
+
matches = re.findall(code_block_pattern, text, re.DOTALL)
|
| 38 |
+
|
| 39 |
+
for potential_json in matches:
|
| 40 |
+
try:
|
| 41 |
+
# Remove any triple backticks that might be included
|
| 42 |
+
cleaned = potential_json.strip().replace('```', '')
|
| 43 |
+
if cleaned.startswith('{') and cleaned.endswith('}'):
|
| 44 |
+
return json.loads(cleaned)
|
| 45 |
+
except json.JSONDecodeError:
|
| 46 |
+
continue
|
| 47 |
+
|
| 48 |
+
return None
|
| 49 |
+
|
| 50 |
+
def parse_data_with_llm(extracted_text):
|
| 51 |
+
"""Extract structured JSON from extracted text but do calculations in Python."""
|
| 52 |
+
llm = get_llm()
|
| 53 |
+
|
| 54 |
+
prompt_template = PromptTemplate(
|
| 55 |
+
input_variables=["extracted_data"],
|
| 56 |
+
template= """
|
| 57 |
+
You are a data extraction assistant. Given the following extracted text, structure the output as valid JSON.
|
| 58 |
+
**Do NOT perform any calculations.** Just extract the numeric values exactly as they appear.
|
| 59 |
+
|
| 60 |
+
**Extracted data:**
|
| 61 |
+
{extracted_data}
|
| 62 |
+
|
| 63 |
+
### **Extract the Following Fields (Raw Data Only)**:
|
| 64 |
+
- **Customer Ref**: Extract "Customer PO".
|
| 65 |
+
- **Wt. in gram 14K**: Extract all "14K WG" weight values of parts and return as a list.
|
| 66 |
+
- **Wt. in gram 18K**: Extract all "18K WG" weight values of parts and return as a list.
|
| 67 |
+
- **Wt. in gram PLT**: Extract all "PLT" weight values of parts and return as a list.
|
| 68 |
+
- **Diamonds**: Extract each diamond entry as an object with "Dia Qlty"(is the name under the shape column under diamond details), "MM Size", "Dia Qty", and "Dia Wt.".
|
| 69 |
+
|
| 70 |
+
**Output STRICTLY as JSON:**
|
| 71 |
+
```json
|
| 72 |
+
{{
|
| 73 |
+
"Customer Ref": "...",
|
| 74 |
+
"Wt. in gram 14K": ...,
|
| 75 |
+
"Wt. in gram 18K": ...,
|
| 76 |
+
"Wt. in gram PLT": ...,
|
| 77 |
+
"Diamonds": [
|
| 78 |
+
{{
|
| 79 |
+
"Dia Qlty": "...",
|
| 80 |
+
"MM Size": "...",
|
| 81 |
+
"Dia Qty": ...,
|
| 82 |
+
"Dia Wt.": ...
|
| 83 |
+
}},
|
| 84 |
+
...
|
| 85 |
+
]
|
| 86 |
+
}}
|
| 87 |
+
```
|
| 88 |
+
"""
|
| 89 |
+
)
|
| 90 |
+
|
| 91 |
+
prompt = prompt_template.format(extracted_data=extracted_text)
|
| 92 |
+
response = llm.invoke(prompt)
|
| 93 |
+
|
| 94 |
+
# Extract valid JSON from response
|
| 95 |
+
response_text = response.content.strip() if hasattr(response, 'content') else str(response).strip()
|
| 96 |
+
json_data = extract_json_from_text(response_text)
|
| 97 |
+
|
| 98 |
+
if not json_data:
|
| 99 |
+
cleaned_response = response_text.replace("```json", "").replace("```", "").strip()
|
| 100 |
+
try:
|
| 101 |
+
json_data = json.loads(cleaned_response)
|
| 102 |
+
except json.JSONDecodeError as e:
|
| 103 |
+
raise ValueError(f"LLM response is not in valid JSON format: {str(e)}")
|
| 104 |
+
|
| 105 |
+
return json_data
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
def compute_values(data):
|
| 109 |
+
"""Perform metal calculations and format diamonds separately."""
|
| 110 |
+
# Fetch current prices
|
| 111 |
+
gold_price, platinum_price = fetch_metal_prices()
|
| 112 |
+
|
| 113 |
+
# Calculate prices for different karats
|
| 114 |
+
price_14k = (14 / 24) * gold_price
|
| 115 |
+
price_18k = (18 / 24) * gold_price
|
| 116 |
+
price_pt = platinum_price
|
| 117 |
+
|
| 118 |
+
# Rest of the existing calculations
|
| 119 |
+
wt_14k = sum(float(x) for x in data.get("Wt. in gram 14K", [])) if isinstance(data.get("Wt. in gram 14K"), list) else float(data.get("Wt. in gram 14K", 0))
|
| 120 |
+
wt_18k = sum(float(x) for x in data.get("Wt. in gram 18K", [])) if isinstance(data.get("Wt. in gram 18K"), list) else float(data.get("Wt. in gram 18K", 0))
|
| 121 |
+
wt_plt = sum(float(x) for x in data.get("Wt. in gram PLT", [])) if isinstance(data.get("Wt. in gram PLT"), list) else float(data.get("Wt. in gram PLT", 0))
|
| 122 |
+
|
| 123 |
+
gold_rate_14k = (gold_price / 31.1035) * 0.585 * 1.1
|
| 124 |
+
gold_rate_18k = (gold_price / 31.1035) * 0.75 * 1.1
|
| 125 |
+
plat_rate = (platinum_price / 31.1035) * 0.95 * 1.2
|
| 126 |
+
|
| 127 |
+
gold_value_14k = wt_14k * gold_rate_14k
|
| 128 |
+
gold_value_18k = wt_18k * gold_rate_18k
|
| 129 |
+
plat_value = wt_plt * plat_rate
|
| 130 |
+
|
| 131 |
+
metal_data = {
|
| 132 |
+
"Customer Ref": data.get("Customer Ref", ""),
|
| 133 |
+
"Gold Lock @ $ 2800": gold_price, # Updated to use current price
|
| 134 |
+
"Platinum Lock $ 1040": platinum_price, # Updated to use current price
|
| 135 |
+
"Wt. in gram 14K": wt_14k,
|
| 136 |
+
"Wt. in gram 18K": wt_18k,
|
| 137 |
+
"Wt. in gram PLT": wt_plt,
|
| 138 |
+
"Gold Rate (including loss) 14KT": gold_rate_14k,
|
| 139 |
+
"Gold Rate (including loss) 18KT": gold_rate_18k,
|
| 140 |
+
"Plat rate (including Lab)": plat_rate,
|
| 141 |
+
"Gold Value 14K": gold_value_14k,
|
| 142 |
+
"Gold Value 18K": gold_value_18k,
|
| 143 |
+
"Plat Value": plat_value,
|
| 144 |
+
"Price of USD 14K": price_14k,
|
| 145 |
+
"Price of USD 18K": price_18k,
|
| 146 |
+
"Price of USD PT": price_pt
|
| 147 |
+
}
|
| 148 |
+
|
| 149 |
+
# Process diamonds data
|
| 150 |
+
diamonds = data.get("Diamonds", [])
|
| 151 |
+
for diamond in diamonds:
|
| 152 |
+
if "Dia Qty" in diamond and "Dia Wt." in diamond:
|
| 153 |
+
try:
|
| 154 |
+
diamond["Diamond Weight"] = float(diamond["Dia Qty"]) * float(diamond["Dia Wt."])
|
| 155 |
+
except (ValueError, TypeError):
|
| 156 |
+
diamond["Diamond Weight"] = 0
|
| 157 |
+
|
| 158 |
+
return metal_data, diamonds
|
| 159 |
+
|
| 160 |
+
def process_extracted_text(extracted_text):
|
| 161 |
+
extracted_data = parse_data_with_llm(extracted_text)
|
| 162 |
+
metal_data, diamonds = compute_values(extracted_data)
|
| 163 |
+
|
| 164 |
+
return {"Metal Details": metal_data, "Diamonds": diamonds}
|
| 165 |
+
|
| 166 |
+
def fetch_metal_prices():
|
| 167 |
+
"""Fetch current gold and platinum prices from Kitco API."""
|
| 168 |
+
url = "https://kdb-gw.prod.kitco.com/"
|
| 169 |
+
current_timestamp = int(time.time())
|
| 170 |
+
|
| 171 |
+
payload = {
|
| 172 |
+
"query": """fragment MetalFragment on Metal{
|
| 173 |
+
ID symbol currency name results{...MetalQuoteFragment}
|
| 174 |
+
}
|
| 175 |
+
fragment MetalQuoteFragment on Quote{
|
| 176 |
+
ID ask bid change changePercentage close high low mid open originalTime timestamp unit
|
| 177 |
+
}
|
| 178 |
+
query GoldSilverPlatinumPalladium($currency:String!, $timestamp:Int){
|
| 179 |
+
gold:GetMetalQuoteV3(symbol:"AU" timestamp:$timestamp currency:$currency){...MetalFragment}
|
| 180 |
+
platinum:GetMetalQuoteV3(symbol:"PT" timestamp:$timestamp currency:$currency){...MetalFragment}
|
| 181 |
+
}""",
|
| 182 |
+
"variables": {
|
| 183 |
+
"timestamp": current_timestamp,
|
| 184 |
+
"currency": "USD"
|
| 185 |
+
},
|
| 186 |
+
"operationName": "GoldSilverPlatinumPalladium"
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
headers = {"Content-Type": "application/json"}
|
| 190 |
+
|
| 191 |
+
try:
|
| 192 |
+
response = requests.post(url, json=payload, headers=headers)
|
| 193 |
+
response.raise_for_status()
|
| 194 |
+
data = response.json()
|
| 195 |
+
|
| 196 |
+
gold_price = float(data["data"]["gold"]["results"][0]["bid"])
|
| 197 |
+
platinum_price = float(data["data"]["platinum"]["results"][0]["bid"])
|
| 198 |
+
|
| 199 |
+
return gold_price, platinum_price
|
| 200 |
+
except (requests.exceptions.RequestException, KeyError, ValueError) as e:
|
| 201 |
+
print(f"Error fetching metal prices: {e}")
|
| 202 |
+
return 0, 0 # Default values as backup
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
|