utkarsh134 commited on
Commit
a311739
·
verified ·
1 Parent(s): b3b99d1

Update parsing.py

Browse files
Files changed (1) hide show
  1. parsing.py +205 -142
parsing.py CHANGED
@@ -1,142 +1,205 @@
1
- from langchain_core.prompts import PromptTemplate
2
- from langchain_google_genai import ChatGoogleGenerativeAI
3
- import os
4
- import json
5
- import re
6
- from dotenv import load_dotenv
7
- load_dotenv()
8
-
9
- def get_llm():
10
- """Create and return a ChatGoogleGenerativeAI instance with proper authentication."""
11
- google_api_key = os.getenv("GOOGLE_API_KEY")
12
- if not google_api_key:
13
- raise ValueError("GOOGLE_API_KEY environment variable is not set")
14
- return ChatGoogleGenerativeAI(
15
- model="gemini-2.0-flash",
16
- temperature=0,
17
- google_api_key=google_api_key
18
- )
19
-
20
- def extract_json_from_text(text):
21
- """Extract JSON from text that might contain explanation text or code blocks."""
22
- # Look for text between curly braces
23
- json_pattern = r'\{.*\}'
24
- match = re.search(json_pattern, text, re.DOTALL)
25
-
26
- if match:
27
- potential_json = match.group(0)
28
- try:
29
- return json.loads(potential_json)
30
- except json.JSONDecodeError:
31
- pass
32
-
33
- # Try to find JSON in markdown code blocks
34
- code_block_pattern = r'```(?:json)?\s*([\s\S]*?)\s*```'
35
- matches = re.findall(code_block_pattern, text, re.DOTALL)
36
-
37
- for potential_json in matches:
38
- try:
39
- # Remove any triple backticks that might be included
40
- cleaned = potential_json.strip().replace('```', '')
41
- if cleaned.startswith('{') and cleaned.endswith('}'):
42
- return json.loads(cleaned)
43
- except json.JSONDecodeError:
44
- continue
45
-
46
- return None
47
-
48
- def parse_data_with_llm(extracted_text):
49
- """Extract structured JSON from extracted text but do calculations in Python."""
50
- llm = get_llm()
51
-
52
- prompt_template = PromptTemplate(
53
- input_variables=["extracted_data"],
54
- template= """
55
- You are a data extraction assistant. Given the following extracted text, structure the output as valid JSON.
56
- **Do NOT perform any calculations.** Just extract the numeric values exactly as they appear.
57
-
58
- **Extracted data:**
59
- {extracted_data}
60
-
61
- ### **Extract the Following Fields (Raw Data Only)**:
62
- - **Customer Ref**: Extract "Customer PO".
63
- - **Wt. in gram 14K**: Extract all "14K WG" weight values of parts and return as a list.
64
- - **Wt. in gram 18K**: Extract all "18K WG" weight values of parts and return as a list.
65
- - **Wt. in gram PLT**: Extract all "PLT" weight values of parts and return as a list.
66
-
67
- **Output STRICTLY as JSON (No markdown, no explanations, no formatting issues):**
68
- ```json
69
- {{
70
- "Customer Ref": "...",
71
- "Wt. in gram 14K": [...],
72
- "Wt. in gram 18K": [...],
73
- "Wt. in gram PLT": [...]
74
- }}
75
- ```
76
- """
77
- )
78
-
79
- prompt = prompt_template.format(extracted_data=extracted_text)
80
- response = llm.invoke(prompt)
81
-
82
- # Extract valid JSON from response
83
- response_text = response.content.strip() if hasattr(response, 'content') else str(response).strip()
84
- json_data = extract_json_from_text(response_text)
85
-
86
- if not json_data:
87
- cleaned_response = response_text.replace("```json", "").replace("```", "").strip()
88
- try:
89
- json_data = json.loads(cleaned_response)
90
- except json.JSONDecodeError as e:
91
- raise ValueError(f"LLM response is not in valid JSON format: {str(e)}")
92
-
93
- return json_data
94
-
95
-
96
- def compute_values(data):
97
- """Perform gold and platinum calculations based on extracted values."""
98
- gold_lock = 2800
99
- platinum_lock = 1040
100
-
101
- # Convert extracted weights to float before summing
102
- wt_14k = sum([float(x) for x in data.get("Wt. in gram 14K", []) if x])
103
- wt_18k = sum([float(x) for x in data.get("Wt. in gram 18K", []) if x])
104
- wt_plt = sum([float(x) for x in data.get("Wt. in gram PLT", []) if x])
105
-
106
- # Gold & Platinum Rate Calculations
107
- gold_rate_14k = (gold_lock / 31.1035) * 0.585 * 1.1
108
- gold_rate_18k = (gold_lock / 31.1035) * 0.75 * 1.1
109
- plat_rate = (platinum_lock / 31.1035) * 0.95 * 1.2
110
-
111
- # Final Computed Values
112
- gold_value_14k = wt_14k * gold_rate_14k
113
- gold_value_18k = wt_18k * gold_rate_18k
114
- plat_value = wt_plt * plat_rate
115
-
116
- # Return final JSON
117
- return {
118
- "Customer Ref": data.get("Customer Ref", ""),
119
- "Gold Lock @ $ 2800": gold_lock,
120
- "Platinum Lock $ 1040": platinum_lock,
121
- "Wt. in gram 14K": wt_14k,
122
- "Wt. in gram 18K": wt_18k,
123
- "Wt. in gram PLT": wt_plt,
124
- "Gold Rate (including loss) 14KT": gold_rate_14k,
125
- "Gold Rate (including loss) 18KT": gold_rate_18k,
126
- "Plat rate (including Lab)": plat_rate,
127
- "Gold Value 14K": gold_value_14k,
128
- "Gold Value 18K": gold_value_18k,
129
- "Plat Value": plat_value
130
- }
131
-
132
-
133
-
134
-
135
- def process_extracted_text(extracted_text):
136
- # Step 1: Get extracted numbers from LLM
137
- extracted_data = parse_data_with_llm(extracted_text)
138
-
139
- # Step 2: Perform calculations in Python
140
- final_data = compute_values(extracted_data)
141
-
142
- return final_data
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from langchain_core.prompts import PromptTemplate
2
+ from langchain_google_genai import ChatGoogleGenerativeAI
3
+ import os
4
+ import json
5
+ import re
6
+ from dotenv import load_dotenv
7
+ import requests
8
+ import time
9
+ load_dotenv()
10
+
11
+ def get_llm():
12
+ """Create and return a ChatGoogleGenerativeAI instance with proper authentication."""
13
+ google_api_key = os.getenv("GOOGLE_API_KEY")
14
+ if not google_api_key:
15
+ raise ValueError("GOOGLE_API_KEY environment variable is not set")
16
+ return ChatGoogleGenerativeAI(
17
+ model="gemini-2.0-flash",
18
+ temperature=0,
19
+ google_api_key=google_api_key
20
+ )
21
+
22
+ def extract_json_from_text(text):
23
+ """Extract JSON from text that might contain explanation text or code blocks."""
24
+ # Look for text between curly braces
25
+ json_pattern = r'\{.*\}'
26
+ match = re.search(json_pattern, text, re.DOTALL)
27
+
28
+ if match:
29
+ potential_json = match.group(0)
30
+ try:
31
+ return json.loads(potential_json)
32
+ except json.JSONDecodeError:
33
+ pass
34
+
35
+ # Try to find JSON in markdown code blocks
36
+ code_block_pattern = r'```(?:json)?\s*([\s\S]*?)\s*```'
37
+ matches = re.findall(code_block_pattern, text, re.DOTALL)
38
+
39
+ for potential_json in matches:
40
+ try:
41
+ # Remove any triple backticks that might be included
42
+ cleaned = potential_json.strip().replace('```', '')
43
+ if cleaned.startswith('{') and cleaned.endswith('}'):
44
+ return json.loads(cleaned)
45
+ except json.JSONDecodeError:
46
+ continue
47
+
48
+ return None
49
+
50
+ def parse_data_with_llm(extracted_text):
51
+ """Extract structured JSON from extracted text but do calculations in Python."""
52
+ llm = get_llm()
53
+
54
+ prompt_template = PromptTemplate(
55
+ input_variables=["extracted_data"],
56
+ template= """
57
+ You are a data extraction assistant. Given the following extracted text, structure the output as valid JSON.
58
+ **Do NOT perform any calculations.** Just extract the numeric values exactly as they appear.
59
+
60
+ **Extracted data:**
61
+ {extracted_data}
62
+
63
+ ### **Extract the Following Fields (Raw Data Only)**:
64
+ - **Customer Ref**: Extract "Customer PO".
65
+ - **Wt. in gram 14K**: Extract all "14K WG" weight values of parts and return as a list.
66
+ - **Wt. in gram 18K**: Extract all "18K WG" weight values of parts and return as a list.
67
+ - **Wt. in gram PLT**: Extract all "PLT" weight values of parts and return as a list.
68
+ - **Diamonds**: Extract each diamond entry as an object with "Dia Qlty"(is the name under the shape column under diamond details), "MM Size", "Dia Qty", and "Dia Wt.".
69
+
70
+ **Output STRICTLY as JSON:**
71
+ ```json
72
+ {{
73
+ "Customer Ref": "...",
74
+ "Wt. in gram 14K": ...,
75
+ "Wt. in gram 18K": ...,
76
+ "Wt. in gram PLT": ...,
77
+ "Diamonds": [
78
+ {{
79
+ "Dia Qlty": "...",
80
+ "MM Size": "...",
81
+ "Dia Qty": ...,
82
+ "Dia Wt.": ...
83
+ }},
84
+ ...
85
+ ]
86
+ }}
87
+ ```
88
+ """
89
+ )
90
+
91
+ prompt = prompt_template.format(extracted_data=extracted_text)
92
+ response = llm.invoke(prompt)
93
+
94
+ # Extract valid JSON from response
95
+ response_text = response.content.strip() if hasattr(response, 'content') else str(response).strip()
96
+ json_data = extract_json_from_text(response_text)
97
+
98
+ if not json_data:
99
+ cleaned_response = response_text.replace("```json", "").replace("```", "").strip()
100
+ try:
101
+ json_data = json.loads(cleaned_response)
102
+ except json.JSONDecodeError as e:
103
+ raise ValueError(f"LLM response is not in valid JSON format: {str(e)}")
104
+
105
+ return json_data
106
+
107
+
108
+ def compute_values(data):
109
+ """Perform metal calculations and format diamonds separately."""
110
+ # Fetch current prices
111
+ gold_price, platinum_price = fetch_metal_prices()
112
+
113
+ # Calculate prices for different karats
114
+ price_14k = (14 / 24) * gold_price
115
+ price_18k = (18 / 24) * gold_price
116
+ price_pt = platinum_price
117
+
118
+ # Rest of the existing calculations
119
+ wt_14k = sum(float(x) for x in data.get("Wt. in gram 14K", [])) if isinstance(data.get("Wt. in gram 14K"), list) else float(data.get("Wt. in gram 14K", 0))
120
+ wt_18k = sum(float(x) for x in data.get("Wt. in gram 18K", [])) if isinstance(data.get("Wt. in gram 18K"), list) else float(data.get("Wt. in gram 18K", 0))
121
+ wt_plt = sum(float(x) for x in data.get("Wt. in gram PLT", [])) if isinstance(data.get("Wt. in gram PLT"), list) else float(data.get("Wt. in gram PLT", 0))
122
+
123
+ gold_rate_14k = (gold_price / 31.1035) * 0.585 * 1.1
124
+ gold_rate_18k = (gold_price / 31.1035) * 0.75 * 1.1
125
+ plat_rate = (platinum_price / 31.1035) * 0.95 * 1.2
126
+
127
+ gold_value_14k = wt_14k * gold_rate_14k
128
+ gold_value_18k = wt_18k * gold_rate_18k
129
+ plat_value = wt_plt * plat_rate
130
+
131
+ metal_data = {
132
+ "Customer Ref": data.get("Customer Ref", ""),
133
+ "Gold Lock @ $ 2800": gold_price, # Updated to use current price
134
+ "Platinum Lock $ 1040": platinum_price, # Updated to use current price
135
+ "Wt. in gram 14K": wt_14k,
136
+ "Wt. in gram 18K": wt_18k,
137
+ "Wt. in gram PLT": wt_plt,
138
+ "Gold Rate (including loss) 14KT": gold_rate_14k,
139
+ "Gold Rate (including loss) 18KT": gold_rate_18k,
140
+ "Plat rate (including Lab)": plat_rate,
141
+ "Gold Value 14K": gold_value_14k,
142
+ "Gold Value 18K": gold_value_18k,
143
+ "Plat Value": plat_value,
144
+ "Price of USD 14K": price_14k,
145
+ "Price of USD 18K": price_18k,
146
+ "Price of USD PT": price_pt
147
+ }
148
+
149
+ # Process diamonds data
150
+ diamonds = data.get("Diamonds", [])
151
+ for diamond in diamonds:
152
+ if "Dia Qty" in diamond and "Dia Wt." in diamond:
153
+ try:
154
+ diamond["Diamond Weight"] = float(diamond["Dia Qty"]) * float(diamond["Dia Wt."])
155
+ except (ValueError, TypeError):
156
+ diamond["Diamond Weight"] = 0
157
+
158
+ return metal_data, diamonds
159
+
160
+ def process_extracted_text(extracted_text):
161
+ extracted_data = parse_data_with_llm(extracted_text)
162
+ metal_data, diamonds = compute_values(extracted_data)
163
+
164
+ return {"Metal Details": metal_data, "Diamonds": diamonds}
165
+
166
+ def fetch_metal_prices():
167
+ """Fetch current gold and platinum prices from Kitco API."""
168
+ url = "https://kdb-gw.prod.kitco.com/"
169
+ current_timestamp = int(time.time())
170
+
171
+ payload = {
172
+ "query": """fragment MetalFragment on Metal{
173
+ ID symbol currency name results{...MetalQuoteFragment}
174
+ }
175
+ fragment MetalQuoteFragment on Quote{
176
+ ID ask bid change changePercentage close high low mid open originalTime timestamp unit
177
+ }
178
+ query GoldSilverPlatinumPalladium($currency:String!, $timestamp:Int){
179
+ gold:GetMetalQuoteV3(symbol:"AU" timestamp:$timestamp currency:$currency){...MetalFragment}
180
+ platinum:GetMetalQuoteV3(symbol:"PT" timestamp:$timestamp currency:$currency){...MetalFragment}
181
+ }""",
182
+ "variables": {
183
+ "timestamp": current_timestamp,
184
+ "currency": "USD"
185
+ },
186
+ "operationName": "GoldSilverPlatinumPalladium"
187
+ }
188
+
189
+ headers = {"Content-Type": "application/json"}
190
+
191
+ try:
192
+ response = requests.post(url, json=payload, headers=headers)
193
+ response.raise_for_status()
194
+ data = response.json()
195
+
196
+ gold_price = float(data["data"]["gold"]["results"][0]["bid"])
197
+ platinum_price = float(data["data"]["platinum"]["results"][0]["bid"])
198
+
199
+ return gold_price, platinum_price
200
+ except (requests.exceptions.RequestException, KeyError, ValueError) as e:
201
+ print(f"Error fetching metal prices: {e}")
202
+ return 0, 0 # Default values as backup
203
+
204
+
205
+