utkarsh134 commited on
Commit
a66ab92
·
verified ·
1 Parent(s): 4c1ad4b

Update parsing.py

Browse files
Files changed (1) hide show
  1. parsing.py +212 -54
parsing.py CHANGED
@@ -1,57 +1,60 @@
1
- from langchain_core.prompts import PromptTemplate
2
- from langchain_google_genai import ChatGoogleGenerativeAI
3
- import os
4
  import json
5
- import re
6
- from dotenv import load_dotenv
7
- import requests
8
  import math
 
 
9
  from datetime import datetime
 
 
 
 
 
 
10
  load_dotenv()
11
 
 
12
  def get_llm():
13
  """Create and return a ChatGoogleGenerativeAI instance with proper authentication."""
14
  google_api_key = os.getenv("GOOGLE_API_KEY")
15
  if not google_api_key:
16
  raise ValueError("GOOGLE_API_KEY environment variable is not set")
17
  return ChatGoogleGenerativeAI(
18
- model="gemini-2.0-flash",
19
- temperature=0,
20
- google_api_key=google_api_key
21
  )
22
 
 
23
  def extract_json_from_text(text):
24
  """Extract JSON from text that might contain explanation text or code blocks."""
25
- json_pattern = r'\{.*\}'
26
  match = re.search(json_pattern, text, re.DOTALL)
27
-
28
  if match:
29
  potential_json = match.group(0)
30
  try:
31
  return json.loads(potential_json)
32
  except json.JSONDecodeError:
33
  pass
34
-
35
- code_block_pattern = r'```(?:json)?\s*([\s\S]*?)\s*```'
36
  matches = re.findall(code_block_pattern, text, re.DOTALL)
37
-
38
  for potential_json in matches:
39
  try:
40
- cleaned = potential_json.strip().replace('```', '')
41
- if cleaned.startswith('{') and cleaned.endswith('}'):
42
  return json.loads(cleaned)
43
  except json.JSONDecodeError:
44
  continue
45
-
46
  return None
47
 
 
48
  def parse_data_with_llm(extracted_text):
49
  """Extract structured JSON from extracted text but do calculations in Python."""
50
  llm = get_llm()
51
-
52
  prompt_template = PromptTemplate(
53
  input_variables=["extracted_data"],
54
- template= """
55
  You are a data extraction assistant. Given the following extracted text, structure the output as valid JSON.
56
  **Do NOT perform any calculations.** Just extract the numeric values exactly as they appear.
57
 
@@ -60,6 +63,7 @@ def parse_data_with_llm(extracted_text):
60
 
61
  ### **Extract the Following Fields (Raw Data Only)**:
62
  - **Customer Ref**: Extract "Customer PO".
 
63
  - **Wt. in gram 14K**: Extract all "14K WG" weight values of parts and return as a list.
64
  - **Wt. in gram 18K**: Extract all "18K WG" weight values of parts and return as a list.
65
  - **Wt. in gram PLT**: Extract all "PLT" weight values of parts and return as a list.
@@ -70,6 +74,7 @@ def parse_data_with_llm(extracted_text):
70
  ```json
71
  {{
72
  "Customer Ref": "...",
 
73
  "Wt. in gram 14K": ...,
74
  "Wt. in gram 18K": ...,
75
  "Wt. in gram PLT": ...,
@@ -85,17 +90,23 @@ def parse_data_with_llm(extracted_text):
85
  ]
86
  }}
87
  ```
88
- """
89
- )
90
 
91
  prompt = prompt_template.format(extracted_data=extracted_text)
92
  response = llm.invoke(prompt)
93
 
94
- response_text = response.content.strip() if hasattr(response, 'content') else str(response).strip()
 
 
 
 
95
  json_data = extract_json_from_text(response_text)
96
-
97
  if not json_data:
98
- cleaned_response = response_text.replace("```json", "").replace("```", "").strip()
 
 
99
  try:
100
  json_data = json.loads(cleaned_response)
101
  except json.JSONDecodeError as e:
@@ -107,37 +118,85 @@ def parse_data_with_llm(extracted_text):
107
  def compute_values(data):
108
  """Perform metal calculations and format diamonds separately."""
109
  gold_price, platinum_price = fetch_metal_prices()
110
-
111
  gold_lock = gold_price if gold_price > 0 else 2800
112
  platinum_lock = platinum_price if platinum_price > 0 else 1040
113
-
114
  gold_lock_column = f"Gold Lock @ $ {gold_lock:.2f}"
115
  platinum_lock_column = f"Platinum Lock $ {platinum_lock:.2f}"
116
-
117
  # Extract rhodium value from LLM data
118
  rhodium_value = float(data.get("Rhodium", 0))
119
-
120
  metal_data = {
121
  "Customer Ref": data.get("Customer Ref", ""),
 
122
  gold_lock_column: gold_lock,
123
  platinum_lock_column: platinum_lock,
124
- "Wt. in gram 14K": sum(float(x) for x in data.get("Wt. in gram 14K", [])) if isinstance(data.get("Wt. in gram 14K"), list) else float(data.get("Wt. in gram 14K", 0)),
125
- "Wt. in gram 18K": sum(float(x) for x in data.get("Wt. in gram 18K", [])) if isinstance(data.get("Wt. in gram 18K"), list) else float(data.get("Wt. in gram 18K", 0)),
126
- "Wt. in gram PLT": sum(float(x) for x in data.get("Wt. in gram PLT", [])) if isinstance(data.get("Wt. in gram PLT"), list) else float(data.get("Wt. in gram PLT", 0)),
127
- "Gold Rate (including loss) 14KT": round((gold_lock / 31.1035) * 0.585 * 1.1, 2),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
128
  "Gold Rate (including loss) 18KT": round((gold_lock / 31.1035) * 0.75 * 1.1, 2),
129
  "Plat rate (including Lab)": round((platinum_lock / 31.1035) * 0.95 * 1.2, 2),
130
- "Gold Value 14K": round(sum(float(x) for x in data.get("Wt. in gram 14K", [])) * round((gold_lock / 31.1035) * 0.585 * 1.1, 2), 2),
131
- "Gold Value 18K": round(sum(float(x) for x in data.get("Wt. in gram 18K", [])) * round((gold_lock / 31.1035) * 0.75 * 1.1, 2), 2),
132
- "Plat Value": round(sum(float(x) for x in data.get("Wt. in gram PLT", [])) * round((platinum_lock / 31.1035) * 0.95 * 1.2, 2), 2),
133
- "Labour 14K": round(12 if (sum(float(x) for x in data.get("Wt. in gram 14K", [])) * 7) < 12 else (sum(float(x) for x in data.get("Wt. in gram 14K", [])) * 7), 2),
134
- "Labour 18K": round(12 if (sum(float(x) for x in data.get("Wt. in gram 18K", [])) * 7) < 12 else (sum(float(x) for x in data.get("Wt. in gram 18K", [])) * 7), 2),
135
- "Labour PT": round(15 if (sum(float(x) for x in data.get("Wt. in gram PLT", [])) * 11) < 15 else (sum(float(x) for x in data.get("Wt. in gram PLT", [])) * 11), 2),
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
  "Finding For Gold 14KT": 0,
137
  "Finding For Gold 18KT": 0,
138
  "Finding for Plt.": 0,
139
  "Rhodium": rhodium_value, # Use the extracted rhodium value
140
- "Stamping": 1
141
  }
142
 
143
  diamonds = data.get("Diamonds", [])
@@ -146,22 +205,32 @@ def compute_values(data):
146
  try:
147
  qty = float(diamond["Dia Qty"])
148
  wt = float(diamond["Dia Wt."])
149
- diamond["Diamond Weight"] = round(qty * wt, 3)
150
- diamond["Diamond Rate"] = 0
151
- diamond["Diamond Value"] = 0
152
  except (ValueError, TypeError):
153
  diamond["Diamond Weight"] = 0
154
  diamond["Diamond Rate"] = 0
155
  diamond["Diamond Value"] = 0
156
 
157
-
158
  return metal_data, diamonds
159
 
 
160
  def process_extracted_text(extracted_text):
161
  extracted_data = parse_data_with_llm(extracted_text)
 
 
 
 
 
162
  metal_data, diamonds = compute_values(extracted_data)
163
-
164
- return {"Metal Details": metal_data, "Diamonds": diamonds}
 
 
 
 
 
165
 
166
  def fetch_metal_prices():
167
  """Fetch current gold and platinum prices from Kitco London Fix API."""
@@ -176,28 +245,117 @@ def fetch_metal_prices():
176
  ID currency results{...LondonQuoteFragment}
177
  }
178
  }""",
179
- "variables": {
180
- "year": current_year,
181
- "currency": "USD"
182
- },
183
- "operationName": "LondonFixByYear"
184
  }
185
-
186
  headers = {"Content-Type": "application/json"}
187
-
188
  try:
189
  response = requests.post(url, json=payload, headers=headers)
190
  response.raise_for_status()
191
  data = response.json()
192
-
193
  results = data["data"]["GetLondonFixByYearV3"]["results"]
194
  if results and len(results) > 0:
195
  gold_price = float(results[0]["goldPM"])
196
  platinum_price = float(results[0]["platinumPM"])
197
  else:
198
  raise KeyError("No results found in the response")
199
-
200
  return gold_price, platinum_price
201
  except (requests.exceptions.RequestException, KeyError, ValueError) as e:
202
  print(f"Error fetching metal prices: {e}")
203
  return 0, 0
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import json
 
 
 
2
  import math
3
+ import os
4
+ import re
5
  from datetime import datetime
6
+
7
+ import requests
8
+ from dotenv import load_dotenv
9
+ from langchain_core.prompts import PromptTemplate
10
+ from langchain_google_genai import ChatGoogleGenerativeAI
11
+
12
  load_dotenv()
13
 
14
+
15
  def get_llm():
16
  """Create and return a ChatGoogleGenerativeAI instance with proper authentication."""
17
  google_api_key = os.getenv("GOOGLE_API_KEY")
18
  if not google_api_key:
19
  raise ValueError("GOOGLE_API_KEY environment variable is not set")
20
  return ChatGoogleGenerativeAI(
21
+ model="gemini-2.0-flash", temperature=0, google_api_key=google_api_key
 
 
22
  )
23
 
24
+
25
  def extract_json_from_text(text):
26
  """Extract JSON from text that might contain explanation text or code blocks."""
27
+ json_pattern = r"\{.*\}"
28
  match = re.search(json_pattern, text, re.DOTALL)
29
+
30
  if match:
31
  potential_json = match.group(0)
32
  try:
33
  return json.loads(potential_json)
34
  except json.JSONDecodeError:
35
  pass
36
+
37
+ code_block_pattern = r"```(?:json)?\s*([\s\S]*?)\s*```"
38
  matches = re.findall(code_block_pattern, text, re.DOTALL)
39
+
40
  for potential_json in matches:
41
  try:
42
+ cleaned = potential_json.strip().replace("```", "")
43
+ if cleaned.startswith("{") and cleaned.endswith("}"):
44
  return json.loads(cleaned)
45
  except json.JSONDecodeError:
46
  continue
47
+
48
  return None
49
 
50
+
51
  def parse_data_with_llm(extracted_text):
52
  """Extract structured JSON from extracted text but do calculations in Python."""
53
  llm = get_llm()
54
+
55
  prompt_template = PromptTemplate(
56
  input_variables=["extracted_data"],
57
+ template="""
58
  You are a data extraction assistant. Given the following extracted text, structure the output as valid JSON.
59
  **Do NOT perform any calculations.** Just extract the numeric values exactly as they appear.
60
 
 
63
 
64
  ### **Extract the Following Fields (Raw Data Only)**:
65
  - **Customer Ref**: Extract "Customer PO".
66
+ - **Accessory Type**: IMPORTANT! Look for the jewelry category or type. Extract this information and classify it as one of these types: pendant, ringband, earring, bracelet, necklace. This is often found after "Category:" or similar heading in the document.
67
  - **Wt. in gram 14K**: Extract all "14K WG" weight values of parts and return as a list.
68
  - **Wt. in gram 18K**: Extract all "18K WG" weight values of parts and return as a list.
69
  - **Wt. in gram PLT**: Extract all "PLT" weight values of parts and return as a list.
 
74
  ```json
75
  {{
76
  "Customer Ref": "...",
77
+ "Accessory Type": "...",
78
  "Wt. in gram 14K": ...,
79
  "Wt. in gram 18K": ...,
80
  "Wt. in gram PLT": ...,
 
90
  ]
91
  }}
92
  ```
93
+ """,
94
+ )
95
 
96
  prompt = prompt_template.format(extracted_data=extracted_text)
97
  response = llm.invoke(prompt)
98
 
99
+ response_text = (
100
+ response.content.strip()
101
+ if hasattr(response, "content")
102
+ else str(response).strip()
103
+ )
104
  json_data = extract_json_from_text(response_text)
105
+
106
  if not json_data:
107
+ cleaned_response = (
108
+ response_text.replace("```json", "").replace("```", "").strip()
109
+ )
110
  try:
111
  json_data = json.loads(cleaned_response)
112
  except json.JSONDecodeError as e:
 
118
  def compute_values(data):
119
  """Perform metal calculations and format diamonds separately."""
120
  gold_price, platinum_price = fetch_metal_prices()
121
+
122
  gold_lock = gold_price if gold_price > 0 else 2800
123
  platinum_lock = platinum_price if platinum_price > 0 else 1040
124
+
125
  gold_lock_column = f"Gold Lock @ $ {gold_lock:.2f}"
126
  platinum_lock_column = f"Platinum Lock $ {platinum_lock:.2f}"
127
+
128
  # Extract rhodium value from LLM data
129
  rhodium_value = float(data.get("Rhodium", 0))
130
+
131
  metal_data = {
132
  "Customer Ref": data.get("Customer Ref", ""),
133
+ "Accessory Type": data.get("Accessory Type", ""),
134
  gold_lock_column: gold_lock,
135
  platinum_lock_column: platinum_lock,
136
+ "Wt. in gram 14K": (
137
+ sum(float(x) for x in data.get("Wt. in gram 14K", []))
138
+ if isinstance(data.get("Wt. in gram 14K"), list)
139
+ else float(data.get("Wt. in gram 14K", 0))
140
+ ),
141
+ "Wt. in gram 18K": (
142
+ sum(float(x) for x in data.get("Wt. in gram 18K", []))
143
+ if isinstance(data.get("Wt. in gram 18K"), list)
144
+ else float(data.get("Wt. in gram 18K", 0))
145
+ ),
146
+ "Wt. in gram PLT": (
147
+ sum(float(x) for x in data.get("Wt. in gram PLT", []))
148
+ if isinstance(data.get("Wt. in gram PLT"), list)
149
+ else float(data.get("Wt. in gram PLT", 0))
150
+ ),
151
+ "Gold Rate (including loss) 14KT": round(
152
+ (gold_lock / 31.1035) * 0.585 * 1.1, 2
153
+ ),
154
  "Gold Rate (including loss) 18KT": round((gold_lock / 31.1035) * 0.75 * 1.1, 2),
155
  "Plat rate (including Lab)": round((platinum_lock / 31.1035) * 0.95 * 1.2, 2),
156
+ "Gold Value 14K": round(
157
+ sum(float(x) for x in data.get("Wt. in gram 14K", []))
158
+ * round((gold_lock / 31.1035) * 0.585 * 1.1, 2),
159
+ 2,
160
+ ),
161
+ "Gold Value 18K": round(
162
+ sum(float(x) for x in data.get("Wt. in gram 18K", []))
163
+ * round((gold_lock / 31.1035) * 0.75 * 1.1, 2),
164
+ 2,
165
+ ),
166
+ "Plat Value": round(
167
+ sum(float(x) for x in data.get("Wt. in gram PLT", []))
168
+ * round((platinum_lock / 31.1035) * 0.95 * 1.2, 2),
169
+ 2,
170
+ ),
171
+ "Labour 14K": round(
172
+ (
173
+ 12
174
+ if (sum(float(x) for x in data.get("Wt. in gram 14K", [])) * 7) < 12
175
+ else (sum(float(x) for x in data.get("Wt. in gram 14K", [])) * 7)
176
+ ),
177
+ 2,
178
+ ),
179
+ "Labour 18K": round(
180
+ (
181
+ 12
182
+ if (sum(float(x) for x in data.get("Wt. in gram 18K", [])) * 7) < 12
183
+ else (sum(float(x) for x in data.get("Wt. in gram 18K", [])) * 7)
184
+ ),
185
+ 2,
186
+ ),
187
+ "Labour PT": round(
188
+ (
189
+ 15
190
+ if (sum(float(x) for x in data.get("Wt. in gram PLT", [])) * 11) < 15
191
+ else (sum(float(x) for x in data.get("Wt. in gram PLT", [])) * 11)
192
+ ),
193
+ 2,
194
+ ),
195
  "Finding For Gold 14KT": 0,
196
  "Finding For Gold 18KT": 0,
197
  "Finding for Plt.": 0,
198
  "Rhodium": rhodium_value, # Use the extracted rhodium value
199
+ "Stamping": 1,
200
  }
201
 
202
  diamonds = data.get("Diamonds", [])
 
205
  try:
206
  qty = float(diamond["Dia Qty"])
207
  wt = float(diamond["Dia Wt."])
208
+ diamond["Diamond Weight"] = round(qty * wt, 3)
209
+ diamond["Diamond Rate"] = 0
210
+ diamond["Diamond Value"] = 0
211
  except (ValueError, TypeError):
212
  diamond["Diamond Weight"] = 0
213
  diamond["Diamond Rate"] = 0
214
  diamond["Diamond Value"] = 0
215
 
 
216
  return metal_data, diamonds
217
 
218
+
219
  def process_extracted_text(extracted_text):
220
  extracted_data = parse_data_with_llm(extracted_text)
221
+
222
+ # Get and map the accessory type
223
+ raw_accessory_type = extracted_data.get("Accessory Type", "")
224
+ accessory_type = map_accessory_type(raw_accessory_type)
225
+
226
  metal_data, diamonds = compute_values(extracted_data)
227
+
228
+ return {
229
+ "Metal Details": metal_data,
230
+ "Diamonds": diamonds,
231
+ "Accessory Type": accessory_type,
232
+ }
233
+
234
 
235
  def fetch_metal_prices():
236
  """Fetch current gold and platinum prices from Kitco London Fix API."""
 
245
  ID currency results{...LondonQuoteFragment}
246
  }
247
  }""",
248
+ "variables": {"year": current_year, "currency": "USD"},
249
+ "operationName": "LondonFixByYear",
 
 
 
250
  }
251
+
252
  headers = {"Content-Type": "application/json"}
253
+
254
  try:
255
  response = requests.post(url, json=payload, headers=headers)
256
  response.raise_for_status()
257
  data = response.json()
258
+
259
  results = data["data"]["GetLondonFixByYearV3"]["results"]
260
  if results and len(results) > 0:
261
  gold_price = float(results[0]["goldPM"])
262
  platinum_price = float(results[0]["platinumPM"])
263
  else:
264
  raise KeyError("No results found in the response")
265
+
266
  return gold_price, platinum_price
267
  except (requests.exceptions.RequestException, KeyError, ValueError) as e:
268
  print(f"Error fetching metal prices: {e}")
269
  return 0, 0
270
+
271
+
272
+ def map_accessory_type(detected_type):
273
+ """Map the detected accessory type to one of the standard types using LLM."""
274
+ if not detected_type or detected_type.strip() == "":
275
+ return "pendant" # Default if no type detected
276
+
277
+ llm = get_llm()
278
+
279
+ prompt_template = PromptTemplate(
280
+ input_variables=["detected_type"],
281
+ template="""
282
+ You are a jewelry categorization expert. Your task is to classify the given jewelry description
283
+ into exactly one of these categories: pendant, ringband, earring, bracelet, necklace.
284
+
285
+ Description: {detected_type}
286
+
287
+ Analyze the description and determine which category it belongs to. If it's ambiguous,
288
+ use your knowledge of jewelry to make the best classification.
289
+
290
+ Respond with ONLY one of these exact words: pendant, ringband, earring, bracelet, necklace.
291
+ """,
292
+ )
293
+
294
+ prompt = prompt_template.format(detected_type=detected_type)
295
+
296
+ try:
297
+ response = llm.invoke(prompt)
298
+ response_text = (
299
+ response.content.strip()
300
+ if hasattr(response, "content")
301
+ else str(response).strip()
302
+ )
303
+
304
+ # Extract just the category name, removing any extra text
305
+ response_text = response_text.lower()
306
+
307
+ valid_types = ["pendant", "ringband", "earring", "bracelet", "necklace"]
308
+
309
+ # Check if response contains a valid type
310
+ for valid_type in valid_types:
311
+ if valid_type in response_text:
312
+ return valid_type
313
+
314
+ # If we couldn't find an exact match, check if there's something close
315
+ if "ring" in response_text or "band" in response_text:
316
+ return "ringband"
317
+ elif "ear" in response_text:
318
+ return "earring"
319
+ elif "neck" in response_text or "chain" in response_text:
320
+ return "necklace"
321
+ elif "brace" in response_text or "cuff" in response_text:
322
+ return "bracelet"
323
+ elif "pend" in response_text or "locket" in response_text:
324
+ return "pendant"
325
+
326
+ # Default to pendant if we can't determine
327
+ return "pendant"
328
+ except Exception as e:
329
+ print(f"Error in LLM accessory type mapping: {str(e)}")
330
+
331
+ # Fall back to basic matching if LLM fails
332
+ detected_type = detected_type.lower()
333
+
334
+ # Direct matches
335
+ if detected_type in ["pendant", "ringband", "earring", "bracelet", "necklace"]:
336
+ return detected_type
337
+
338
+ # Handle common variations and synonyms
339
+ mapping = {
340
+ "pend": "pendant",
341
+ "locket": "pendant",
342
+ "charm": "pendant",
343
+ "ring": "ringband",
344
+ "band": "ringband",
345
+ "ear": "earring",
346
+ "stud": "earring",
347
+ "hoop": "earring",
348
+ "bangle": "bracelet",
349
+ "cuff": "bracelet",
350
+ "wrist": "bracelet",
351
+ "chain": "necklace",
352
+ "choker": "necklace",
353
+ "collar": "necklace",
354
+ }
355
+
356
+ for key, value in mapping.items():
357
+ if key in detected_type:
358
+ return value
359
+
360
+ # Default to pendant if we can't determine the type
361
+ return "pendant"