FastAPI-Img2CSV / main.py
utkarsh134's picture
Update main.py
4c1ad4b verified
Raw History Blame Contribute Delete
22 kB
import csv
import hashlib
import json
import math
from io import BytesIO
import pandas as pd
from fastapi import (FastAPI, File, Form, HTTPException,
UploadFile)
from fastapi.responses import StreamingResponse
from acc_processor import AccessoryProcessor
from cache import textract_cache
from extract import extract_data_from_image
from parsing import map_accessory_type, process_extracted_text
app = FastAPI()
def get_file_hash(file_content: bytes) -> str:
"""Generate a unique hash for a file to use as cache key"""
return hashlib.md5(file_content).hexdigest()
async def get_extracted_text(file: UploadFile = None, extraction_id: str = None) -> str:
"""Get extracted text from cache or Textract based on file or extraction ID"""
if extraction_id:
# Try to get from cache using extraction ID
cached_text = textract_cache.get(extraction_id)
if cached_text is not None:
return cached_text
raise HTTPException(
status_code=404,
detail=f"Extraction ID {extraction_id} not found or expired",
)
if not file:
raise HTTPException(
status_code=400, detail="Either file or extraction_id must be provided"
)
file_content = await file.read()
file_hash = get_file_hash(file_content)
# Try to get from cache using file hash
cached_text = textract_cache.get(file_hash)
if cached_text is not None:
# Ensure file.file is repositioned for any subsequent reads
await file.seek(0)
return cached_text
# Not in cache, extract using Textract
extracted_text = extract_data_from_image(BytesIO(file_content))
# Store in cache (no need to return the ID here)
textract_cache.set(file_hash, extracted_text)
# Ensure file.file is repositioned for any subsequent reads
await file.seek(0)
return extracted_text
@app.post("/extract-raw-data/")
async def extract_raw_data(file: UploadFile = File(...)):
"""
Extract raw text data from an image and cache it for later use.
Returns an extraction_id that can be used in other endpoints.
"""
try:
file_content = await file.read()
file_hash = get_file_hash(file_content)
# Try to get from cache first
cached_text = textract_cache.get(file_hash)
if cached_text is None:
# Not in cache, extract using Textract
extracted_text = extract_data_from_image(BytesIO(file_content))
else:
extracted_text = cached_text
# Store in cache and get extraction ID
extraction_id = textract_cache.set(file_hash, extracted_text)
# Return a short sample of the extracted text for verification
text_sample = (
extracted_text[:200] + "..."
if len(extracted_text) > 200
else extracted_text
)
return {
"extraction_id": extraction_id,
"text_sample": text_sample,
"message": "Extraction successful. Use this extraction_id for subsequent processing calls.",
}
except Exception as e:
raise HTTPException(status_code=500, detail=f"Extraction failed: {str(e)}")
@app.post("/extract-diamonds/")
async def extract_diamonds(
file: UploadFile = File(None), extraction_id: str = Form(None)
):
"""Extract diamonds data using either a file or an extraction_id"""
try:
if not file and not extraction_id:
raise HTTPException(
status_code=400, detail="Either file or extraction_id must be provided"
)
extracted_text = await get_extracted_text(file, extraction_id)
processed_data = process_extracted_text(extracted_text)
return {"diamonds": processed_data["Diamonds"]}
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
@app.post("/detect-accessory-type/")
async def detect_accessory_type(
file: UploadFile = File(None), extraction_id: str = Form(None)
):
"""Detect accessory type using either a file or an extraction_id"""
try:
if not file and not extraction_id:
raise HTTPException(
status_code=400, detail="Either file or extraction_id must be provided"
)
extracted_text = await get_extracted_text(file, extraction_id)
processed_data = process_extracted_text(extracted_text)
detected_type = processed_data.get("Accessory Type", "pendant")
return {
"detected_accessory_type": detected_type,
"raw_text_sample": (
extracted_text[:500] + "..."
if len(extracted_text) > 500
else extracted_text
),
}
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
@app.post("/process-image/")
async def process_image(
file: UploadFile = File(None),
extraction_id: str = Form(None),
accessory_type: str = Form(None),
finding_14kt: float = Form(...),
finding_18kt: float = Form(...),
finding_plt: float = Form(...),
diamond_rates: str = Form(...),
gold_markup_type: str = Form(...),
gold_markup_value: float = Form(...),
platinum_markup_type: str = Form(...),
platinum_markup_value: float = Form(...),
diamond_markup_type: str = Form(...),
diamond_markup_value: float = Form(...),
finding_markup_type: str = Form(...),
finding_markup_value: float = Form(...),
):
"""Process image data using either a file or an extraction_id"""
try:
if not file and not extraction_id:
raise HTTPException(
status_code=400, detail="Either file or extraction_id must be provided"
)
extracted_text = await get_extracted_text(file, extraction_id)
processed_data = process_extracted_text(extracted_text)
detected_accessory_type = processed_data.get("Accessory Type", "pendant")
accessory_type = accessory_type or detected_accessory_type
processor = AccessoryProcessor(accessory_type)
diamond_rates_dict = json.loads(diamond_rates)
metal_data = processed_data["Metal Details"]
diamonds = processed_data["Diamonds"]
diamond_values_sum = 0
settings_sum = 0
for idx, diamond in enumerate(diamonds):
rate = float(diamond_rates_dict.get(str(idx), 0))
diamond["Diamond Rate"] = rate
diamond_weight = float(diamond.get("Diamond Weight", 0))
diamond_value = round(diamond_weight * rate, 2)
diamond["Diamond Value"] = diamond_value
diamond_values_sum += diamond_value
if accessory_type in ["bracelet", "necklace"]:
dia_qty = float(diamond.get("Dia Qty", 0))
diamond["Setting"] = round(dia_qty * 0.25, 2)
else:
diamond["Setting"] = 0
settings_sum = (
sum(diamond["Setting"] for diamond in diamonds)
if accessory_type in ["bracelet", "necklace"]
else 0
)
if finding_markup_type == "percentage":
finding_markup_factor = 1 + (finding_markup_value / 100)
finding_14kt = round(finding_14kt * finding_markup_factor, 2)
finding_18kt = round(finding_18kt * finding_markup_factor, 2)
finding_plt = round(finding_plt * finding_markup_factor, 2)
elif finding_markup_type == "fixed":
finding_14kt = round(finding_14kt + finding_markup_value, 2)
finding_18kt = round(finding_18kt + finding_markup_value, 2)
finding_plt = round(finding_plt + finding_markup_value, 2)
elif finding_markup_type == "times":
finding_14kt = round(finding_14kt * finding_markup_value, 2)
finding_18kt = round(finding_18kt * finding_markup_value, 2)
finding_plt = round(finding_plt * finding_markup_value, 2)
metal_data = processor.calculate_labour(metal_data)
finding_key_map = {
"pendant": {
"14kt": "Finding For Gold 14KT",
"18kt": "Finding For Gold 18KT",
"plt": "Finding for Plt.",
},
"ringband": {
"14kt": "CFP for Gold 14KT",
"18kt": "CFP for Gold 18KT",
"plt": "CFP for Plt.",
},
"earring": {
"14kt": "Finding & CFP For Gold 14KT",
"18kt": "Finding For Gold 18KT",
"plt": "Finding for Plt.",
},
"bracelet": {
"14kt": "CFP For Gold 14KT",
"18kt": "CFP For Gold 18KT",
"plt": "CFP For Gold PLT",
},
"necklace": {
"14kt": "CFP For Gold 14KT",
"18kt": "CFP For Gold 18KT",
"plt": "CFP For Gold PLT",
},
}[accessory_type]
if accessory_type in ["bracelet", "necklace"]:
metal_data.update(
{
"CFP For Gold 14KT": finding_14kt,
"CFP For Gold 18KT": finding_18kt,
"CFP For Gold PLT": finding_plt,
"Finding Markup Type": finding_markup_type,
"Finding Markup Value": finding_markup_value,
"Setting": settings_sum,
}
)
elif accessory_type == "ringband":
metal_data.update(
{
"CFP for Gold 14KT": finding_14kt,
"CFP for Gold 18KT": finding_18kt,
"CFP for Plt.": finding_plt,
"Finding Markup Type": finding_markup_type,
"Finding Markup Value": finding_markup_value,
}
)
elif accessory_type == "earring":
metal_data.update(
{
"Finding & CFP For Gold 14KT": finding_14kt,
"Finding For Gold 18KT": finding_18kt,
"Finding for Plt.": finding_plt,
"Finding Markup Type": finding_markup_type,
"Finding Markup Value": finding_markup_value,
}
)
else: # pendant case
metal_data.update(
{
"Finding For Gold 14KT": finding_14kt,
"Finding For Gold 18KT": finding_18kt,
"Finding for Plt.": finding_plt,
"Finding Markup Type": finding_markup_type,
"Finding Markup Value": finding_markup_value,
}
)
metal_data.update(
{
"Gold Lock": round(
metal_data.get("Wt. in gram 14K", 0) * 0.15
+ metal_data.get("Wt. in gram 18K", 0) * 0.15,
2,
),
"Platinum Lock": round(metal_data.get("Wt. in gram PLT", 0) * 0.25, 2),
}
)
if gold_markup_type == "percentage":
gold_markup_factor = 1 + (gold_markup_value / 100)
metal_data["Gold Value 14K"] = round(
metal_data["Gold Value 14K"] * gold_markup_factor, 2
)
metal_data["Gold Value 18K"] = round(
metal_data["Gold Value 18K"] * gold_markup_factor, 2
)
elif gold_markup_type == "fixed":
metal_data["Gold Value 14K"] = round(
metal_data["Gold Value 14K"] + gold_markup_value, 2
)
metal_data["Gold Value 18K"] = round(
metal_data["Gold Value 18K"] + gold_markup_value, 2
)
elif gold_markup_type == "times":
metal_data["Gold Value 14K"] = round(
metal_data["Gold Value 14K"] * gold_markup_value, 2
)
metal_data["Gold Value 18K"] = round(
metal_data["Gold Value 18K"] * gold_markup_value, 2
)
if platinum_markup_type == "percentage":
platinum_markup_factor = 1 + (platinum_markup_value / 100)
metal_data["Plat Value"] = round(
metal_data["Plat Value"] * platinum_markup_factor, 2
)
elif platinum_markup_type == "fixed":
metal_data["Plat Value"] = round(
metal_data["Plat Value"] + platinum_markup_value, 2
)
elif platinum_markup_type == "times":
metal_data["Plat Value"] = round(
metal_data["Plat Value"] * platinum_markup_value, 2
)
if diamond_markup_type == "percentage":
diamond_markup_factor = 1 + (diamond_markup_value / 100)
for diamond in diamonds:
diamond["Diamond Value"] = round(
diamond["Diamond Value"] * diamond_markup_factor, 2
)
elif diamond_markup_type == "fixed":
for diamond in diamonds:
diamond["Diamond Value"] = round(
diamond["Diamond Value"] + diamond_markup_value, 2
)
elif diamond_markup_type == "times":
for diamond in diamonds:
diamond["Diamond Value"] = round(
diamond["Diamond Value"] * diamond_markup_value, 2
)
diamond_values_sum = sum(diamond["Diamond Value"] for diamond in diamonds)
if accessory_type in ["bracelet", "necklace"]:
usd_14k = (
metal_data["Gold Value 14K"]
+ finding_14kt
+ diamond_values_sum
+ settings_sum
+ metal_data["Rhodium"]
)
usd_14k = math.ceil(usd_14k) + 1
usd_18k = (
metal_data["Gold Value 18K"]
+ finding_18kt
+ diamond_values_sum
+ settings_sum
+ metal_data["Rhodium"]
)
usd_18k = math.ceil(usd_18k) + 1
usd_pt = (
metal_data["Plat Value"]
+ finding_plt
+ diamond_values_sum
+ settings_sum
+ metal_data["Rhodium"]
)
usd_pt = math.ceil(usd_pt) + 1
else:
usd_14k = (
metal_data["Gold Value 14K"]
+ finding_14kt
+ diamond_values_sum
+ metal_data["Labour 14K"]
+ metal_data["Rhodium"]
)
usd_14k = math.ceil(usd_14k) + 1
usd_18k = (
metal_data["Gold Value 18K"]
+ finding_18kt
+ diamond_values_sum
+ metal_data["Labour 18K"]
+ metal_data["Rhodium"]
)
usd_18k = math.ceil(usd_18k) + 1
usd_pt = (
metal_data["Plat Value"]
+ finding_plt
+ diamond_values_sum
+ metal_data["Labour PT"]
+ metal_data["Rhodium"]
)
usd_pt = math.ceil(usd_pt) + 1
gbp_14k = usd_14k * 0.78
gbp_18k = usd_18k * 0.78
gbp_pt = usd_pt * 0.78
duty_14k = gbp_14k * 1.025
duty_18k = gbp_18k * 1.025
duty_pt = gbp_pt * 1.025
freight = 75
landing_14k = duty_14k + freight
landing_18k = duty_18k + freight
landing_pt = duty_pt + freight
metal_data.update(
{
"Gold Markup Type": gold_markup_type,
"Gold Markup Value": gold_markup_value,
"Platinum Markup Type": platinum_markup_type,
"Platinum Markup Value": platinum_markup_value,
"Diamond Markup Type": diamond_markup_type,
"Diamond Markup Value": diamond_markup_value,
"Price of USD 14K": f"$ {usd_14k:.2f}",
"Price of USD 18K": f"$ {usd_18k:.2f}",
"Price of USD PT": f"$ {usd_pt:.2f}",
"Price of GBP 14K @ 1/0.78": f"£ {gbp_14k:.2f}",
"Price of GBP 18K @ 1/0.78": f"£ {gbp_18k:.2f}",
"Price of GBP PT @ 1/0.78": f"£ {gbp_pt:.2f}",
"14K Duty @ 2.5% Including": f"£ {duty_14k:.2f}",
"18K Duty @ 2.5% Including": f"£ {duty_18k:.2f}",
"PT950 Duty @ 2.5% Including": f"£ {duty_pt:.2f}",
"Freight": f"£ {freight:.2f}",
"14K Landing Cost (Duty + Freight)": f"£ {landing_14k:.2f}",
"18K Landing Cost (Duty + Freight)": f"£ {landing_18k:.2f}",
"PT950 Landing Cost (Duty + Freight)": f"£ {landing_pt:.2f}",
}
)
if not diamonds:
final_df = pd.DataFrame([metal_data])
else:
columns_config = processor.get_column_config()
gold_lock_col = next(
col for col in metal_data.keys() if "Gold Lock @" in col
)
platinum_lock_col = next(
col for col in metal_data.keys() if "Platinum Lock" in col
)
metal_columns = [
col(metal_data) if callable(col) else col
for col in columns_config["metal_columns"]
]
metal_data_basic = {
k: v for k, v in metal_data.items() if k in metal_columns
}
findings_data = {}
for col in columns_config["finding_columns"]:
findings_data[col] = metal_data.get(col, 0) # Use 0 as default value
findings_df = pd.DataFrame([findings_data])
labour_data = {
k: v
for k, v in metal_data.items()
if k in columns_config["labour_columns"]
}
price_data = {
k: v
for k, v in metal_data.items()
if k in columns_config["price_columns"]
}
metal_df = pd.DataFrame([metal_data_basic])
labour_df = pd.DataFrame([labour_data])
price_df = pd.DataFrame([price_data])
diamonds_df = pd.DataFrame(diamonds)
empty_metal = pd.DataFrame(
[{k: "" for k in metal_columns}] * (len(diamonds_df) - 1)
)
empty_findings = pd.DataFrame(
[{k: "" for k in columns_config["finding_columns"]}]
* (len(diamonds_df) - 1)
)
empty_labour = pd.DataFrame(
[{k: "" for k in columns_config["labour_columns"]}]
* (len(diamonds_df) - 1)
)
empty_prices = pd.DataFrame(
[{k: "" for k in columns_config["price_columns"]}]
* (len(diamonds_df) - 1)
)
combined_metal = pd.concat([metal_df, empty_metal], ignore_index=True)
combined_findings = pd.concat(
[findings_df, empty_findings], ignore_index=True
)
combined_labour = pd.concat([labour_df, empty_labour], ignore_index=True)
combined_prices = pd.concat([price_df, empty_prices], ignore_index=True)
final_df = pd.concat(
[
combined_metal,
diamonds_df,
combined_findings,
combined_labour,
combined_prices,
],
axis=1,
)
for col in columns_config["price_columns"]:
if col in final_df.columns: # Check if column exists
final_df[col] = final_df[col].astype(str)
final_df[col] = final_df[col].apply(
lambda x: x.strip('"').strip("'") if x else ""
)
if "GBP" in col:
final_df[col] = final_df[col].apply(
lambda x: x.replace("£", "£") if x else ""
)
print("Finding columns config:", columns_config["finding_columns"])
print(
"Metal data finding keys:", [k for k in metal_data.keys() if "Finding" in k]
)
print("Findings data:", findings_data)
output = BytesIO()
final_df.to_csv(
output,
index=False,
encoding="utf-8-sig",
quoting=csv.QUOTE_MINIMAL,
escapechar="\\",
)
output.seek(0)
return StreamingResponse(
output,
media_type="text/csv; charset=utf-8",
headers={
"Content-Disposition": "attachment; filename=output.csv",
"Content-Type": "text/csv; charset=utf-8",
},
)
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
@app.post("/test-accessory-mapping/")
async def test_accessory_mapping(description: str = Form(...)):
"""Test endpoint to see how the LLM maps different accessory descriptions."""
try:
mapped_type = map_accessory_type(description)
return {"original_description": description, "mapped_type": mapped_type}
except Exception as e:
raise HTTPException(status_code=400, detail=str(e))
@app.post("/admin/clear-cache/")
async def clear_cache(key: str = None):
"""Admin endpoint to clear the cache"""
textract_cache.clear(key)
return {"status": "success", "message": "Cache cleared"}
@app.get("/check-extraction/")
async def check_extraction(extraction_id: str):
"""Check if an extraction ID is valid and not expired"""
is_valid = textract_cache.is_valid_id(extraction_id)
return {
"extraction_id": extraction_id,
"valid": is_valid,
"message": (
"Extraction ID is valid and can be used"
if is_valid
else "Extraction ID not found or expired"
),
}