Spaces:
Sleeping
Sleeping
Update backend with high-quality catalog and fixes
Browse files- app/api/admin.py +18 -6
- app/api/product_quality_checker_lib.py +35 -22
- app/api/settings.py +0 -1
- app/core/backup_manager.py +5 -2
- app/core/logging.py +5 -2
- app/main.py +0 -13
app/api/admin.py
CHANGED
|
@@ -826,13 +826,21 @@ def restore_flagged_product(
|
|
| 826 |
catalog_data["metadata"]["total"] = len(catalog_data["products"])
|
| 827 |
|
| 828 |
# 5. Save main catalog
|
| 829 |
-
|
| 830 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 831 |
|
| 832 |
# 6. Remove from pending
|
| 833 |
pending = [p for p in pending if p["id"] != product_id]
|
| 834 |
-
|
| 835 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 836 |
|
| 837 |
log_audit_event("RESTORED_FROM_QUARANTINE", f"Product {product_id} restored to main catalog", {"product_id": product_id})
|
| 838 |
|
|
@@ -861,8 +869,12 @@ async def delete_flagged_product(
|
|
| 861 |
if len(flagged_data) == original_count:
|
| 862 |
raise HTTPException(status_code=404, detail="Product not found in quarantine")
|
| 863 |
|
| 864 |
-
|
| 865 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 866 |
|
| 867 |
# Log the permanent deletion with metadata
|
| 868 |
log_audit_event("PERMANENTLY_DELETED", f"Deleted product ID: {product_id}", {"product_id": product_id, "source": "quarantine"})
|
|
|
|
| 826 |
catalog_data["metadata"]["total"] = len(catalog_data["products"])
|
| 827 |
|
| 828 |
# 5. Save main catalog
|
| 829 |
+
try:
|
| 830 |
+
with open(catalog_path, "w", encoding="utf-8") as f:
|
| 831 |
+
json.dump(catalog_data, f, ensure_ascii=False, indent=2)
|
| 832 |
+
except Exception as e:
|
| 833 |
+
print(f"[ADMIN] Warning: Could not update local catalog JSON: {e}")
|
| 834 |
+
|
| 835 |
|
| 836 |
# 6. Remove from pending
|
| 837 |
pending = [p for p in pending if p["id"] != product_id]
|
| 838 |
+
try:
|
| 839 |
+
with open(PENDING_DELETION_FILE, "w", encoding="utf-8") as f:
|
| 840 |
+
json.dump(pending, f, ensure_ascii=False, indent=2)
|
| 841 |
+
except Exception as e:
|
| 842 |
+
print(f"[ADMIN] Warning: Could not update pending_deletion.json: {e}")
|
| 843 |
+
|
| 844 |
|
| 845 |
log_audit_event("RESTORED_FROM_QUARANTINE", f"Product {product_id} restored to main catalog", {"product_id": product_id})
|
| 846 |
|
|
|
|
| 869 |
if len(flagged_data) == original_count:
|
| 870 |
raise HTTPException(status_code=404, detail="Product not found in quarantine")
|
| 871 |
|
| 872 |
+
try:
|
| 873 |
+
with open(PENDING_DELETION_FILE, "w", encoding="utf-8") as f:
|
| 874 |
+
json.dump(flagged_data, f, ensure_ascii=False, indent=2)
|
| 875 |
+
except Exception as e:
|
| 876 |
+
print(f"[ADMIN] Warning: Could not update pending_deletion.json: {e}")
|
| 877 |
+
|
| 878 |
|
| 879 |
# Log the permanent deletion with metadata
|
| 880 |
log_audit_event("PERMANENTLY_DELETED", f"Deleted product ID: {product_id}", {"product_id": product_id, "source": "quarantine"})
|
app/api/product_quality_checker_lib.py
CHANGED
|
@@ -39,14 +39,21 @@ except ImportError:
|
|
| 39 |
DB_AVAILABLE = False
|
| 40 |
|
| 41 |
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
logger = logging.getLogger(__name__)
|
| 51 |
|
| 52 |
|
|
@@ -385,11 +392,13 @@ class StandaloneQualityChecker:
|
|
| 385 |
|
| 386 |
logger.info(f"Validation complete: {results['summary']['valid']} valid, {results['summary']['invalid']} invalid")
|
| 387 |
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
|
|
|
|
|
|
| 393 |
|
| 394 |
return results
|
| 395 |
|
|
@@ -425,10 +434,12 @@ def create_backup(db_session) -> Optional[str]:
|
|
| 425 |
backup_filename = f"products_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
| 426 |
backup_path = os.path.join(backup_dir, backup_filename)
|
| 427 |
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
|
|
|
|
|
|
| 432 |
return backup_path
|
| 433 |
|
| 434 |
except Exception as e:
|
|
@@ -480,11 +491,13 @@ def main():
|
|
| 480 |
checker = ProductQualityChecker(db)
|
| 481 |
report = checker.run_quality_check(args.batch_size)
|
| 482 |
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
|
|
|
|
|
|
| 488 |
|
| 489 |
finally:
|
| 490 |
db.close()
|
|
|
|
| 39 |
DB_AVAILABLE = False
|
| 40 |
|
| 41 |
|
| 42 |
+
try:
|
| 43 |
+
logging.basicConfig(
|
| 44 |
+
level=logging.INFO,
|
| 45 |
+
format='%(asctime)s - %(levelname)s - %(message)s',
|
| 46 |
+
handlers=[
|
| 47 |
+
logging.FileHandler(f'product_quality_{datetime.now().strftime("%Y%m%d_%H%M%S")}.log'),
|
| 48 |
+
logging.StreamHandler()
|
| 49 |
+
]
|
| 50 |
+
)
|
| 51 |
+
except Exception:
|
| 52 |
+
logging.basicConfig(
|
| 53 |
+
level=logging.INFO,
|
| 54 |
+
format='%(asctime)s - %(levelname)s - %(message)s',
|
| 55 |
+
handlers=[logging.StreamHandler()]
|
| 56 |
+
)
|
| 57 |
logger = logging.getLogger(__name__)
|
| 58 |
|
| 59 |
|
|
|
|
| 392 |
|
| 393 |
logger.info(f"Validation complete: {results['summary']['valid']} valid, {results['summary']['invalid']} invalid")
|
| 394 |
|
| 395 |
+
try:
|
| 396 |
+
output_path = self.json_path.replace('.json', '_validation.json')
|
| 397 |
+
with open(output_path, 'w', encoding='utf-8') as f:
|
| 398 |
+
json.dump(results, f, ensure_ascii=False, indent=2)
|
| 399 |
+
logger.info(f"Results saved to {output_path}")
|
| 400 |
+
except Exception as e:
|
| 401 |
+
logger.warning(f"Could not save validation results to disk: {e}")
|
| 402 |
|
| 403 |
return results
|
| 404 |
|
|
|
|
| 434 |
backup_filename = f"products_backup_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
| 435 |
backup_path = os.path.join(backup_dir, backup_filename)
|
| 436 |
|
| 437 |
+
try:
|
| 438 |
+
with open(backup_path, 'w', encoding='utf-8') as f:
|
| 439 |
+
json.dump(backup_data, f, ensure_ascii=False, indent=2)
|
| 440 |
+
logger.info(f"Backup created: {backup_path}")
|
| 441 |
+
except Exception as e:
|
| 442 |
+
logger.warning(f"Could not write backup to disk: {e}")
|
| 443 |
return backup_path
|
| 444 |
|
| 445 |
except Exception as e:
|
|
|
|
| 491 |
checker = ProductQualityChecker(db)
|
| 492 |
report = checker.run_quality_check(args.batch_size)
|
| 493 |
|
| 494 |
+
try:
|
| 495 |
+
report_path = os.path.join(os.path.dirname(__file__), f"quality_report_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json")
|
| 496 |
+
with open(report_path, 'w', encoding='utf-8') as f:
|
| 497 |
+
json.dump(report.to_dict(), f, ensure_ascii=False, indent=2)
|
| 498 |
+
logger.info(f"Quality report saved: {report_path}")
|
| 499 |
+
except Exception as e:
|
| 500 |
+
logger.warning(f"Could not save quality report to disk: {e}")
|
| 501 |
|
| 502 |
finally:
|
| 503 |
db.close()
|
app/api/settings.py
CHANGED
|
@@ -1,7 +1,6 @@
|
|
| 1 |
from fastapi import APIRouter, Depends, HTTPException, status, UploadFile, File
|
| 2 |
from datetime import datetime
|
| 3 |
import os
|
| 4 |
-
import shutil
|
| 5 |
from app.core.config import settings as app_settings
|
| 6 |
from sqlalchemy.orm import Session
|
| 7 |
from typing import Any
|
|
|
|
| 1 |
from fastapi import APIRouter, Depends, HTTPException, status, UploadFile, File
|
| 2 |
from datetime import datetime
|
| 3 |
import os
|
|
|
|
| 4 |
from app.core.config import settings as app_settings
|
| 5 |
from sqlalchemy.orm import Session
|
| 6 |
from typing import Any
|
app/core/backup_manager.py
CHANGED
|
@@ -17,8 +17,11 @@ class BackupManager:
|
|
| 17 |
self.catalog_path = os.path.join(base_dir, "backend", "real_products_200.json")
|
| 18 |
self.retention_days = 30
|
| 19 |
|
| 20 |
-
|
| 21 |
-
os.
|
|
|
|
|
|
|
|
|
|
| 22 |
|
| 23 |
def create_snapshot(self, reason: str = "automated"):
|
| 24 |
"""Creates a timestamped backup of the current catalog."""
|
|
|
|
| 17 |
self.catalog_path = os.path.join(base_dir, "backend", "real_products_200.json")
|
| 18 |
self.retention_days = 30
|
| 19 |
|
| 20 |
+
try:
|
| 21 |
+
if not os.path.exists(self.backup_dir):
|
| 22 |
+
os.makedirs(self.backup_dir, exist_ok=True)
|
| 23 |
+
except Exception as e:
|
| 24 |
+
logger.warning(f"Could not create backup directory: {e}")
|
| 25 |
|
| 26 |
def create_snapshot(self, reason: str = "automated"):
|
| 27 |
"""Creates a timestamped backup of the current catalog."""
|
app/core/logging.py
CHANGED
|
@@ -9,8 +9,11 @@ def log_audit_event(action: str, details: str, metadata: dict = None, severity:
|
|
| 9 |
"""Log an event to the formal JSON audit trail with rich metadata."""
|
| 10 |
try:
|
| 11 |
log_dir = os.path.dirname(AUDIT_LOG_FILE)
|
| 12 |
-
|
| 13 |
-
os.
|
|
|
|
|
|
|
|
|
|
| 14 |
|
| 15 |
logs = []
|
| 16 |
if os.path.exists(AUDIT_LOG_FILE):
|
|
|
|
| 9 |
"""Log an event to the formal JSON audit trail with rich metadata."""
|
| 10 |
try:
|
| 11 |
log_dir = os.path.dirname(AUDIT_LOG_FILE)
|
| 12 |
+
try:
|
| 13 |
+
if not os.path.exists(log_dir):
|
| 14 |
+
os.makedirs(log_dir, exist_ok=True)
|
| 15 |
+
except Exception as e:
|
| 16 |
+
print(f"[LOGGING] Warning: Could not create log directory: {e}")
|
| 17 |
|
| 18 |
logs = []
|
| 19 |
if os.path.exists(AUDIT_LOG_FILE):
|
app/main.py
CHANGED
|
@@ -85,19 +85,6 @@ def create_app() -> FastAPI:
|
|
| 85 |
app.include_router(admin_router, prefix="/api")
|
| 86 |
app.include_router(notifications_router, prefix="/api")
|
| 87 |
|
| 88 |
-
from fastapi.staticfiles import StaticFiles
|
| 89 |
-
import os
|
| 90 |
-
|
| 91 |
-
# Mount uploads directory from DATA_DIR
|
| 92 |
-
uploads_base = os.path.join(settings.DATA_DIR, "uploads")
|
| 93 |
-
try:
|
| 94 |
-
os.makedirs(os.path.join(uploads_base, "receipts"), exist_ok=True)
|
| 95 |
-
except Exception as e:
|
| 96 |
-
print(f"[MAIN] Warning: Could not create upload directories: {e}")
|
| 97 |
-
|
| 98 |
-
if os.path.exists(uploads_base):
|
| 99 |
-
app.mount("/uploads", StaticFiles(directory=uploads_base), name="uploads")
|
| 100 |
-
|
| 101 |
app.include_router(settings_router, prefix="/api")
|
| 102 |
|
| 103 |
VERSION = "2.0.0-FINAL-CATALOG"
|
|
|
|
| 85 |
app.include_router(admin_router, prefix="/api")
|
| 86 |
app.include_router(notifications_router, prefix="/api")
|
| 87 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 88 |
app.include_router(settings_router, prefix="/api")
|
| 89 |
|
| 90 |
VERSION = "2.0.0-FINAL-CATALOG"
|