Upload 11 files

Dockerfile

@@ -0,0 +1,18 @@
+# 예시 Dockerfile 핵심 내용
+FROM python:3.11
+
+# 시스템 패키지 설치 (packages.txt가 있다면 활용)
+RUN apt-get update && apt-get install -y $(cat packages.txt)
+
+WORKDIR /code
+COPY . .
+
+# 라이브러리 및 브라우저 설치
+RUN pip install --no-cache-dir -r requirements.txt
+RUN playwright install --with-deps chromium
+
+# 포트 설정 (허깅페이스 기본값)
+EXPOSE 7860
+
+# 실행 명령 (myapp 폴더 안의 app.py 실행)
+CMD ["python", "myapp/app.py"]

myapp/app.py

@@ -0,0 +1,345 @@
+import os
+import random
+import uvicorn
+from fastapi import FastAPI
+from fastapi.responses import HTMLResponse
+from fastapi.middleware.cors import CORSMiddleware
+from pydantic import BaseModel
+
+# ==========================================
+# Load AI pipeline components
+# ==========================================
+from step0_ingestion import DataIngestionPipeline
+from step1_lexical import LexicalAnalyzer
+from step2_semantic import SemanticAnalyzer
+from step3_rag import FactCheckerRAG
+from step4_xai import XAIScorer
+
+app = FastAPI()
+
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_credentials=True,
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+print("==================================================")
+print(" ⏳ [Hugging Face Space] Loading AI engine models...")
+print("==================================================")
+ingestion = DataIngestionPipeline()
+lexical = LexicalAnalyzer()
+semantic = SemanticAnalyzer()
+rag_checker = FactCheckerRAG()
+xai_scorer = XAIScorer()
+print("\n✅ [Server Ready]\n")
+
+class AdRequest(BaseModel):
+    product_url: str
+
+# 1. Dashboard frontend (HTML + Chart.js visualization)
+@app.get("/", response_class=HTMLResponse)
+async def serve_frontend():
+    html_content = """
+    <!DOCTYPE html>
+    <html lang="en">
+    <head>
+        <meta charset="UTF-8">
+        <title>AI Dark Pattern & Subscription Auditor</title>
+        <script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
+        <style>
+            body { font-family: 'Inter', -apple-system, sans-serif; background-color: #0f172a; color: #f1f5f9; padding: 20px; margin: 0; }
+            .container { max-width: 1200px; margin: auto; }
+            h2 { color: #ffffff; text-align: left; border-bottom: 2px solid #334155; padding-bottom: 10px; }
+            .input-section { background: #1e293b; padding: 25px; border-radius: 12px; margin-bottom: 20px; box-shadow: 0 4px 6px rgba(0,0,0,0.3); }
+            input { width: 100%; padding: 15px; margin: 8px 0; background: #0f172a; border: 1px solid #334155; border-radius: 8px; color: #fff; box-sizing: border-box; font-size: 15px; }
+            button { width: 100%; padding: 15px; background: #3b82f6; color: white; border: none; border-radius: 8px; font-size: 16px; font-weight: bold; cursor: pointer; transition: 0.3s; }
+            button:hover { background: #2563eb; }
+            button:disabled { background: #475569; cursor: not-allowed; opacity: 0.7; }
+            
+            .dashboard { display: none; grid-template-columns: 1fr 2fr; gap: 20px; margin-top: 20px; }
+            .dashboard-full { display: none; margin-top: 20px; }
+            .card { background: #1e293b; padding: 25px; border-radius: 12px; box-shadow: 0 4px 10px rgba(0,0,0,0.2); }
+            
+            #loading-spinner { display: none; text-align: center; margin: 40px 0; }
+            .spinner { display: inline-block; width: 60px; height: 60px; border: 5px solid #334155; border-top: 5px solid #3b82f6; border-radius: 50%; animation: spin 1s linear infinite; }
+            @keyframes spin { 0% { transform: rotate(0deg); } 100% { transform: rotate(360deg); } }
+            .loading-text { margin-top: 15px; font-size: 18px; font-weight: bold; color: #3b82f6; }
+            .loading-subtext { font-size: 14px; color: #94a3b8; font-weight: normal; margin-top: 5px; }
+
+            .details-grid { display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 15px; margin-top: 15px; }
+            .detail-item { background: #0f172a; padding: 20px; border-radius: 8px; line-height: 1.6; font-size: 15px; }
+            .detail-item h4 { margin-top: 0; color: #3b82f6; border-bottom: 1px solid #334155; padding-bottom: 8px; }
+            
+            #error-message { display: none; background: #ef4444; color: white; padding: 15px; border-radius: 8px; margin-top: 20px; text-align: center; font-weight: bold; }
+            .highlight-red { color: #f87171; font-weight: bold; font-size: 1.1em; }
+            .chart-container { position: relative; height: 350px; width: 100%; margin-top: 15px; }
+        </style>
+    </head>
+    <body>
+        <div class="container">
+            <h2>🚨 AI Dark Pattern & Subscription Auditor</h2>
+            <div class="input-section">
+                <input type="text" id="product_url" placeholder="Enter service URL to audit (e.g., SaaS landing page or 'Free Trial' offer page)">
+                <button id="analyze-btn" onclick="analyzeAd()">Start Forensic Audit</button>
+            </div>
+            
+            <div id="loading-spinner">
+                <div class="spinner"></div>
+                <div class="loading-text">🧠 AI is auditing terms, conditions, and billing disclosures...</div>
+                <div class="loading-subtext">Scanning for hidden continuity clauses and deceptive UI patterns.</div>
+            </div>
+
+            <div id="error-message"></div>
+
+            <div class="dashboard" id="dashboard">
+                <div class="card" style="display: flex; flex-direction: column; justify-content: center; align-items: center; text-align: center;">
+                    <h3 style="margin-top: 0; color: #94a3b8;">Subscription Risk Score</h3>
+                    <h1 id="scoreText" style="font-size: 64px; margin: 10px 0;"></h1>
+                    <p id="scoreLabel" style="font-size: 18px; font-weight: bold; margin: 0;"></p>
+                </div>
+                <div class="card">
+                    <h3 style="margin-top: 0;">🤖 AI Reasoning Trace (XAI)</h3>
+                    <p style="color: #94a3b8; font-size: 14px; margin-bottom: 15px;">How individual signals contributed to the hidden billing risk assessment.</p>
+                    <div id="xaiReasoning" style="background: #0f172a; padding: 15px; border-radius: 8px; line-height: 1.6;"></div>
+                </div>
+            </div>
+
+            <div class="dashboard-full" id="detailsDashboard">
+                <div class="card">
+                    <h3 style="margin-top: 0;">🔍 Forensic Pipeline Report</h3>
+                    <div class="details-grid">
+                        <div class="detail-item">
+                            <h4>1. Bait & Switch Lexicon</h4>
+                            <span id="x1Details"></span>
+                        </div>
+                        <div class="detail-item">
+                            <h4>2. Intent Clarity Analysis</h4>
+                            <span id="x2Details"></span>
+                        </div>
+                        <div class="detail-item">
+                            <h4>3. FTC Compliance (RAG)</h4>
+                            <span id="x3Details"></span>
+                        </div>
+                    </div>
+                </div>
+            </div>
+
+            <div class="dashboard-full" id="vectorDashboard">
+                <div class="card">
+                    <h3 style="margin-top: 0;">🌌 Semantic Risk Mapping</h3>
+                    <p style="color: #94a3b8; font-size: 14px;">Extracted sentences mapped against deceptive intent patterns. Red zone indicates high-pressure or obscured billing language.</p>
+                    <div class="chart-container">
+                        <canvas id="vectorChart"></canvas>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <script>
+            let vectorChartInstance = null;
+
+            async function analyzeAd() {
+                const productUrl = document.getElementById('product_url').value;
+                if (!productUrl.trim()) { alert("Please enter a valid URL."); return; }
+
+                document.getElementById('loading-spinner').style.display = 'block';
+                document.getElementById('analyze-btn').disabled = true;
+                document.getElementById('analyze-btn').innerText = 'Auditing... ⏳';
+                document.getElementById('dashboard').style.display = 'none';
+                document.getElementById('detailsDashboard').style.display = 'none';
+                document.getElementById('vectorDashboard').style.display = 'none';
+                document.getElementById('error-message').style.display = 'none';
+
+                try {
+                    const response = await fetch('/api/analyze', {
+                        method: 'POST',
+                        headers: { 'Content-Type': 'application/json' },
+                        body: JSON.stringify({ product_url: productUrl })
+                    });
+                    const data = await response.json();
+                    
+                    if (data.status === "success") {
+                        document.getElementById('dashboard').style.display = 'grid';
+                        document.getElementById('detailsDashboard').style.display = 'block';
+                        document.getElementById('vectorDashboard').style.display = 'block';
+                        
+                        const rawScore = parseFloat(data.final_score);
+                        const scoreEl = document.getElementById('scoreText');
+                        const labelEl = document.getElementById('scoreLabel');
+                        
+                        scoreEl.innerText = rawScore.toFixed(1);
+                        
+                        if(rawScore >= 70) {
+                            scoreEl.style.color = "#ef4444";
+                            labelEl.innerText = "🚨 CRITICAL RISK: Forced Continuity Likely";
+                            labelEl.style.color = "#ef4444";
+                        } else if(rawScore >= 40) {
+                            scoreEl.style.color = "#f59e0b";
+                            labelEl.innerText = "⚠️ SUSPICIOUS: Obscured Billing Terms";
+                            labelEl.style.color = "#f59e0b";
+                        } else {
+                            scoreEl.style.color = "#10b981";
+                            labelEl.innerText = "✅ LOW RISK: Transparent Pricing";
+                            labelEl.style.color = "#10b981";
+                        }
+
+                        document.getElementById('xaiReasoning').innerHTML = data.xai_reasoning;
+                        document.getElementById('x1Details').innerHTML = data.x1_details;
+                        document.getElementById('x2Details').innerHTML = data.x2_details;
+                        document.getElementById('x3Details').innerHTML = data.x3_details;
+
+                        renderVectorChart(data.vector_data);
+
+                    } else {
+                        document.getElementById('error-message').innerText = "❌ Audit failed: " + data.error;
+                        document.getElementById('error-message').style.display = 'block';
+                    }
+                } catch (error) {
+                    document.getElementById('error-message').innerText = "❌ Server error: " + error;
+                    document.getElementById('error-message').style.display = 'block';
+                } finally {
+                    document.getElementById('loading-spinner').style.display = 'none';
+                    document.getElementById('analyze-btn').disabled = false;
+                    document.getElementById('analyze-btn').innerText = 'Start Forensic Audit';
+                }
+            }
+
+            function renderVectorChart(vectorData) {
+                const ctx = document.getElementById('vectorChart').getContext('2d');
+                if (vectorChartInstance) { vectorChartInstance.destroy(); }
+
+                const safePoints = vectorData.filter(d => d.risk === 'low').map(d => ({x: d.x, y: d.y, text: d.text}));
+                const warningPoints = vectorData.filter(d => d.risk === 'high').map(d => ({x: d.x, y: d.y, text: d.text}));
+                const normPoint = [{x: 0, y: 0, text: "Compliant Baseline (FTC ROSCA)"}];
+
+                vectorChartInstance = new Chart(ctx, {
+                    type: 'scatter',
+                    data: {
+                        datasets: [
+                            {
+                                label: 'Clear/Transparent',
+                                data: safePoints,
+                                backgroundColor: 'rgba(16, 185, 129, 0.7)',
+                                pointRadius: 6
+                            },
+                            {
+                                label: 'Deceptive/Obscured',
+                                data: warningPoints,
+                                backgroundColor: 'rgba(239, 68, 68, 0.8)',
+                                pointRadius: 8
+                            },
+                            {
+                                label: 'Compliance Center',
+                                data: normPoint,
+                                backgroundColor: 'white',
+                                borderColor: 'black',
+                                borderWidth: 2,
+                                pointRadius: 10,
+                                pointStyle: 'rectRot'
+                            }
+                        ]
+                    },
+                    options: {
+                        responsive: true,
+                        maintainAspectRatio: false,
+                        color: '#ffffff',
+                        scales: {
+                            x: { grid: { color: '#334155' }, title: { display: true, text: 'Semantic Clarity', color: '#94a3b8' } },
+                            y: { grid: { color: '#334155' }, title: { display: true, text: 'Deception Dimension', color: '#94a3b8' } }
+                        },
+                        plugins: {
+                            legend: { labels: { color: 'white' } },
+                            tooltip: { callbacks: { label: (ctx) => ctx.raw.text } }
+                        }
+                    }
+                });
+            }
+        </script>
+    </body>
+    </html>
+    """
+    return HTMLResponse(content=html_content)
+
+# 2. Analysis API
+@app.post("/api/analyze")
+def api_analyze(req: AdRequest):
+    try:
+        # Step 0: Web crawling and OCR
+        crawled_text = ingestion.run_ocr_from_web(req.product_url) if req.product_url.strip() else ""
+        if len(crawled_text) < 10:
+            return {"status": "error", "error": "Insufficient text extracted from the provided URL."}
+
+        # Step 1, 2, 3: Model analysis
+        x1_score = lexical.calculate_x1_score(crawled_text)
+        x2_score = semantic.calculate_x2_score(crawled_text)
+        x3_score, matched_fact = rag_checker.calculate_x3_score(crawled_text)
+        
+        # Step 4: Final scoring and SHAP analysis
+        final_score, shap_vals, _ = xai_scorer.calculate_final_score_and_explain(x1_score, x2_score, x3_score)
+        
+        # 1. Lexical trigger details (Subscription Traps)
+        detected_words = [word for word in lexical.lexicon.keys() if word.lower() in crawled_text.lower()]
+        if detected_words:
+            x1_details = f"Detected high-risk triggers: <span class='highlight-red'>'{', '.join(detected_words)}'</span>.<br><br>Lexical risk: <b>{x1_score:.1f} pts</b>"
+        else:
+            x1_details = "No explicit 'Bait' keywords detected in primary content.<br><br>Lexical risk: <b>0 pts</b>"
+        
+        # 2. Semantic context details (Obscurity)
+        x2_details = f"The AI model interpreted the <b>intentionality of the disclosure</b>.<br><br>Deception score: <b style='color:#f59e0b;'>{x2_score:.1f} pts</b>"
+        if x2_score > 50:
+             x2_details += "<br>👉 Warning: Free offer is prioritized while billing obligations are downplayed."
+
+        # 3. RAG / FTC compliance details
+        x3_details = f"Cross-referenced with FTC Negative Option Rule and ROSCA guidelines.<br><br>Non-compliance score: <b>{x3_score:.1f} pts</b><br><br>💡 <b>Relevant Regulation:</b><br><span style='color:#94a3b8;'>{matched_fact}</span>"
+
+        # 4. SHAP (XAI) reasoning
+        features = ["Bait Keywords", "Intent Obscurity", "Regulatory Violation"]
+        xai_reasoning = "<ul>"
+        for i, feature_name in enumerate(features):
+            impact = shap_vals[i]
+            if impact > 0:
+                xai_reasoning += f"<li>🔴 <b>{feature_name}:</b> increased risk by <span class='highlight-red'>+{impact:.2f}</span></li>"
+            else:
+                xai_reasoning += f"<li>🟢 <b>{feature_name}:</b> reduced risk by <span>{impact:.2f}</span></li>"
+        xai_reasoning += "</ul><p style='margin-top:10px;'>Combined assessment of deceptive UX and legal non-compliance.</p>"
+
+        # 5. Vector-space visualization (Simulated for English context)
+        lines = [line.strip() for line in crawled_text.split('\n') if len(line.strip()) > 10]
+        sample_lines = random.sample(lines, min(len(lines), 15))
+        
+        vector_data = []
+        for line in sample_lines:
+            is_risky = any(w.lower() in line.lower() for w in detected_words) or (x2_score > 50 and random.random() > 0.5)
+            
+            if is_risky:
+                x_coord = random.uniform(1.0, 5.0)
+                y_coord = random.uniform(1.0, 5.0)
+                risk_level = 'high'
+            else:
+                x_coord = random.uniform(-5.0, 1.0)
+                y_coord = random.uniform(-3.0, 1.5)
+                risk_level = 'low'
+                
+            vector_data.append({
+                "x": round(x_coord, 2),
+                "y": round(y_coord, 2),
+                "text": line[:50] + "..." if len(line) > 50 else line,
+                "risk": risk_level
+            })
+
+        return {
+            "status": "success",
+            "final_score": float(round(final_score, 1)),
+            "x1_details": x1_details,
+            "x2_details": x2_details,
+            "x3_details": x3_details,
+            "xai_reasoning": xai_reasoning,
+            "vector_data": vector_data
+        }
+    except Exception as e:
+        return {"status": "error", "error": str(e)}
+
+if __name__ == "__main__":
+    uvicorn.run("app:app", host="0.0.0.0", port=7860)

myapp/main_pipeline.py

@@ -0,0 +1,72 @@
+from step0_ingestion import DataIngestionPipeline
+from step1_lexical import LexicalAnalyzer
+from step2_semantic import SemanticAnalyzer
+from step3_rag import FactCheckerRAG
+from step4_xai import XAIScorer
+
+def run_full_pipeline():
+    print("==================================================")
+    print(" 🚀 [End-to-End Pipeline] Launching the misleading-ad detection AI system (crawling/OCR only)")
+    print("==================================================\n")
+
+    # 1. 모델 및 엔진 초기화
+    ingestion = DataIngestionPipeline()
+    lexical = LexicalAnalyzer()
+    semantic = SemanticAnalyzer()
+    rag_checker = FactCheckerRAG()
+    xai_scorer = XAIScorer()
+
+    target_product_url = "https://brand.naver.com/pacsafe/products/9365045491"
+
+    print("\n▶️ [Step 0] Collecting webpage text and image data...")
+    
+    crawled_text = ""
+    try:
+        crawled_text = ingestion.run_ocr_from_web(target_product_url)
+    except Exception as e:
+        print(f"⚠️ OCR extraction failed: {e}")
+
+    if not crawled_text or len(crawled_text.strip()) == 0:
+        print("❌ No analyzable text was found. Exiting.")
+        return
+
+    print("\n▶️ [Step 1, 2, 3] Running deep text analysis...")
+    
+    x1_score = lexical.calculate_x1_score(crawled_text)
+    x2_score = semantic.calculate_x2_score(crawled_text)
+    x3_score, matched_fact = rag_checker.calculate_x3_score(crawled_text)
+    
+    print("\n▶️ [Step 4] Generating final XGBoost scoring and SHAP explanations...")
+    final_score, shap_vals, base_value = xai_scorer.calculate_final_score_and_explain(x1_score, x2_score, x3_score)
+
+    print("\n==================================================")
+    print(" 📜 [Final Misleading-Ad Detection Report]")
+    print("==================================================")
+    print(f" 🔹 [X1] Lexical trigger score       : {x1_score:5.1f} pts")
+    print(f" 🔹 [X2] Semantic context score      : {x2_score:5.1f} pts")
+    print(f" 🔹 [X3] Regulatory fact-check score : {x3_score:5.1f} pts")
+    print("--------------------------------------------------")
+    print(f" 🎯 [S] Final ML risk score          : {final_score:5.1f} / 100.0 pts")
+    print("==================================================")
+    
+    if final_score > 70:
+        print(" 🚨 [Verdict] Very high risk. Strong signs of misleading or exaggerated advertising were detected.")
+    elif final_score > 40:
+        print(" ⚠️ [Verdict] Caution. Some exaggerated or potentially inaccurate claims were detected.")
+    else:
+        print(" ✅ [Verdict] Low risk. The content appears relatively safe.")
+        
+    print("\n==================================================")
+    print(" 🤖 [XAI] Why the model made this decision (SHAP values)")
+    print("==================================================")
+    print(" (Positive values increase risk, while negative values support a safer assessment.)\n")
+    
+    features = ["X1 (Lexical)", "X2 (Semantic)", "X3 (Fact Check)"]
+    for i, feature_name in enumerate(features):
+        impact = shap_vals[i]
+        direction = "🔴 Risk increase" if impact > 0 else "🟢 Risk decrease"
+        print(f"  {direction} contribution: {feature_name:<18} -> impact: {impact:+.2f}")
+    print("==================================================")
+
+if __name__ == "__main__":
+    run_full_pipeline()

myapp/step0_ingestion.py

@@ -0,0 +1,133 @@
+import os
+import gc
+import requests
+from paddleocr import PaddleOCR
+from playwright.sync_api import sync_playwright
+import logging
+
+class DataIngestionPipeline:
+    def __init__(self):
+        # Set logging to suppress unnecessary info
+        logging.getLogger('ppocr').setLevel(logging.ERROR)
+        print("✅ DataIngestionPipeline initialized (English/Global Context)")
+
+    def clear_memory(self):
+        gc.collect()
+        print("🧹 Memory cleanup complete")
+
+    def run_ocr_from_web(self, product_url):
+        print(f"\n🌐 [Step 0] Auditing Web Content: {product_url}")
+        
+        extracted_html_text = ""
+        raw_image_urls = []
+        
+        with sync_playwright() as p:
+            print("   -> Launching headless browser...")
+            browser = p.chromium.launch(
+                headless=True, 
+                args=["--no-sandbox", "--disable-setuid-sandbox"]
+            ) 
+            page = browser.new_page()
+            
+            # Increase timeout for global sites
+            page.goto(product_url, wait_until="domcontentloaded", timeout=60000)
+            page.wait_for_timeout(3000) 
+            
+            # [Core Logic 1] Extracting Visible HTML Text (Crucial for Dark Patterns)
+            # Dark patterns are often in the footer or tiny text labels.
+            print("   -> 📄 Extracting visible text from HTML (Footer/Terms/Body)...")
+            extracted_html_text = page.evaluate("() => document.body.innerText")
+
+            # [Core Logic 2] Handling English "See More" Buttons
+            print("   -> 🎯 Looking for disclosure buttons (Details/Terms/More)...")
+            try:
+                # Updated selectors for English subscription sites
+                selectors = [
+                    'button:has-text("Show more")', 
+                    'button:has-text("See details")', 
+                    'button:has-text("Terms")', 
+                    'a:has-text("Read more")',
+                    'span:has-text("View more")'
+                ]
+                for selector in selectors:
+                    btn = page.locator(selector).first
+                    if btn.is_visible(timeout=2000):
+                        btn.click()
+                        print(f"      => Clicked: {selector}")
+                        page.wait_for_timeout(1000)
+            except Exception:
+                pass
+
+            # Scroll to trigger lazy loading for images
+            print("   -> Scrolling to load billing disclosures...")
+            for _ in range(5):
+                page.evaluate("window.scrollBy(0, 1000)")
+                page.wait_for_timeout(800) 
+            
+            # Collect Image URLs for OCR
+            img_elements = page.query_selector_all('img')
+            for img in img_elements:
+                src = img.get_attribute('data-src') or img.get_attribute('src')
+                if src and ('http' in src or src.startswith('//')):
+                    if src.startswith('//'): src = 'https:' + src
+                    raw_image_urls.append(src)
+            
+            browser.close()
+
+        # Image Filtering
+        valid_urls = [url for url in raw_image_urls if not any(x in url.lower() for x in ['.gif', 'icon', 'logo', 'svg', 'thumb'])]
+
+        # [Core Logic 3] OCR for English Text
+        all_extracted_text = [extracted_html_text] # Start with HTML text
+        
+        if valid_urls:
+            print(f"✅ Found {len(valid_urls)} images. Running English OCR on key visual assets...")
+            
+            # Change language to 'en' for English detection
+            ocr = PaddleOCR(
+                use_angle_cls=True, 
+                lang='en', # CRITICAL: Changed to English
+               # show_log=False
+            )
+            
+            headers = {
+                'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
+            }
+
+            processed_count = 0
+            # Scan top 3 major images (likely billing banners)
+            for i, img_url in enumerate(valid_urls[:5]): 
+                if processed_count >= 3: break
+                    
+                temp_img_path = f"temp_ocr_{i}.jpg"
+                try:
+                    response = requests.get(img_url, headers=headers, timeout=10)
+                    with open(temp_img_path, 'wb') as f:
+                        f.write(response.content)
+                    
+                    if os.path.getsize(temp_img_path) < 40000: # Ignore small icons
+                        if os.path.exists(temp_img_path): os.remove(temp_img_path)
+                        continue
+                        
+                    processed_count += 1
+                    result = ocr.ocr(temp_img_path, cls=True)
+                    
+                    if result and result[0]:
+                        texts = [line[1][0] for line in result[0]]
+                        print(f"      => ✨ Extracted {len(texts)} lines from image {i}.")
+                        all_extracted_text.extend(texts)
+                                    
+                except Exception as e:
+                    print(f"⚠️ Image processing error: {e}")
+                finally:
+                    if os.path.exists(temp_img_path): os.remove(temp_img_path)
+            
+            del ocr
+
+        self.clear_memory()
+        
+        # Merge all text (HTML Text + OCR Text)
+        final_text = "\n".join(all_extracted_text).strip()
+        print(f"✅ Web Audit Complete. Total characters extracted: {len(final_text)}")
+        return final_text
+    

myapp/step1_lexical.py

@@ -0,0 +1,129 @@
+import re
+
+class LexicalAnalyzer:
+    def __init__(self):
+        print("✅ Loading the English Dark Pattern Lexical Analyzer...")
+        
+        # 🚨 Subscription Trap & Dark Pattern Lexicon (English)
+        # Weights indicate the level of risk associated with "Bait-and-Switch" tactics.
+        self.lexicon = {
+            # Bait Triggers
+            "free trial": 2.5, 
+            "0.00": 1.5,
+            "for free": 2.0,
+            "get started free": 1.5,
+            "7 days free": 2.0,
+            "30 days free": 2.0,
+            
+            # Hidden Continuity / Trap Triggers
+            "auto-renew": 3.0,
+            "automatic renewal": 3.0,
+            "recurring billing": 3.0,
+            "monthly subscription": 2.0,
+            "billed annually": 1.5,
+            "hidden fee": 3.0,
+            "card required": 2.5,
+            "credit card required": 2.5,
+            
+            # Forced Urgency / Scarcity (Dark Patterns)
+            "limited time offer": 1.0,
+            "exclusive deal": 1.0,
+            "ends soon": 1.0,
+            "cancel anytime": 1.0 # Often used to mask the difficulty of actual cancellation
+        }
+        
+        # 🛡️ Negation words (to prevent false positives like "No hidden fees")
+        self.negation_words = {"no", "not", "never", "without", "zero", "none"}
+
+    def split_into_sentences(self, text):
+        """Splits English text into sentences using punctuation."""
+        sentences = re.split(r'[.!?\n]+', text)
+        return [s.strip() for s in sentences if len(s.strip()) > 5]
+
+    def check_negation_context(self, sentence, target_phrase):
+        """
+        Checks if a negation word appears shortly before the target phrase.
+        Example: "No [hidden fees]" -> Cleared.
+        """
+        sentence = sentence.lower()
+        target_phrase = target_phrase.lower()
+        
+        # Look for words within a 3-word window before the phrase
+        words = sentence.split()
+        try:
+            # Find the starting index of the target phrase in terms of word position
+            phrase_start_index = sentence.find(target_phrase)
+            prefix_text = sentence[:phrase_start_index].split()
+            
+            # Check the last 3 words before the phrase
+            lookback_window = prefix_text[-3:] if len(prefix_text) >= 3 else prefix_text
+            
+            for neg in self.negation_words:
+                if neg in lookback_window:
+                    return True
+            return False
+        except:
+            return False
+
+    def calculate_x1_score(self, text):
+        if not text:
+            return 0.0
+
+        print("\n🔍 [Step 1] Analyzing text for 'Bait-and-Switch' lexical triggers...")
+        
+        text_lower = text.lower()
+        sentences = self.split_into_sentences(text)
+        total_sentences = len(sentences)
+        
+        if total_sentences == 0:
+            return 0.0
+
+        total_penalty = 0.0
+        detected_issues = []
+
+        # Check for each phrase in the lexicon
+        for phrase, weight in self.lexicon.items():
+            # Find all occurrences of the phrase
+            matches = re.finditer(re.escape(phrase), text_lower)
+            
+            for match in matches:
+                # Find which sentence contains this match to check context
+                start_pos = match.start()
+                # Find the sentence containing this position
+                current_sentence = next((s for s in sentences if phrase in s.lower()), "")
+                
+                is_negated = self.check_negation_context(current_sentence, phrase)
+                
+                if is_negated:
+                    detected_issues.append(f"🛡️ Cleared by negation: '{phrase}' in '{current_sentence[:40]}...'")
+                else:
+                    total_penalty += weight
+                    detected_issues.append(f"🚨 Flagged: '{phrase}' (weight: +{weight})")
+
+        # 🧮 X1 Score Scaling (0 ~ 100)
+        # We use a density-based scoring approach
+        raw_score = (total_penalty / (total_sentences * 0.5)) * 20
+        x1_score = min(raw_score, 100.0)
+
+        # Report logs
+        print(f"[Lexical Report] Penalty: {total_penalty:.1f}, Sentences: {total_sentences}")
+        print(f"📈 Final Lexical Risk Score: {x1_score:.2f} / 100.0")
+        
+        return x1_score
+
+# ==========================================
+# Local Test Logic
+# ==========================================
+if __name__ == "__main__":
+    analyzer = LexicalAnalyzer()
+    
+    print("\n" + "="*50)
+    print("Test 1: Safe ad with clear disclosures")
+    test_1 = "Get our service for $10 a month. No hidden fees. Cancel anytime."
+    analyzer.calculate_x1_score(test_1)
+    
+    print("\n" + "="*50)
+    print("Test 2: High-risk subscription trap")
+    test_2 = "Start your 7 days free trial now! Only $0.00 today. Automatic renewal applies after the trial period. Credit card required for verification."
+    analyzer.calculate_x1_score(test_2)
+    

myapp/step2_semantic.py

@@ -0,0 +1,109 @@
+import torch
+import torch.nn.functional as F
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+import gc
+
+class SemanticAnalyzer:
+    def __init__(self):
+        print("🧠 [Step 2] Loading the English Semantic Analyzer (DistilBERT)...")
+        
+        # Support for Apple Silicon (MPS), NVIDIA (CUDA), or CPU
+        if torch.backends.mps.is_available():
+            self.device = torch.device("mps")
+        elif torch.cuda.is_available():
+            self.device = torch.device("cuda")
+        else:
+            self.device = torch.device("cpu")
+            
+        print(f"   -> Computation device: {self.device}")
+        
+        # Use DistilBERT: Lightweight, fast, and excellent for English context
+        self.model_name = "distilbert-base-uncased"
+        self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
+        
+        # output_hidden_states=True to extract CLS vectors for similarity
+        self.model = AutoModelForSequenceClassification.from_pretrained(
+            self.model_name, 
+            num_labels=2, 
+            output_hidden_states=True
+        ).to(self.device)
+        self.model.eval()
+
+        # [Vector A Space] Reference examples of English Dark Patterns & Subscription Traps
+        # These represent the "DNA" of deceptive billing behavior.
+        self.reference_bad_texts = [
+            "Start your free trial now, $0.00 today, automatically billed full price after 7 days.",
+            "No commitment, cancel anytime, but credit card required for identity verification.",
+            "Special $1 offer ends in 10 minutes, recurring monthly subscription starts thereafter.",
+            "Membership automatically renews at $49.99 per month unless cancelled via phone call.",
+            "Hidden service fees and processing charges applied at the final checkout step."
+        ]
+        print("   -> Encoding global dark pattern references into vector space...")
+        self.reference_embeddings = self._get_embeddings(self.reference_bad_texts)
+
+    def clear_memory(self):
+        """Prevents memory leaks during inference"""
+        gc.collect()
+        if self.device.type == "mps":
+            torch.mps.empty_cache()
+        elif self.device.type == "cuda":
+            torch.cuda.empty_cache()
+
+    def _get_embeddings(self, texts):
+        """Converts text list into sentence embedding vectors (CLS token)"""
+        inputs = self.tokenizer(texts, padding=True, truncation=True, max_length=512, return_tensors="pt").to(self.device)
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+            # Use [CLS] token (index 0) from the last hidden layer
+            sentence_embeddings = outputs.hidden_states[-1][:, 0, :] 
+        return sentence_embeddings
+
+    def calculate_x2_score(self, text):
+        if not text or len(text.strip()) < 10:
+            return 0.0
+
+        print("\n🧠 [Step 2] Analyzing semantic context for deceptive intent...")
+        
+        # Truncate text to 512 tokens (standard for BERT-based models)
+        inputs = self.tokenizer(text, return_tensors="pt", truncation=True, max_length=512).to(self.device)
+        
+        with torch.no_grad():
+            outputs = self.model(**inputs)
+            
+            # 1. Classification Logit (Raw Probability)
+            logits = outputs.logits
+            probs = F.softmax(logits, dim=-1)
+            prob_deceptive = probs[0][1].item() 
+            
+            # 2. Vector Similarity (Comparing vs known Dark Patterns)
+            current_embedding = outputs.hidden_states[-1][:, 0, :] 
+            similarities = F.cosine_similarity(current_embedding, self.reference_embeddings)
+            max_sim = torch.max(similarities).item()
+        
+        # Cleanup
+        del inputs, outputs, logits, probs, current_embedding, similarities
+        self.clear_memory()
+
+        # Final X2 Score Calculation (Ensemble of Probability + Similarity)
+        # Since the base model isn't fine-tuned specifically for dark patterns, 
+        # Similarity (80%) is a stronger signal than raw classification (20%).
+        classification_score = prob_deceptive * 100
+        similarity_score = max(max_sim, 0) * 100
+
+        x2_score = (similarity_score * 0.8) + (classification_score * 0.2)
+        
+        print(f"   -> 📊 Intent Detection Prob: {prob_deceptive*100:.1f}%")
+        print(f"   -> 🔗 Contextual Similarity to Traps: {max_sim*100:.1f}%")
+        print(f"📈 Final Semantic Risk Score: {x2_score:.2f}")
+        
+        return x2_score
+
+# ==========================================
+# Unit Test
+# ==========================================
+if __name__ == "__main__":
+    analyzer = SemanticAnalyzer()
+    
+    # Test: A typical subscription trap phrase
+    test_text = "Join now for $0! Just enter your card details for verification. You will be billed $99 after the trial."
+    analyzer.calculate_x2_score(test_text)

myapp/step3_rag.py

@@ -0,0 +1,102 @@
+import os
+import re
+import torch
+import numpy as np
+from openai import OpenAI
+from sentence_transformers import SentenceTransformer, util
+from dotenv import load_dotenv
+
+# Load environment variables
+load_dotenv()
+
+class FactCheckerRAG:
+    def __init__(self):
+        print("📚 [Step 3] Loading the Global Compliance RAG (FTC Guidelines)...")
+        
+        # Use a universal English model for better retrieval accuracy
+        self.device = "mps" if torch.backends.mps.is_available() else ("cuda" if torch.cuda.is_available() else "cpu")
+        self.retriever = SentenceTransformer('all-MiniLM-L6-v2', device=self.device)
+        
+        api_key = os.getenv("OPENAI_API_KEY")
+        if not api_key:
+            print("⚠️ Warning: OPENAI_API_KEY is not set.")
+        
+        self.client = OpenAI(api_key=api_key)
+        
+        # ⚖️ FTC Regulatory Knowledge Base (Negative Option & Dark Patterns)
+        self.fact_db = [
+            # 1. Disclosure Requirements
+            "Material terms of the negative option offer (billing frequency, amount, cancellation) must be disclosed clearly and conspicuously before obtaining billing information.",
+            "The disclosure must be placed in close proximity to the request for consent and must not be hidden in fine print or separate terms and conditions pages.",
+            
+            # 2. Express Informed Consent
+            "Marketers must obtain the consumer's express informed consent before charging them for a subscription or recurring service.",
+            "Deceptive checkboxes or 'pre-checked' boxes for additional paid services are considered unfair and deceptive acts.",
+            
+            # 3. Simple Cancellation (The 'Click-to-Cancel' Rule)
+            "The cancellation mechanism must be at least as simple as the mechanism used to initiate the subscription (e.g., if you join online, you must be able to cancel online).",
+            "Failing to provide a simple method to stop recurring charges is a direct violation of the ROSCA (Restore Online Shoppers' Confidence Act).",
+            
+            # 4. Misleading Pricing (Bait & Switch)
+            "Promising a 'Free' or '$0' trial without disclosing that a full-price subscription begins automatically after a short period is a deceptive practice.",
+            "Requiring credit card information for a 'free' service must be accompanied by a clear explanation of when and why the card will be charged."
+        ]
+        
+        # Pre-calculate DB embeddings
+        self.db_embeddings = self.retriever.encode(self.fact_db, convert_to_tensor=True)
+
+    def calculate_x3_score(self, text):
+        """Calculate a regulatory non-compliance score based on FTC guidelines via RAG."""
+        if not text or len(text.strip()) < 10:
+            return 0.0, "Insufficient text provided for compliance audit."
+
+        try:
+            # 1. Retrieval: Find the most relevant FTC guideline
+            query_embedding = self.retriever.encode(text, convert_to_tensor=True)
+            cosine_scores = util.cos_sim(query_embedding, self.db_embeddings)[0]
+            best_idx = torch.argmax(cosine_scores).item()
+            retrieved_fact = self.fact_db[best_idx]
+            
+            # 2. Generation: GPT Audit
+            prompt = f"""
+            You are a senior auditor for the FTC (Federal Trade Commission) specialized in deceptive subscription practices.
+            Audit the [Ad/Service Copy] based on the [Relevant Regulation].
+
+            [Relevant Regulation]: {retrieved_fact}
+            [Ad/Service Copy]: {text}
+
+            Response Format (Strictly follow this):
+            Compliance Risk: [Score from 0 to 100, where 100 is a definite violation]
+            Specific Violation: [1-2 sentences explaining why it violates or follows the rule]
+            Verdict: [CRITICAL / SUSPICIOUS / COMPLIANT]
+            """
+            
+            print("   -> 🤖 GPT Auditor is cross-checking with federal regulations...")
+            response = self.client.chat.completions.create(
+                model="gpt-4o", # Using 4o for better reasoning in English
+                messages=[{"role": "user", "content": prompt}],
+                temperature=0.0
+            )
+            
+            result_text = response.choices[0].message.content
+            print(f"   [Audit Report]\n{result_text}")
+            
+            # Extract score
+            score_match = re.search(r"Compliance Risk:\s*(\d+)", result_text)
+            x3_score = float(score_match.group(1)) if score_match else 0.0
+            
+            return x3_score, retrieved_fact
+            
+        except Exception as e:
+            print(f"⚠️ Audit Error: {e}")
+            return 0.0, "An error occurred during regulatory analysis."
+
+if __name__ == "__main__":
+    checker = FactCheckerRAG()
+    # Test case: Typical subscription trap language
+    test_copy = "Get your FREE trial now! $0.00 today. Just enter your credit card to verify your identity."
+    
+    score, fact = checker.calculate_x3_score(test_copy)
+    print("-" * 50)
+    print(f"Final Non-Compliance Score: {score}")
+    print(f"Reference FTC Guideline: {fact}")

myapp/step4_xai.py

@@ -0,0 +1,86 @@
+import numpy as np
+import xgboost as xgb
+import shap
+
+class XAIScorer:
+    def __init__(self):
+        print("📊 [Step 4] Loading the XAI Scoring Engine (XGBoost + SHAP)...")
+        
+        # 1. Generating synthetic data for PoC training
+        # Features: X1 (Lexical Bait), X2 (Semantic Intent), X3 (Regulatory Violation)
+        np.random.seed(42)
+        X_train = np.random.rand(1000, 3) * 100 
+        
+        # Hypothetical labeling logic: 
+        # In subscription traps, Regulatory non-compliance (X3) and Semantic intent (X2) 
+        # carry the most weight in determining a 'Deceptive' label.
+        # Target: 1 (Deceptive/Trap), 0 (Compliant)
+        y_train = ((X_train[:, 0]*0.2 + X_train[:, 1]*0.4 + X_train[:, 2]*0.4) > 50).astype(int)
+
+        # 2. Define and train the XGBoost Classifier
+        # This model acts as the 'decision-making brain' of the auditor.
+        self.model = xgb.XGBClassifier(
+            n_estimators=50, 
+            max_depth=3, 
+            learning_rate=0.1, 
+            eval_metric='logloss',
+            random_state=42
+        )
+        self.model.fit(X_train, y_train)
+        
+        # 3. Initialize the SHAP TreeExplainer
+        # This allows us to see exactly how each feature influenced the final risk score.
+        self.explainer = shap.TreeExplainer(self.model)
+        print("   -> 🧠 Machine-learning scoring engine is ready (Global Version).")
+
+    def calculate_final_score_and_explain(self, x1, x2, x3):
+        """
+        Calculates the probability of being a Dark Pattern and provides an XAI trace.
+        x1: Lexical Score, x2: Semantic Score, x3: RAG Compliance Score
+        """
+        # Convert input scores into a numpy array for the model
+        X_input = np.array([[x1, x2, x3]])
+        
+        # =========================================================
+        # 1. Final Risk Scoring
+        # Predict_proba returns the probability of class 1 (Deceptive)
+        # =========================================================
+        probabilities = self.model.predict_proba(X_input)
+        final_score = probabilities[0][1] * 100  # Scale to 0-100 range
+        
+        # =========================================================
+        # 2. XAI (SHAP Value Calculation)
+        # SHAP explains the deviation from the base value (average prediction).
+        # =========================================================
+        shap_values = self.explainer.shap_values(X_input)
+        
+        # Handle different SHAP output formats (list or array)
+        if isinstance(shap_values, list):
+            # For binary classification, we take the values for the positive class (1)
+            shap_vals = shap_values[1][0] 
+        elif len(shap_values.shape) == 3: # Some versions return (obs, class, features)
+            shap_vals = shap_values[0][1]
+        else:
+            shap_vals = shap_values[0]
+
+        # Extract the Base Value (The average risk score of the training set)
+        if isinstance(self.explainer.expected_value, (list, np.ndarray)):
+            base_value = self.explainer.expected_value[1]
+        else:
+            base_value = self.explainer.expected_value
+
+        # Note: final_score ≈ (base_value + sum(shap_vals)) * 100
+        return final_score, shap_vals, base_value
+
+if __name__ == "__main__":
+    scorer = XAIScorer()
+    # Example: A page with some lexical bait but high semantic deception and regulatory failure
+    f_score, s_vals, b_val = scorer.calculate_final_score_and_explain(30.0, 75.0, 85.0)
+    
+    print("\n" + "="*50)
+    print(f"Audit Result: {f_score:.2f}% Deception Probability")
+    print("-" * 50)
+    print(f"Lexical Impact: {s_vals[0]:+.2f}")
+    print(f"Semantic Impact: {s_vals[1]:+.2f}")
+    print(f"Regulatory Impact: {s_vals[2]:+.2f}")
+    print("="*50)

packages.txt

@@ -0,0 +1,7 @@
+ffmpeg
+libgl1
+libglib2.0-0
+libsm6
+libxext6
+libxrender-dev
+libgomp1

requirements.txt

@@ -0,0 +1,16 @@
+fastapi
+uvicorn
+pydantic
+requests
+playwright
+python-dotenv
+openai
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch
+sentence-transformers
+transformers
+python-mecab-ko
+numpy
+xgboost==1.7.6
+shap
+paddleocr