| |
| """ |
| File Upload Interface |
| ==================== |
| Web-based file upload interface for high capacity input processing. |
| """ |
|
|
| from flask import Flask, request, jsonify, render_template_string, redirect, url_for |
| import os |
| import json |
| from pathlib import Path |
| from werkzeug.utils import secure_filename |
| from high_capacity_input_processor import HighCapacityInputProcessor |
| import threading |
| import time |
|
|
| app = Flask(__name__) |
| app.config['MAX_CONTENT_LENGTH'] = 100 * 1024 * 1024 |
|
|
| |
| processor = HighCapacityInputProcessor() |
|
|
| |
| UPLOAD_TEMPLATE = """ |
| <!DOCTYPE html> |
| <html lang="en"> |
| <head> |
| <meta charset="UTF-8"> |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> |
| <title>LiMp High Capacity Input Processor</title> |
| <style> |
| body { |
| font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; |
| max-width: 1200px; |
| margin: 0 auto; |
| padding: 20px; |
| background-color: #f5f5f5; |
| } |
| .container { |
| background: white; |
| padding: 30px; |
| border-radius: 10px; |
| box-shadow: 0 2px 10px rgba(0,0,0,0.1); |
| } |
| .header { |
| text-align: center; |
| margin-bottom: 30px; |
| } |
| .header h1 { |
| color: #333; |
| margin-bottom: 10px; |
| } |
| .header p { |
| color: #666; |
| font-size: 16px; |
| } |
| .upload-section { |
| border: 2px dashed #ddd; |
| border-radius: 10px; |
| padding: 40px; |
| text-align: center; |
| margin-bottom: 30px; |
| transition: border-color 0.3s; |
| } |
| .upload-section:hover { |
| border-color: #4CAF50; |
| } |
| .upload-section.dragover { |
| border-color: #4CAF50; |
| background-color: #f0f8f0; |
| } |
| .file-input { |
| margin: 20px 0; |
| } |
| .file-input input[type="file"] { |
| display: none; |
| } |
| .file-input label { |
| display: inline-block; |
| padding: 12px 24px; |
| background-color: #4CAF50; |
| color: white; |
| border-radius: 5px; |
| cursor: pointer; |
| font-size: 16px; |
| transition: background-color 0.3s; |
| } |
| .file-input label:hover { |
| background-color: #45a049; |
| } |
| .text-input { |
| margin: 20px 0; |
| } |
| .text-input textarea { |
| width: 100%; |
| height: 200px; |
| padding: 15px; |
| border: 1px solid #ddd; |
| border-radius: 5px; |
| font-family: monospace; |
| font-size: 14px; |
| resize: vertical; |
| } |
| .submit-btn { |
| background-color: #2196F3; |
| color: white; |
| padding: 15px 30px; |
| border: none; |
| border-radius: 5px; |
| font-size: 16px; |
| cursor: pointer; |
| margin: 10px 5px; |
| transition: background-color 0.3s; |
| } |
| .submit-btn:hover { |
| background-color: #1976D2; |
| } |
| .submit-btn:disabled { |
| background-color: #ccc; |
| cursor: not-allowed; |
| } |
| .progress { |
| width: 100%; |
| height: 20px; |
| background-color: #f0f0f0; |
| border-radius: 10px; |
| overflow: hidden; |
| margin: 20px 0; |
| display: none; |
| } |
| .progress-bar { |
| height: 100%; |
| background-color: #4CAF50; |
| width: 0%; |
| transition: width 0.3s; |
| } |
| .results { |
| margin-top: 30px; |
| padding: 20px; |
| background-color: #f9f9f9; |
| border-radius: 5px; |
| display: none; |
| } |
| .stats { |
| display: grid; |
| grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); |
| gap: 20px; |
| margin: 20px 0; |
| } |
| .stat-card { |
| background: white; |
| padding: 20px; |
| border-radius: 5px; |
| text-align: center; |
| box-shadow: 0 2px 5px rgba(0,0,0,0.1); |
| } |
| .stat-number { |
| font-size: 24px; |
| font-weight: bold; |
| color: #4CAF50; |
| } |
| .stat-label { |
| color: #666; |
| margin-top: 5px; |
| } |
| .file-list { |
| margin-top: 20px; |
| } |
| .file-item { |
| background: white; |
| padding: 15px; |
| margin: 10px 0; |
| border-radius: 5px; |
| box-shadow: 0 2px 5px rgba(0,0,0,0.1); |
| } |
| .file-name { |
| font-weight: bold; |
| color: #333; |
| } |
| .file-info { |
| color: #666; |
| font-size: 14px; |
| margin-top: 5px; |
| } |
| .chunk-info { |
| color: #888; |
| font-size: 12px; |
| margin-top: 5px; |
| } |
| .error { |
| color: #f44336; |
| background-color: #ffebee; |
| padding: 15px; |
| border-radius: 5px; |
| margin: 20px 0; |
| } |
| .success { |
| color: #4CAF50; |
| background-color: #e8f5e8; |
| padding: 15px; |
| border-radius: 5px; |
| margin: 20px 0; |
| } |
| .download-btn { |
| background-color: #FF9800; |
| color: white; |
| padding: 8px 16px; |
| border: none; |
| border-radius: 3px; |
| font-size: 12px; |
| cursor: pointer; |
| margin-top: 10px; |
| } |
| .download-btn:hover { |
| background-color: #F57C00; |
| } |
| </style> |
| </head> |
| <body> |
| <div class="container"> |
| <div class="header"> |
| <h1>🧠 LiMp High Capacity Input Processor</h1> |
| <p>Upload files or enter large text for intelligent chunking and training data generation</p> |
| </div> |
| |
| <form id="uploadForm" enctype="multipart/form-data"> |
| <div class="upload-section" id="uploadSection"> |
| <h3>📁 File Upload</h3> |
| <p>Drag and drop files here or click to select</p> |
| <div class="file-input"> |
| <label for="fileInput">Choose Files</label> |
| <input type="file" id="fileInput" name="files" multiple accept=".txt,.md,.py,.js,.html,.css,.json,.jsonl,.csv,.pdf,.doc,.docx,.xml,.yaml,.yml"> |
| </div> |
| <p><small>Supported formats: TXT, MD, PY, JS, HTML, CSS, JSON, CSV, PDF, DOC, DOCX, XML, YAML</small></p> |
| </div> |
| |
| <div class="text-input"> |
| <h3>📝 Large Text Input</h3> |
| <textarea id="textInput" name="text" placeholder="Enter large text content here (up to 100MB)..."></textarea> |
| <p><small>Character count: <span id="charCount">0</span></small></p> |
| </div> |
| |
| <div> |
| <button type="submit" class="submit-btn" id="submitBtn">Process Input</button> |
| <button type="button" class="submit-btn" onclick="generateTrainingData()">Generate Training Data</button> |
| <button type="button" class="submit-btn" onclick="clearAll()">Clear All</button> |
| </div> |
| |
| <div class="progress" id="progress"> |
| <div class="progress-bar" id="progressBar"></div> |
| </div> |
| </form> |
| |
| <div class="results" id="results"> |
| <h3>📊 Processing Results</h3> |
| <div id="resultsContent"></div> |
| </div> |
| </div> |
| |
| <script> |
| let uploads = []; |
| |
| // File input handling |
| const fileInput = document.getElementById('fileInput'); |
| const uploadSection = document.getElementById('uploadSection'); |
| const textInput = document.getElementById('textInput'); |
| const charCount = document.getElementById('charCount'); |
| const submitBtn = document.getElementById('submitBtn'); |
| const progress = document.getElementById('progress'); |
| const progressBar = document.getElementById('progressBar'); |
| const results = document.getElementById('results'); |
| const resultsContent = document.getElementById('resultsContent'); |
| |
| // Character count update |
| textInput.addEventListener('input', function() { |
| charCount.textContent = this.value.length.toLocaleString(); |
| }); |
| |
| // Drag and drop handling |
| uploadSection.addEventListener('dragover', function(e) { |
| e.preventDefault(); |
| this.classList.add('dragover'); |
| }); |
| |
| uploadSection.addEventListener('dragleave', function(e) { |
| e.preventDefault(); |
| this.classList.remove('dragover'); |
| }); |
| |
| uploadSection.addEventListener('drop', function(e) { |
| e.preventDefault(); |
| this.classList.remove('dragover'); |
| fileInput.files = e.dataTransfer.files; |
| updateFileList(); |
| }); |
| |
| fileInput.addEventListener('change', updateFileList); |
| |
| function updateFileList() { |
| const files = fileInput.files; |
| if (files.length > 0) { |
| let fileList = '<h4>Selected Files:</h4>'; |
| for (let file of files) { |
| fileList += `<div class="file-item"> |
| <div class="file-name">${file.name}</div> |
| <div class="file-info">Size: ${(file.size / 1024 / 1024).toFixed(2)} MB, Type: ${file.type}</div> |
| </div>`; |
| } |
| uploadSection.innerHTML = fileList + uploadSection.innerHTML; |
| } |
| } |
| |
| // Form submission |
| document.getElementById('uploadForm').addEventListener('submit', async function(e) { |
| e.preventDefault(); |
| |
| const formData = new FormData(); |
| const files = fileInput.files; |
| const text = textInput.value.trim(); |
| |
| if (files.length === 0 && text === '') { |
| alert('Please select files or enter text content'); |
| return; |
| } |
| |
| // Add files |
| for (let file of files) { |
| formData.append('files', file); |
| } |
| |
| // Add text |
| if (text) { |
| formData.append('text', text); |
| } |
| |
| submitBtn.disabled = true; |
| progress.style.display = 'block'; |
| progressBar.style.width = '0%'; |
| |
| try { |
| // Simulate progress |
| let progressValue = 0; |
| const progressInterval = setInterval(() => { |
| progressValue += Math.random() * 15; |
| if (progressValue > 90) progressValue = 90; |
| progressBar.style.width = progressValue + '%'; |
| }, 200); |
| |
| const response = await fetch('/upload', { |
| method: 'POST', |
| body: formData |
| }); |
| |
| clearInterval(progressInterval); |
| progressBar.style.width = '100%'; |
| |
| const result = await response.json(); |
| |
| if (result.success) { |
| uploads = result.uploads || []; |
| showResults(result); |
| } else { |
| showError(result.error); |
| } |
| |
| } catch (error) { |
| showError('Upload failed: ' + error.message); |
| } finally { |
| submitBtn.disabled = false; |
| setTimeout(() => { |
| progress.style.display = 'none'; |
| progressBar.style.width = '0%'; |
| }, 1000); |
| } |
| }); |
| |
| async function generateTrainingData() { |
| if (uploads.length === 0) { |
| alert('No uploads available. Please upload files or enter text first.'); |
| return; |
| } |
| |
| try { |
| const response = await fetch('/generate_training_data', { |
| method: 'POST', |
| headers: { |
| 'Content-Type': 'application/json', |
| }, |
| body: JSON.stringify({ file_ids: uploads.map(u => u.file_id) }) |
| }); |
| |
| const result = await response.json(); |
| |
| if (result.success) { |
| showSuccess(`Training data generated: ${result.training_data_file}`); |
| // Add download link |
| const downloadLink = document.createElement('a'); |
| downloadLink.href = `/download/${result.training_data_file}`; |
| downloadLink.textContent = 'Download Training Data'; |
| downloadLink.className = 'download-btn'; |
| resultsContent.appendChild(downloadLink); |
| } else { |
| showError(result.error); |
| } |
| |
| } catch (error) { |
| showError('Training data generation failed: ' + error.message); |
| } |
| } |
| |
| function showResults(data) { |
| let html = '<div class="success">Processing completed successfully!</div>'; |
| |
| // Statistics |
| html += '<div class="stats">'; |
| html += `<div class="stat-card"> |
| <div class="stat-number">${data.stats.total_files}</div> |
| <div class="stat-label">Files Processed</div> |
| </div>`; |
| html += `<div class="stat-card"> |
| <div class="stat-number">${data.stats.total_chunks}</div> |
| <div class="stat-label">Chunks Created</div> |
| </div>`; |
| html += `<div class="stat-card"> |
| <div class="stat-number">${data.stats.total_size_mb.toFixed(2)}</div> |
| <div class="stat-label">Total Size (MB)</div> |
| </div>`; |
| html += '</div>'; |
| |
| // File list |
| if (data.uploads && data.uploads.length > 0) { |
| html += '<div class="file-list">'; |
| html += '<h4>Processed Files:</h4>'; |
| for (let upload of data.uploads) { |
| html += `<div class="file-item"> |
| <div class="file-name">${upload.filename}</div> |
| <div class="file-info">Size: ${(upload.file_size / 1024 / 1024).toFixed(2)} MB, Type: ${upload.mime_type}</div> |
| <div class="chunk-info">Chunks: ${upload.chunks.length}</div> |
| </div>`; |
| } |
| html += '</div>'; |
| } |
| |
| resultsContent.innerHTML = html; |
| results.style.display = 'block'; |
| } |
| |
| function showError(message) { |
| resultsContent.innerHTML = `<div class="error">Error: ${message}</div>`; |
| results.style.display = 'block'; |
| } |
| |
| function showSuccess(message) { |
| resultsContent.innerHTML = `<div class="success">${message}</div>`; |
| results.style.display = 'block'; |
| } |
| |
| function clearAll() { |
| fileInput.value = ''; |
| textInput.value = ''; |
| charCount.textContent = '0'; |
| uploads = []; |
| results.style.display = 'none'; |
| uploadSection.innerHTML = ` |
| <h3>📁 File Upload</h3> |
| <p>Drag and drop files here or click to select</p> |
| <div class="file-input"> |
| <label for="fileInput">Choose Files</label> |
| <input type="file" id="fileInput" name="files" multiple accept=".txt,.md,.py,.js,.html,.css,.json,.jsonl,.csv,.pdf,.doc,.docx,.xml,.yaml,.yml"> |
| </div> |
| <p><small>Supported formats: TXT, MD, PY, JS, HTML, CSS, JSON, CSV, PDF, DOC, DOCX, XML, YAML</small></p> |
| `; |
| } |
| </script> |
| </body> |
| </html> |
| """ |
|
|
| @app.route('/') |
| def index(): |
| """Main upload page.""" |
| return render_template_string(UPLOAD_TEMPLATE) |
|
|
| @app.route('/upload', methods=['POST']) |
| def upload_files(): |
| """Handle file uploads and text input.""" |
| try: |
| uploads = [] |
| |
| |
| if 'files' in request.files: |
| files = request.files.getlist('files') |
| for file in files: |
| if file.filename: |
| |
| filename = secure_filename(file.filename) |
| temp_path = Path(processor.upload_dir) / filename |
| file.save(str(temp_path)) |
| |
| |
| file_upload = processor.process_file_upload(temp_path) |
| uploads.append({ |
| 'file_id': file_upload.file_id, |
| 'filename': file_upload.filename, |
| 'file_size': file_upload.file_size, |
| 'mime_type': file_upload.mime_type, |
| 'chunks': len(file_upload.chunks) |
| }) |
| |
| |
| text_content = request.form.get('text', '').strip() |
| if text_content: |
| chunks = processor.process_high_capacity_input(text_content) |
| uploads.append({ |
| 'file_id': 'text_input_' + str(int(time.time())), |
| 'filename': 'text_input.txt', |
| 'file_size': len(text_content), |
| 'mime_type': 'text/plain', |
| 'chunks': len(chunks) |
| }) |
| |
| |
| stats = processor.get_processing_stats() |
| |
| return jsonify({ |
| 'success': True, |
| 'uploads': uploads, |
| 'stats': stats |
| }) |
| |
| except Exception as e: |
| return jsonify({ |
| 'success': False, |
| 'error': str(e) |
| }), 400 |
|
|
| @app.route('/generate_training_data', methods=['POST']) |
| def generate_training_data(): |
| """Generate training data from processed uploads.""" |
| try: |
| data = request.get_json() |
| file_ids = data.get('file_ids', []) |
| |
| |
| all_uploads = processor.get_all_uploads() |
| |
| |
| if file_ids: |
| filtered_uploads = [upload for upload in all_uploads if upload.file_id in file_ids] |
| else: |
| filtered_uploads = all_uploads |
| |
| |
| training_data_file = processor.create_training_data_from_chunks( |
| filtered_uploads, |
| output_format='jsonl', |
| include_metadata=True |
| ) |
| |
| return jsonify({ |
| 'success': True, |
| 'training_data_file': Path(training_data_file).name, |
| 'file_path': training_data_file, |
| 'total_examples': sum(len(upload.chunks) for upload in filtered_uploads) |
| }) |
| |
| except Exception as e: |
| return jsonify({ |
| 'success': False, |
| 'error': str(e) |
| }), 400 |
|
|
| @app.route('/download/<filename>') |
| def download_file(filename): |
| """Download generated training data file.""" |
| file_path = processor.training_data_dir / filename |
| if file_path.exists(): |
| return send_file(str(file_path), as_attachment=True) |
| else: |
| return "File not found", 404 |
|
|
| @app.route('/stats') |
| def get_stats(): |
| """Get processing statistics.""" |
| stats = processor.get_processing_stats() |
| return jsonify(stats) |
|
|
| @app.route('/uploads') |
| def list_uploads(): |
| """List all uploads.""" |
| uploads = processor.get_all_uploads() |
| return jsonify([{ |
| 'file_id': upload.file_id, |
| 'filename': upload.filename, |
| 'file_size': upload.file_size, |
| 'mime_type': upload.mime_type, |
| 'upload_timestamp': upload.upload_timestamp, |
| 'chunks': len(upload.chunks) |
| } for upload in uploads]) |
|
|
| if __name__ == '__main__': |
| print("🚀 Starting LiMp High Capacity Input Processor") |
| print("📁 Upload directory:", processor.upload_dir) |
| print("📁 Chunk directory:", processor.chunk_dir) |
| print("📁 Training data directory:", processor.training_data_dir) |
| print("🌐 Web interface: http://localhost:5000") |
| |
| app.run(debug=True, host='0.0.0.0', port=5000) |
|
|
