ozsparkhub commited on
Commit
92fea71
·
verified ·
1 Parent(s): 86d0fd9

Create PDF table extraction API

Browse files
Files changed (5) hide show
  1. .gitignore +18 -0
  2. Dockerfile +21 -0
  3. README.md +98 -5
  4. app.py +319 -0
  5. requirements.txt +8 -0
.gitignore ADDED
@@ -0,0 +1,18 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ __pycache__/
2
+ *.py[cod]
3
+ *$py.class
4
+ *.so
5
+ .Python
6
+ *.egg-info/
7
+ dist/
8
+ build/
9
+ *.egg
10
+ .env
11
+ .venv
12
+ venv/
13
+ *.log
14
+ *.tmp
15
+ *.xlsx
16
+ *.csv
17
+ *.pdf
18
+ flagged/
Dockerfile ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.10-slim
2
+
3
+ WORKDIR /app
4
+
5
+ # Install system dependencies
6
+ RUN apt-get update && apt-get install -y \
7
+ build-essential \
8
+ && rm -rf /var/lib/apt/lists/*
9
+
10
+ # Copy requirements and install Python packages
11
+ COPY requirements.txt .
12
+ RUN pip install --no-cache-dir -r requirements.txt
13
+
14
+ # Copy application code
15
+ COPY app.py .
16
+
17
+ # Expose port
18
+ EXPOSE 7860
19
+
20
+ # Run FastAPI with uvicorn
21
+ CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]
README.md CHANGED
@@ -1,10 +1,103 @@
1
  ---
2
- title: Pdf Table Extract Api
3
- emoji: 🦀
4
- colorFrom: purple
5
- colorTo: gray
6
  sdk: docker
 
7
  pinned: false
 
8
  ---
9
 
10
- Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  ---
2
+ title: PDF to Excel Converter API
3
+ emoji: 📊
4
+ colorFrom: green
5
+ colorTo: blue
6
  sdk: docker
7
+ app_port: 7860
8
  pinned: false
9
+ license: mit
10
  ---
11
 
12
+ # PDF to Excel Converter API
13
+
14
+ Extract tables from text-based PDF files to Excel (XLSX) or CSV format.
15
+
16
+ ## Features
17
+
18
+ ✅ **Direct Table Extraction** - Uses pdfplumber first for text-based PDFs
19
+ ✅ **Dual Format Support** - Export as Excel (.xlsx) or CSV (.csv)
20
+ ✅ **Multiple Tables** - Creates separate sheets for each table (XLSX)
21
+ ✅ **Markdown Fallback** - Falls back to Markdown table parsing when direct extraction is not enough
22
+ ✅ **Free & Unlimited** - No API limits or costs
23
+ ✅ **100% Privacy** - Files processed temporarily, not stored
24
+
25
+ ## How It Works
26
+
27
+ 1. **PDF → Tables** - pdfplumber extracts rows and columns from text-based PDFs
28
+ 2. **Fallback → Markdown Tables** - A document parser is used when direct extraction finds no tables
29
+ 3. **Tables → Excel/CSV** - pandas creates spreadsheet files
30
+
31
+ ## API Usage
32
+
33
+ ### JavaScript/TypeScript
34
+
35
+ ```javascript
36
+ const API_URL = "https://ozsparkhub-pdf-to-excel-api.hf.space";
37
+
38
+ async function convertPdfToExcel(pdfFile, format = "xlsx") {
39
+ const formData = new FormData();
40
+ formData.append('file', pdfFile);
41
+ formData.append('output_format', format);
42
+
43
+ const response = await fetch(`${API_URL}/call/convert`, {
44
+ method: 'POST',
45
+ body: formData
46
+ });
47
+
48
+ const result = await response.json();
49
+ return result.file_data; // base64 Excel/CSV data
50
+ }
51
+ ```
52
+
53
+ ### Python
54
+
55
+ ```python
56
+ import requests
57
+
58
+ with open("path/to/file.pdf", "rb") as pdf:
59
+ response = requests.post(
60
+ "https://ozsparkhub-pdf-to-excel-api.hf.space/call/convert",
61
+ files={"file": pdf},
62
+ data={"output_format": "xlsx"},
63
+ )
64
+
65
+ print(response.json())
66
+ ```
67
+
68
+ ## Technical Stack
69
+
70
+ - **pdfplumber** - Direct table extraction for text-based PDFs
71
+ - **Document parser fallback** - Markdown table fallback
72
+ - **pandas** - Data manipulation and Excel generation
73
+ - **openpyxl** - Excel file creation engine
74
+ - **FastAPI** - API interface
75
+
76
+ ## Use Cases
77
+
78
+ 📊 **Data Extraction** - Extract financial tables from PDF reports
79
+ 💼 **Business Analytics** - Convert invoices and statements to Excel
80
+ 📈 **Research Data** - Extract scientific tables for analysis
81
+ 🏢 **Document Processing** - Automate PDF table extraction workflows
82
+ 📑 **Report Generation** - Convert PDF reports to editable spreadsheets
83
+
84
+ ## Limitations
85
+
86
+ - Maximum file size: 10MB
87
+ - Works best with text-based PDFs, not scanned/image-only documents
88
+ - Complex merged cells may need manual adjustment
89
+ - Borderless or heavily designed tables may need manual cleanup
90
+
91
+ ## Credits
92
+
93
+ Created by [OzSparkHub Business Tools](https://biz.ozsparkhub.com.au)
94
+
95
+ Powered by:
96
+ - [pdfplumber](https://github.com/jsvine/pdfplumber)
97
+ - [pandas](https://pandas.pydata.org/)
98
+ - [openpyxl](https://openpyxl.readthedocs.io/)
99
+ - [FastAPI](https://fastapi.tiangolo.com/)
100
+
101
+ ## License
102
+
103
+ MIT License - Free for commercial and personal use
app.py ADDED
@@ -0,0 +1,319 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ PDF to Excel table extraction API.
3
+ FastAPI version - No Gradio needed.
4
+ """
5
+
6
+ from fastapi import FastAPI, File, UploadFile, Form
7
+ from fastapi.middleware.cors import CORSMiddleware
8
+ from markitdown import MarkItDown as DocumentConverter
9
+ import pandas as pd
10
+ import pdfplumber
11
+ import tempfile
12
+ import os
13
+ import base64
14
+ import PyPDF2
15
+ import re
16
+
17
+ app = FastAPI(title="PDF to Excel Converter API")
18
+
19
+ # Enable CORS
20
+ app.add_middleware(
21
+ CORSMiddleware,
22
+ allow_origins=["*"],
23
+ allow_credentials=True,
24
+ allow_methods=["*"],
25
+ allow_headers=["*"],
26
+ )
27
+
28
+ md_converter = DocumentConverter()
29
+ MIN_TEXT_CHARS_PER_PAGE = 40
30
+
31
+ def clean_text(text):
32
+ """Remove NULL bytes and control characters"""
33
+ if not text:
34
+ return text
35
+ # Remove NULL bytes
36
+ text = text.replace('\x00', '')
37
+ # Remove other control characters except newline, tab, and carriage return
38
+ text = re.sub(r'[\x01-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F]', '', text)
39
+ return text
40
+
41
+ def normalize_cell(value):
42
+ """Normalize table cell values from PDF extraction."""
43
+ if value is None:
44
+ return ""
45
+ return clean_text(str(value)).strip()
46
+
47
+ def dedupe_headers(headers):
48
+ """Ensure spreadsheet column names are present and unique."""
49
+ used = {}
50
+ output = []
51
+
52
+ for index, header in enumerate(headers):
53
+ base = normalize_cell(header) or f"Column_{index + 1}"
54
+ count = used.get(base, 0)
55
+ used[base] = count + 1
56
+ output.append(base if count == 0 else f"{base}_{count + 1}")
57
+
58
+ return output
59
+
60
+ def dataframe_from_rows(rows):
61
+ """Convert extracted raw rows into a clean DataFrame."""
62
+ cleaned_rows = [
63
+ [normalize_cell(cell) for cell in row]
64
+ for row in rows
65
+ if row and any(normalize_cell(cell) for cell in row)
66
+ ]
67
+
68
+ if len(cleaned_rows) < 2:
69
+ return None
70
+
71
+ max_width = max(len(row) for row in cleaned_rows)
72
+ normalized_rows = [row + [""] * (max_width - len(row)) for row in cleaned_rows]
73
+
74
+ # Drop columns that are empty from top to bottom.
75
+ useful_columns = [
76
+ index
77
+ for index in range(max_width)
78
+ if any(row[index] for row in normalized_rows)
79
+ ]
80
+ normalized_rows = [[row[index] for index in useful_columns] for row in normalized_rows]
81
+
82
+ if not normalized_rows or len(normalized_rows[0]) < 2:
83
+ return None
84
+
85
+ headers = dedupe_headers(normalized_rows[0])
86
+ data_rows = normalized_rows[1:]
87
+
88
+ if not any(any(cell for cell in row) for row in data_rows):
89
+ return None
90
+
91
+ return pd.DataFrame(data_rows, columns=headers)
92
+
93
+ def unique_tables(tables):
94
+ """Remove duplicate tables returned by multiple extraction strategies."""
95
+ unique = []
96
+ fingerprints = set()
97
+
98
+ for df in tables:
99
+ if df is None or df.empty:
100
+ continue
101
+
102
+ fingerprint = df.to_csv(index=False)
103
+ if fingerprint in fingerprints:
104
+ continue
105
+
106
+ fingerprints.add(fingerprint)
107
+ unique.append(df)
108
+
109
+ return unique
110
+
111
+ def extract_tables_with_pdfplumber(pdf_path):
112
+ """Extract tables directly from text-based PDFs before falling back to Markdown."""
113
+ tables = []
114
+ text_chars = 0
115
+
116
+ table_settings = [
117
+ {
118
+ "vertical_strategy": "lines",
119
+ "horizontal_strategy": "lines",
120
+ "snap_tolerance": 3,
121
+ "join_tolerance": 3,
122
+ "intersection_tolerance": 3,
123
+ },
124
+ {
125
+ "vertical_strategy": "text",
126
+ "horizontal_strategy": "text",
127
+ "snap_tolerance": 3,
128
+ "join_tolerance": 3,
129
+ "intersection_tolerance": 5,
130
+ "min_words_vertical": 2,
131
+ "min_words_horizontal": 1,
132
+ },
133
+ ]
134
+
135
+ with pdfplumber.open(pdf_path) as pdf:
136
+ for page in pdf.pages:
137
+ text_chars += len(page.extract_text() or "")
138
+
139
+ for settings in table_settings:
140
+ for raw_table in page.extract_tables(table_settings=settings) or []:
141
+ df = dataframe_from_rows(raw_table)
142
+ if df is not None:
143
+ tables.append(df)
144
+
145
+ return unique_tables(tables), text_chars
146
+
147
+ def extract_tables_from_markdown(markdown_text):
148
+ """Extract tables from Markdown text"""
149
+ tables = []
150
+ lines = markdown_text.split('\n')
151
+
152
+ current_table = []
153
+ in_table = False
154
+
155
+ for line in lines:
156
+ if '|' in line and line.strip():
157
+ in_table = True
158
+ current_table.append(line)
159
+ else:
160
+ if in_table and current_table:
161
+ df = parse_markdown_table(current_table)
162
+ if df is not None and not df.empty:
163
+ tables.append(df)
164
+ current_table = []
165
+ in_table = False
166
+
167
+ if current_table:
168
+ df = parse_markdown_table(current_table)
169
+ if df is not None and not df.empty:
170
+ tables.append(df)
171
+
172
+ return tables
173
+
174
+ def parse_markdown_table(table_lines):
175
+ """Parse markdown table lines into DataFrame"""
176
+ if len(table_lines) < 2:
177
+ return None
178
+
179
+ table_lines = [line.strip() for line in table_lines if line.strip()]
180
+
181
+ # Extract header
182
+ header_line = table_lines[0]
183
+ headers = [cell.strip() for cell in header_line.split('|')[1:-1]]
184
+
185
+ # Skip separator line
186
+ data_lines = table_lines[2:] if len(table_lines) > 2 else []
187
+
188
+ # Extract data rows
189
+ rows = []
190
+ for line in data_lines:
191
+ cells = [cell.strip() for cell in line.split('|')[1:-1]]
192
+ if len(cells) == len(headers):
193
+ rows.append(cells)
194
+
195
+ if not rows:
196
+ return None
197
+
198
+ df = pd.DataFrame(rows, columns=headers)
199
+ return df
200
+
201
+ def estimate_text_chars(pdf_path):
202
+ """Estimate whether a PDF contains selectable text."""
203
+ text_chars = 0
204
+ with open(pdf_path, 'rb') as f:
205
+ pdf_reader = PyPDF2.PdfReader(f)
206
+ for page in pdf_reader.pages:
207
+ text_chars += len(page.extract_text() or "")
208
+ return text_chars
209
+
210
+ @app.post("/call/convert")
211
+ async def convert_pdf_to_excel(file: UploadFile = File(...), output_format: str = Form("xlsx")):
212
+ """Convert PDF to Excel - compatible with Gradio API format"""
213
+
214
+ try:
215
+ # Save uploaded file temporarily
216
+ with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as tmp_pdf:
217
+ content = await file.read()
218
+ tmp_pdf.write(content)
219
+ pdf_path = tmp_pdf.name
220
+
221
+ # Check page limit
222
+ with open(pdf_path, 'rb') as f:
223
+ pdf_reader = PyPDF2.PdfReader(f)
224
+ num_pages = len(pdf_reader.pages)
225
+
226
+ if num_pages > 10:
227
+ os.remove(pdf_path)
228
+ return {
229
+ "event_id": "error",
230
+ "message": f"📄 Your PDF has {num_pages} pages. Our free plan converts up to 10 pages at a time.\n\n💎 Want unlimited pages? Pro plan coming soon! Split your PDF or try a smaller file for now."
231
+ }
232
+
233
+ # Try direct table extraction first. This is stronger for many
234
+ # text-based invoices, bank statements, and reports than Markdown.
235
+ tables, text_chars = extract_tables_with_pdfplumber(pdf_path)
236
+
237
+ if not tables:
238
+ # Fall back to a document parser for PDFs that emit clean Markdown tables.
239
+ result = md_converter.convert(pdf_path)
240
+ markdown_text = clean_text(result.text_content or "")
241
+ tables = extract_tables_from_markdown(markdown_text)
242
+
243
+ if text_chars < max(MIN_TEXT_CHARS_PER_PAGE * num_pages, 1):
244
+ os.remove(pdf_path)
245
+ return {
246
+ "event_id": "error",
247
+ "message": "This PDF looks like a scanned or image-only document, so we cannot reliably extract tables yet.\n\nTry a text-based PDF, export the original file again, or run OCR before uploading."
248
+ }
249
+
250
+ if not tables:
251
+ os.remove(pdf_path)
252
+ return {
253
+ "event_id": "error",
254
+ "message": "No convertible tables were found.\n\nThis tool works best with text-based PDFs that contain clear rows and columns. Borderless tables, merged cells, and heavily designed reports may need manual cleanup."
255
+ }
256
+
257
+ # Convert to Excel or CSV
258
+ if output_format == "xlsx":
259
+ output_path = tempfile.NamedTemporaryFile(delete=False, suffix='.xlsx').name
260
+ with pd.ExcelWriter(output_path, engine='openpyxl') as writer:
261
+ for i, df in enumerate(tables):
262
+ sheet_name = f'Table_{i+1}'
263
+ df.to_excel(writer, sheet_name=sheet_name, index=False)
264
+ else: # csv
265
+ output_path = tempfile.NamedTemporaryFile(delete=False, suffix='.csv').name
266
+ if len(tables) == 1:
267
+ tables[0].to_csv(output_path, index=False)
268
+ else:
269
+ combined_df = pd.concat(tables, ignore_index=True)
270
+ combined_df.to_csv(output_path, index=False)
271
+
272
+ # Read file and convert to base64
273
+ with open(output_path, 'rb') as f:
274
+ file_bytes = f.read()
275
+ base64_data = base64.b64encode(file_bytes).decode('utf-8')
276
+
277
+ # Cleanup
278
+ os.remove(pdf_path)
279
+ os.remove(output_path)
280
+
281
+ # Return in Gradio-compatible format
282
+ extension = '.xlsx' if output_format == 'xlsx' else '.csv'
283
+ return {
284
+ "event_id": "success",
285
+ "file_data": base64_data,
286
+ "filename": file.filename.replace('.pdf', extension),
287
+ "table_count": len(tables),
288
+ "row_count": int(sum(len(df.index) for df in tables))
289
+ }
290
+
291
+ except Exception as e:
292
+ error_msg = str(e)
293
+
294
+ # Provide user-friendly error messages
295
+ if "XML compatible" in error_msg or "NULL bytes" in error_msg or "encoding" in error_msg.lower():
296
+ friendly_msg = "😔 Sorry! Your PDF contains special characters that we couldn't process.\n\n✨ Try:\n• Re-saving your PDF from the original document\n• Using a different PDF viewer to export it\n• Contact us if the issue persists"
297
+ elif "password" in error_msg.lower() or "encrypted" in error_msg.lower():
298
+ friendly_msg = "🔒 This PDF is password-protected.\n\n💡 Please remove the password first, then try again!"
299
+ elif "corrupt" in error_msg.lower() or "damaged" in error_msg.lower():
300
+ friendly_msg = "⚠️ Your PDF file appears to be corrupted.\n\n🔧 Try:\n• Re-downloading the original PDF\n• Opening and re-saving it in Adobe Reader"
301
+ else:
302
+ friendly_msg = f"😔 Oops! Something went wrong during conversion.\n\n🛠️ Technical details: {error_msg}\n\n💡 Try uploading a different PDF or contact support if this persists."
303
+
304
+ return {
305
+ "event_id": "error",
306
+ "message": friendly_msg
307
+ }
308
+
309
+ @app.get("/")
310
+ async def root():
311
+ return {
312
+ "message": "PDF to Excel Converter API",
313
+ "powered_by": "table extraction pipeline",
314
+ "endpoint": "/call/convert"
315
+ }
316
+
317
+ if __name__ == "__main__":
318
+ import uvicorn
319
+ uvicorn.run(app, host="0.0.0.0", port=7860)
requirements.txt ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ fastapi==0.115.0
2
+ uvicorn[standard]==0.32.0
3
+ python-multipart==0.0.12
4
+ markitdown==0.0.1a2
5
+ pandas==2.2.3
6
+ openpyxl==3.1.5
7
+ PyPDF2==3.0.1
8
+ pdfplumber==0.11.5