code vulnerability checks
Browse files- .github/workflows/ci.yml +16 -10
- .github/workflows/docker.yml +35 -0
- README.md +12 -0
- poetry.lock +0 -0
- pyproject.toml +4 -1
- src/compilation/latex_compilation.py +4 -4
- src/processing/images_processing.py +6 -4
- src/services/prompt.py +3 -1
- src/telemetry/logging_utils.py +2 -2
.github/workflows/ci.yml
CHANGED
|
@@ -2,9 +2,9 @@ name: CI
|
|
| 2 |
|
| 3 |
on:
|
| 4 |
push:
|
| 5 |
-
branches: [main]
|
| 6 |
pull_request:
|
| 7 |
-
branches: [main]
|
| 8 |
|
| 9 |
jobs:
|
| 10 |
lint:
|
|
@@ -39,7 +39,6 @@ jobs:
|
|
| 39 |
with:
|
| 40 |
commit_message: "style: Automatic formatting via CI (Black + isort)"
|
| 41 |
branch: ${{ github.head_ref || github.ref_name }}
|
| 42 |
-
# Optional: set these to limit what gets committed
|
| 43 |
file_pattern: "*.py"
|
| 44 |
|
| 45 |
- name: Run Mypy
|
|
@@ -52,14 +51,21 @@ jobs:
|
|
| 52 |
echo "Pylint score: $score"
|
| 53 |
awk -v s="$score" 'BEGIN {exit (s >= 9.5) ? 0 : 1}'
|
| 54 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
- name: Build docs
|
| 56 |
working-directory: docs
|
| 57 |
run: poetry run sphinx-build -b html . _build/html
|
| 58 |
|
| 59 |
-
- name: Deploy to GitHub Pages
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
|
|
|
| 2 |
|
| 3 |
on:
|
| 4 |
push:
|
| 5 |
+
branches: [main, dev]
|
| 6 |
pull_request:
|
| 7 |
+
branches: [main, dev]
|
| 8 |
|
| 9 |
jobs:
|
| 10 |
lint:
|
|
|
|
| 39 |
with:
|
| 40 |
commit_message: "style: Automatic formatting via CI (Black + isort)"
|
| 41 |
branch: ${{ github.head_ref || github.ref_name }}
|
|
|
|
| 42 |
file_pattern: "*.py"
|
| 43 |
|
| 44 |
- name: Run Mypy
|
|
|
|
| 51 |
echo "Pylint score: $score"
|
| 52 |
awk -v s="$score" 'BEGIN {exit (s >= 9.5) ? 0 : 1}'
|
| 53 |
|
| 54 |
+
- name: Run Bandit security scan
|
| 55 |
+
run: poetry run bandit -r src/
|
| 56 |
+
|
| 57 |
+
- name: Run Safety vulnerability check (warnings only)
|
| 58 |
+
run: |
|
| 59 |
+
poetry run safety check --full-report || true
|
| 60 |
+
|
| 61 |
- name: Build docs
|
| 62 |
working-directory: docs
|
| 63 |
run: poetry run sphinx-build -b html . _build/html
|
| 64 |
|
| 65 |
+
# - name: Deploy to GitHub Pages
|
| 66 |
+
# if: github.actor != 'nektos/act'
|
| 67 |
+
# uses: peaceiris/actions-gh-pages@v4
|
| 68 |
+
# with:
|
| 69 |
+
# github_token: ${{ secrets.GITHUB_TOKEN }}
|
| 70 |
+
# publish_dir: ./docs/_build/html
|
| 71 |
+
# publish_branch: gh-pages
|
.github/workflows/docker.yml
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Docker CI
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
push:
|
| 5 |
+
branches: [main]
|
| 6 |
+
tags:
|
| 7 |
+
- 'v*' # e.g. v1.0.0
|
| 8 |
+
pull_request:
|
| 9 |
+
branches: [main]
|
| 10 |
+
|
| 11 |
+
jobs:
|
| 12 |
+
docker-build:
|
| 13 |
+
runs-on: ubuntu-latest
|
| 14 |
+
|
| 15 |
+
steps:
|
| 16 |
+
- name: Checkout code
|
| 17 |
+
uses: actions/checkout@v4
|
| 18 |
+
|
| 19 |
+
- name: Set up Docker Buildx
|
| 20 |
+
uses: docker/setup-buildx-action@v3
|
| 21 |
+
|
| 22 |
+
- name: Log in to DockerHub
|
| 23 |
+
uses: docker/login-action@v3
|
| 24 |
+
if: github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')
|
| 25 |
+
with:
|
| 26 |
+
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
| 27 |
+
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
| 28 |
+
|
| 29 |
+
- name: Build Docker image
|
| 30 |
+
uses: docker/build-push-action@v5
|
| 31 |
+
with:
|
| 32 |
+
context: .
|
| 33 |
+
file: Dockerfile
|
| 34 |
+
push: ${{ github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v') }}
|
| 35 |
+
tags: yourusername/ragntex:latest
|
README.md
CHANGED
|
@@ -43,6 +43,18 @@ The model outputs fully formatted compilable LaTeX code following a clear and co
|
|
| 43 |
|
| 44 |
It also incorporates relevant images extracted from the documents, organizing them into LaTeX Beamer-friendly layouts (e.g., *Core Idea 2* and *Core Idea 3* slide formats).
|
| 45 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
### Run docker:
|
| 47 |
```bash
|
| 48 |
docker run \
|
|
|
|
| 43 |
|
| 44 |
It also incorporates relevant images extracted from the documents, organizing them into LaTeX Beamer-friendly layouts (e.g., *Core Idea 2* and *Core Idea 3* slide formats).
|
| 45 |
|
| 46 |
+
### Prerequisites
|
| 47 |
+
|
| 48 |
+
This project requires the LaTeX toolchain, specifically `pdflatex`, to generate PDF presentations.
|
| 49 |
+
|
| 50 |
+
Please install it before running the project:
|
| 51 |
+
|
| 52 |
+
### On Debian/Ubuntu:
|
| 53 |
+
|
| 54 |
+
```bash
|
| 55 |
+
sudo apt-get update && sudo apt-get install -y texlive-latex-base texlive-fonts-recommended texlive-latex-extra
|
| 56 |
+
|
| 57 |
+
|
| 58 |
### Run docker:
|
| 59 |
```bash
|
| 60 |
docker run \
|
poetry.lock
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
pyproject.toml
CHANGED
|
@@ -21,7 +21,8 @@ dependencies = [
|
|
| 21 |
"pymupdf (>=1.25.5,<2.0.0)",
|
| 22 |
"gradio (>=5.27.1,<6.0.0)",
|
| 23 |
"langfuse (>=2.60.5,<3.0.0)",
|
| 24 |
-
"pydantic (>=2.
|
|
|
|
| 25 |
]
|
| 26 |
|
| 27 |
|
|
@@ -35,6 +36,8 @@ black = "^25.1.0"
|
|
| 35 |
pytest = "^8.4.0"
|
| 36 |
mypy = "^1.16.0"
|
| 37 |
isort = "^6.0.1"
|
|
|
|
|
|
|
| 38 |
|
| 39 |
|
| 40 |
[tool.poetry.group.docs.dependencies]
|
|
|
|
| 21 |
"pymupdf (>=1.25.5,<2.0.0)",
|
| 22 |
"gradio (>=5.27.1,<6.0.0)",
|
| 23 |
"langfuse (>=2.60.5,<3.0.0)",
|
| 24 |
+
"pydantic (>=2.6.0,<2.10.0)",
|
| 25 |
+
"rsa (>=4.7,<5.0)",
|
| 26 |
]
|
| 27 |
|
| 28 |
|
|
|
|
| 36 |
pytest = "^8.4.0"
|
| 37 |
mypy = "^1.16.0"
|
| 38 |
isort = "^6.0.1"
|
| 39 |
+
safety = "^3.5.2"
|
| 40 |
+
bandit = "^1.8.3"
|
| 41 |
|
| 42 |
|
| 43 |
[tool.poetry.group.docs.dependencies]
|
src/compilation/latex_compilation.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
"""Module for compiling LaTeX presentations into PDF format."""
|
| 2 |
|
| 3 |
-
import os
|
| 4 |
-
import subprocess
|
| 5 |
|
| 6 |
from langfuse.decorators import langfuse_context, observe
|
| 7 |
|
|
@@ -45,13 +45,13 @@ def compile_presentation(latex_code, work_dir) -> str:
|
|
| 45 |
check=True,
|
| 46 |
stdout=subprocess.PIPE,
|
| 47 |
stderr=subprocess.PIPE,
|
| 48 |
-
)
|
| 49 |
subprocess.run(
|
| 50 |
["pdflatex", "-interaction=nonstopmode", "presentation.tex"],
|
| 51 |
check=True,
|
| 52 |
stdout=subprocess.PIPE,
|
| 53 |
stderr=subprocess.PIPE,
|
| 54 |
-
)
|
| 55 |
|
| 56 |
except subprocess.CalledProcessError as e:
|
| 57 |
# Handle error in case of failed LaTeX compilation
|
|
|
|
| 1 |
"""Module for compiling LaTeX presentations into PDF format."""
|
| 2 |
|
| 3 |
+
import os # nosec B404 - safe usage
|
| 4 |
+
import subprocess # nosec B404 - safe usage
|
| 5 |
|
| 6 |
from langfuse.decorators import langfuse_context, observe
|
| 7 |
|
|
|
|
| 45 |
check=True,
|
| 46 |
stdout=subprocess.PIPE,
|
| 47 |
stderr=subprocess.PIPE,
|
| 48 |
+
) # nosec B603 B607 - fixed command, no untrusted input
|
| 49 |
subprocess.run(
|
| 50 |
["pdflatex", "-interaction=nonstopmode", "presentation.tex"],
|
| 51 |
check=True,
|
| 52 |
stdout=subprocess.PIPE,
|
| 53 |
stderr=subprocess.PIPE,
|
| 54 |
+
) # nosec B603 B607 - fixed command, no untrusted input
|
| 55 |
|
| 56 |
except subprocess.CalledProcessError as e:
|
| 57 |
# Handle error in case of failed LaTeX compilation
|
src/processing/images_processing.py
CHANGED
|
@@ -97,7 +97,7 @@ def extract_images(pdf, doc, page, page_num) -> list[dict]:
|
|
| 97 |
xref = img[0]
|
| 98 |
base_image = doc.extract_image(xref)
|
| 99 |
image_bytes = base_image["image"]
|
| 100 |
-
image_hash = hashlib.md5(image_bytes).hexdigest()
|
| 101 |
image_name = f"doc{pdf}_page{page_num}_img{img_index}_hash{image_hash[:8]}.png"
|
| 102 |
|
| 103 |
# Find the bbox
|
|
@@ -339,7 +339,7 @@ def extract_vector(pdf, page, page_num) -> list[dict]:
|
|
| 339 |
scale_mat = fitz.Matrix(ZOOM, ZOOM)
|
| 340 |
figure_pix = page.get_pixmap(matrix=scale_mat, clip=figure_bbox)
|
| 341 |
figure_bytes = figure_pix.tobytes("png")
|
| 342 |
-
figure_hash = hashlib.md5(figure_bytes).hexdigest()
|
| 343 |
figure_name = (
|
| 344 |
f"doc{pdf}_page{page_num}_fig{group_num}_hash{figure_hash[:8]}.png"
|
| 345 |
)
|
|
@@ -404,7 +404,7 @@ def save_pdf_images(pdf_path: str, req_imgs: list, images_dir: str) -> bool:
|
|
| 404 |
xref = img[0]
|
| 405 |
base_image = doc.extract_image(xref)
|
| 406 |
image_bytes = base_image["image"]
|
| 407 |
-
image_hash = hashlib.md5(image_bytes).hexdigest()
|
| 408 |
|
| 409 |
image_found = any(
|
| 410 |
pdf == img["doc"]
|
|
@@ -485,7 +485,9 @@ def save_pdf_figures(pdf_path: str, req_figs: list, figures_dir: str) -> bool:
|
|
| 485 |
scale_mat = fitz.Matrix(ZOOM, ZOOM)
|
| 486 |
figure_pix = page.get_pixmap(matrix=scale_mat, clip=figure_bbox)
|
| 487 |
figure_bytes = figure_pix.tobytes("png")
|
| 488 |
-
figure_hash = hashlib.md5(
|
|
|
|
|
|
|
| 489 |
|
| 490 |
figure_found = any(
|
| 491 |
pdf == fig["doc"]
|
|
|
|
| 97 |
xref = img[0]
|
| 98 |
base_image = doc.extract_image(xref)
|
| 99 |
image_bytes = base_image["image"]
|
| 100 |
+
image_hash = hashlib.md5(image_bytes, usedforsecurity=False).hexdigest()
|
| 101 |
image_name = f"doc{pdf}_page{page_num}_img{img_index}_hash{image_hash[:8]}.png"
|
| 102 |
|
| 103 |
# Find the bbox
|
|
|
|
| 339 |
scale_mat = fitz.Matrix(ZOOM, ZOOM)
|
| 340 |
figure_pix = page.get_pixmap(matrix=scale_mat, clip=figure_bbox)
|
| 341 |
figure_bytes = figure_pix.tobytes("png")
|
| 342 |
+
figure_hash = hashlib.md5(figure_bytes, usedforsecurity=False).hexdigest()
|
| 343 |
figure_name = (
|
| 344 |
f"doc{pdf}_page{page_num}_fig{group_num}_hash{figure_hash[:8]}.png"
|
| 345 |
)
|
|
|
|
| 404 |
xref = img[0]
|
| 405 |
base_image = doc.extract_image(xref)
|
| 406 |
image_bytes = base_image["image"]
|
| 407 |
+
image_hash = hashlib.md5(image_bytes, usedforsecurity=False).hexdigest()
|
| 408 |
|
| 409 |
image_found = any(
|
| 410 |
pdf == img["doc"]
|
|
|
|
| 485 |
scale_mat = fitz.Matrix(ZOOM, ZOOM)
|
| 486 |
figure_pix = page.get_pixmap(matrix=scale_mat, clip=figure_bbox)
|
| 487 |
figure_bytes = figure_pix.tobytes("png")
|
| 488 |
+
figure_hash = hashlib.md5(
|
| 489 |
+
figure_bytes, usedforsecurity=False
|
| 490 |
+
).hexdigest()
|
| 491 |
|
| 492 |
figure_found = any(
|
| 493 |
pdf == fig["doc"]
|
src/services/prompt.py
CHANGED
|
@@ -1,6 +1,8 @@
|
|
| 1 |
"""Module contatining the master prompt."""
|
| 2 |
|
| 3 |
|
|
|
|
|
|
|
| 4 |
def get_prompt(
|
| 5 |
presentation_theme: str = "default", color_theme: str = "default"
|
| 6 |
) -> str:
|
|
@@ -120,7 +122,7 @@ def get_prompt(
|
|
| 120 |
- Skip any image with caption `"None"`.
|
| 121 |
- Use **Core Idea 2** layout for **vertical** and **square** images.
|
| 122 |
- Use **Core Idea 3** layout for **horizontal** images
|
| 123 |
-
(when a two-column layout is not suitable).\n\n"""
|
| 124 |
|
| 125 |
return prompt
|
| 126 |
|
|
|
|
| 1 |
"""Module contatining the master prompt."""
|
| 2 |
|
| 3 |
|
| 4 |
+
# This is an AI prompt template, not a SQL statement.
|
| 5 |
+
# Bandit flagged it mistakenly as a possible SQL injection.
|
| 6 |
def get_prompt(
|
| 7 |
presentation_theme: str = "default", color_theme: str = "default"
|
| 8 |
) -> str:
|
|
|
|
| 122 |
- Skip any image with caption `"None"`.
|
| 123 |
- Use **Core Idea 2** layout for **vertical** and **square** images.
|
| 124 |
- Use **Core Idea 3** layout for **horizontal** images
|
| 125 |
+
(when a two-column layout is not suitable).\n\n""" # nosec B608
|
| 126 |
|
| 127 |
return prompt
|
| 128 |
|
src/telemetry/logging_utils.py
CHANGED
|
@@ -38,7 +38,7 @@ class Logger:
|
|
| 38 |
"""
|
| 39 |
Returns the global logger instance.
|
| 40 |
"""
|
|
|
|
| 41 |
if cls._logger is None:
|
| 42 |
-
|
| 43 |
-
assert cls._logger is not None
|
| 44 |
return cls._logger
|
|
|
|
| 38 |
"""
|
| 39 |
Returns the global logger instance.
|
| 40 |
"""
|
| 41 |
+
cls.setup_logging() # Ensure logging is set up before returning the logger
|
| 42 |
if cls._logger is None:
|
| 43 |
+
raise RuntimeError("Logger was not properly initialized")
|
|
|
|
| 44 |
return cls._logger
|