Spaces:
Running on Zero
Running on Zero
Upload 32 files
Browse files- .gitignore +220 -0
- Dockerfile +27 -0
- ESOL +91 -0
- LICENSE +201 -0
- PyFundaments – Function Overview.md +145 -0
- PyFundaments.md +232 -0
- README.md +515 -14
- REVIEW.md +250 -0
- app/.pyfun +561 -0
- app/README.md +2 -0
- app/__init__.py +7 -0
- app/app.py +317 -0
- app/config.py +298 -0
- app/db_sync.py +554 -0
- app/mcp.py +495 -0
- app/models.py +75 -0
- app/pricing.py +449 -0
- app/providers.py +825 -0
- app/shellmaster.py +323 -0
- app/tools.py +276 -0
- example-mcp___.env +86 -0
- fundaments/__init__.py +1 -0
- fundaments/access_control.py +247 -0
- fundaments/config_handler.py +115 -0
- fundaments/debug.py +91 -0
- fundaments/encryption.py +233 -0
- fundaments/postgresql.py +179 -0
- fundaments/security.py +82 -0
- fundaments/user_handler.py +370 -0
- main.py +202 -0
- requirements.txt +34 -0
.gitignore
ADDED
|
@@ -0,0 +1,220 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Universal-MCP-server-sandox
|
| 2 |
+
|
| 3 |
+
README.MD
|
| 4 |
+
.gitignore
|
| 5 |
+
app/*
|
| 6 |
+
|
| 7 |
+
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
.env
|
| 13 |
+
# Byte-compiled / optimized / DLL files
|
| 14 |
+
__pycache__/
|
| 15 |
+
*.py[codz]
|
| 16 |
+
*$py.class
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
# C extensions
|
| 20 |
+
*.so
|
| 21 |
+
|
| 22 |
+
# Distribution / packaging
|
| 23 |
+
.Python
|
| 24 |
+
build/
|
| 25 |
+
develop-eggs/
|
| 26 |
+
dist/
|
| 27 |
+
downloads/
|
| 28 |
+
eggs/
|
| 29 |
+
.eggs/
|
| 30 |
+
lib/
|
| 31 |
+
lib64/
|
| 32 |
+
parts/
|
| 33 |
+
sdist/
|
| 34 |
+
var/
|
| 35 |
+
wheels/
|
| 36 |
+
share/python-wheels/
|
| 37 |
+
*.egg-info/
|
| 38 |
+
.installed.cfg
|
| 39 |
+
*.egg
|
| 40 |
+
MANIFEST
|
| 41 |
+
|
| 42 |
+
# PyInstaller
|
| 43 |
+
# Usually these files are written by a python script from a template
|
| 44 |
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
| 45 |
+
*.manifest
|
| 46 |
+
*.spec
|
| 47 |
+
|
| 48 |
+
# Installer logs
|
| 49 |
+
pip-log.txt
|
| 50 |
+
pip-delete-this-directory.txt
|
| 51 |
+
|
| 52 |
+
# Unit test / coverage reports
|
| 53 |
+
htmlcov/
|
| 54 |
+
.tox/
|
| 55 |
+
.nox/
|
| 56 |
+
.coverage
|
| 57 |
+
.coverage.*
|
| 58 |
+
.cache
|
| 59 |
+
nosetests.xml
|
| 60 |
+
coverage.xml
|
| 61 |
+
*.cover
|
| 62 |
+
*.py.cover
|
| 63 |
+
.hypothesis/
|
| 64 |
+
.pytest_cache/
|
| 65 |
+
cover/
|
| 66 |
+
|
| 67 |
+
# Translations
|
| 68 |
+
*.mo
|
| 69 |
+
*.pot
|
| 70 |
+
|
| 71 |
+
# Django stuff:
|
| 72 |
+
*.log
|
| 73 |
+
local_settings.py
|
| 74 |
+
db.sqlite3
|
| 75 |
+
db.sqlite3-journal
|
| 76 |
+
|
| 77 |
+
# Flask stuff:
|
| 78 |
+
instance/
|
| 79 |
+
.webassets-cache
|
| 80 |
+
|
| 81 |
+
# Scrapy stuff:
|
| 82 |
+
.scrapy
|
| 83 |
+
|
| 84 |
+
# Sphinx documentation
|
| 85 |
+
docs/_build/
|
| 86 |
+
|
| 87 |
+
# PyBuilder
|
| 88 |
+
.pybuilder/
|
| 89 |
+
target/
|
| 90 |
+
|
| 91 |
+
# Jupyter Notebook
|
| 92 |
+
.ipynb_checkpoints
|
| 93 |
+
|
| 94 |
+
# IPython
|
| 95 |
+
profile_default/
|
| 96 |
+
ipython_config.py
|
| 97 |
+
|
| 98 |
+
# pyenv
|
| 99 |
+
# For a library or package, you might want to ignore these files since the code is
|
| 100 |
+
# intended to run in multiple environments; otherwise, check them in:
|
| 101 |
+
# .python-version
|
| 102 |
+
|
| 103 |
+
# pipenv
|
| 104 |
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
| 105 |
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
| 106 |
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
| 107 |
+
# install all needed dependencies.
|
| 108 |
+
#Pipfile.lock
|
| 109 |
+
|
| 110 |
+
# UV
|
| 111 |
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
| 112 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 113 |
+
# commonly ignored for libraries.
|
| 114 |
+
#uv.lock
|
| 115 |
+
|
| 116 |
+
# poetry
|
| 117 |
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
| 118 |
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
| 119 |
+
# commonly ignored for libraries.
|
| 120 |
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
| 121 |
+
#poetry.lock
|
| 122 |
+
#poetry.toml
|
| 123 |
+
|
| 124 |
+
# pdm
|
| 125 |
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
| 126 |
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
| 127 |
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
| 128 |
+
#pdm.lock
|
| 129 |
+
#pdm.toml
|
| 130 |
+
.pdm-python
|
| 131 |
+
.pdm-build/
|
| 132 |
+
|
| 133 |
+
# pixi
|
| 134 |
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
| 135 |
+
#pixi.lock
|
| 136 |
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
| 137 |
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
| 138 |
+
.pixi
|
| 139 |
+
|
| 140 |
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
| 141 |
+
__pypackages__/
|
| 142 |
+
|
| 143 |
+
# Celery stuff
|
| 144 |
+
celerybeat-schedule
|
| 145 |
+
celerybeat.pid
|
| 146 |
+
|
| 147 |
+
# SageMath parsed files
|
| 148 |
+
*.sage.py
|
| 149 |
+
|
| 150 |
+
# Environments
|
| 151 |
+
.env
|
| 152 |
+
.envrc
|
| 153 |
+
.venv
|
| 154 |
+
env/
|
| 155 |
+
venv/
|
| 156 |
+
ENV/
|
| 157 |
+
env.bak/
|
| 158 |
+
venv.bak/
|
| 159 |
+
|
| 160 |
+
# Spyder project settings
|
| 161 |
+
.spyderproject
|
| 162 |
+
.spyproject
|
| 163 |
+
|
| 164 |
+
# Rope project settings
|
| 165 |
+
.ropeproject
|
| 166 |
+
|
| 167 |
+
# mkdocs documentation
|
| 168 |
+
/site
|
| 169 |
+
|
| 170 |
+
# mypy
|
| 171 |
+
.mypy_cache/
|
| 172 |
+
.dmypy.json
|
| 173 |
+
dmypy.json
|
| 174 |
+
|
| 175 |
+
# Pyre type checker
|
| 176 |
+
.pyre/
|
| 177 |
+
|
| 178 |
+
# pytype static type analyzer
|
| 179 |
+
.pytype/
|
| 180 |
+
|
| 181 |
+
# Cython debug symbols
|
| 182 |
+
cython_debug/
|
| 183 |
+
|
| 184 |
+
# PyCharm
|
| 185 |
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
| 186 |
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
| 187 |
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
| 188 |
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
| 189 |
+
#.idea/
|
| 190 |
+
|
| 191 |
+
# Abstra
|
| 192 |
+
# Abstra is an AI-powered process automation framework.
|
| 193 |
+
# Ignore directories containing user credentials, local state, and settings.
|
| 194 |
+
# Learn more at https://abstra.io/docs
|
| 195 |
+
.abstra/
|
| 196 |
+
|
| 197 |
+
# Visual Studio Code
|
| 198 |
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
| 199 |
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
| 200 |
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
| 201 |
+
# you could uncomment the following to ignore the entire vscode folder
|
| 202 |
+
# .vscode/
|
| 203 |
+
|
| 204 |
+
# Ruff stuff:
|
| 205 |
+
.ruff_cache/
|
| 206 |
+
|
| 207 |
+
# PyPI configuration file
|
| 208 |
+
.pypirc
|
| 209 |
+
|
| 210 |
+
# Cursor
|
| 211 |
+
# Cursor is an AI-powered code editor. `.cursorignore` specifies files/directories to
|
| 212 |
+
# exclude from AI features like autocomplete and code analysis. Recommended for sensitive data
|
| 213 |
+
# refer to https://docs.cursor.com/context/ignore-files
|
| 214 |
+
.cursorignore
|
| 215 |
+
.cursorindexingignore
|
| 216 |
+
|
| 217 |
+
# Marimo
|
| 218 |
+
marimo/_static/
|
| 219 |
+
marimo/_lsp/
|
| 220 |
+
__marimo__/
|
Dockerfile
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Use an official Python runtime as a parent image.
|
| 2 |
+
# We choose a slim version to keep the image size small.
|
| 3 |
+
FROM python:3.10-slim
|
| 4 |
+
|
| 5 |
+
# Set the working directory in the container.
|
| 6 |
+
# All subsequent commands will be executed in this directory.
|
| 7 |
+
WORKDIR /app
|
| 8 |
+
|
| 9 |
+
# Copy the requirements file into the container.
|
| 10 |
+
# We do this first to leverage Docker's layer caching.
|
| 11 |
+
# If requirements.txt doesn't change, this step is skipped.
|
| 12 |
+
COPY requirements.txt ./
|
| 13 |
+
|
| 14 |
+
# Install any needed packages specified in requirements.txt.
|
| 15 |
+
# The --no-cache-dir flag helps to keep the image smaller.
|
| 16 |
+
RUN pip install --no-cache-dir -r requirements.txt
|
| 17 |
+
|
| 18 |
+
# Copy the rest of the application code into the working directory.
|
| 19 |
+
COPY . .
|
| 20 |
+
|
| 21 |
+
# Expose the hub port (matches HUB_PORT in app/.pyfun)
|
| 22 |
+
EXPOSE 7860
|
| 23 |
+
|
| 24 |
+
# Define the command to run your application.
|
| 25 |
+
# This command will be executed when the container starts.
|
| 26 |
+
# We use main.py as the entry point, as per our architecture.
|
| 27 |
+
CMD ["python", "main.py"]
|
ESOL
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
## Ethical Security Operations License (ESOL v1.1)
|
| 2 |
+
since 2021
|
| 3 |
+
updated on 05.02.2026
|
| 4 |
+
|
| 5 |
+
### Section 1: Preamble and Scope
|
| 6 |
+
The **Ethical Security Operations License (ESOL v1.1)** is an **additional, non-severable condition** supplementing the **MIT License** under which the covered software (hereinafter "the Work") is distributed.
|
| 7 |
+
|
| 8 |
+
By downloading, copying, modifying, or executing the Work, the Licensee **irrevocably agrees** to adhere to both the terms of the MIT License and the specific, mandatory ethical constraints defined herein. **Lack of awareness of these terms does not constitute a defense.**
|
| 9 |
+
|
| 10 |
+
### Section 2: Mandatory Ethical Use and Purpose
|
| 11 |
+
The grant of rights under this License is **expressly and exclusively conditioned** upon the Licensee's continuous adherence to the following use limitations:
|
| 12 |
+
|
| 13 |
+
1. **Authorized Use Only:** The Work shall be used **exclusively** for:
|
| 14 |
+
- Defensive security operations (Blue Teaming)
|
| 15 |
+
- Authorized penetration testing (Red Teaming) with documented consent
|
| 16 |
+
- Vulnerability research on systems owned or explicitly authorized by the researcher
|
| 17 |
+
- Security compliance auditing with contractual authorization
|
| 18 |
+
|
| 19 |
+
2. **Explicit Written Authorization Required:** Any security testing, scanning, exploitation, or enumeration against **any system not wholly owned by the Licensee** requires **explicit, documented, written authorization** from the rightful owner **prior to execution**. Verbal permission, implicit consent, or "bug bounty program existence" does **not** constitute authorization without explicit scope documentation.
|
| 20 |
+
|
| 21 |
+
3. **Educational Use Constraints:** Educational or research use must:
|
| 22 |
+
- Occur only in isolated, controlled environments (virtual machines, dedicated test networks)
|
| 23 |
+
- Not target production systems, public infrastructure, or third-party services
|
| 24 |
+
- Comply with institutional ethics board requirements where applicable
|
| 25 |
+
|
| 26 |
+
### Section 3: Prohibited Use (Malicious Activities)
|
| 27 |
+
The Licensee is **strictly and unconditionally prohibited** from using the Work for:
|
| 28 |
+
|
| 29 |
+
* **Unauthorized Access or Reconnaissance:** Scanning, probing, accessing, or attempting to access any computer system, network, database, or data without **explicit prior written authorization** from the lawful owner.
|
| 30 |
+
|
| 31 |
+
* **Malicious Code Operations:** Creating, distributing, executing, or facilitating:
|
| 32 |
+
- Malware, viruses, worms, trojans, ransomware, or rootkits
|
| 33 |
+
- Cryptominers operating without system owner consent
|
| 34 |
+
- Phishing infrastructure, scam campaigns, or social engineering attacks
|
| 35 |
+
- Exploits targeting zero-day vulnerabilities outside authorized disclosure processes
|
| 36 |
+
|
| 37 |
+
* **Service Disruption:** Denial-of-service (DoS/DDoS) attacks, resource exhaustion, or any unauthorized degradation of system availability or performance.
|
| 38 |
+
|
| 39 |
+
* **Data Exfiltration or Manipulation:** Unauthorized copying, modification, deletion, or encryption of data not owned by the Licensee.
|
| 40 |
+
|
| 41 |
+
* **Legal Violations:** Any activity violating the Computer Fraud and Abuse Act (CFAA), GDPR, national cybercrime laws, or international treaties concerning unauthorized computer access.
|
| 42 |
+
|
| 43 |
+
* **Circumvention of Security Controls:** Bypassing authentication, encryption, access controls, or security monitoring systems without documented authorization.
|
| 44 |
+
|
| 45 |
+
### Section 4: Compliance Verification and Audit Rights
|
| 46 |
+
The Licensor reserves the right to:
|
| 47 |
+
- Request documentation of authorization for any deployment of the Work
|
| 48 |
+
- Audit compliance upon reasonable notice if misuse is suspected
|
| 49 |
+
- Publicly disclose violations (including Licensee identity) to warn the security community
|
| 50 |
+
|
| 51 |
+
**Licensees conducting authorized security research are encouraged to maintain contemporaneous records of authorization as evidence of compliance.**
|
| 52 |
+
|
| 53 |
+
### Section 5: No Warranty Regarding Legal Compliance
|
| 54 |
+
**THE WORK IS PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND. THE LICENSOR MAKES NO REPRESENTATION THAT USE OF THE WORK, EVEN IN COMPLIANCE WITH THIS LICENSE, WILL BE LEGAL IN ALL JURISDICTIONS. THE LICENSEE BEARS SOLE RESPONSIBILITY FOR ENSURING COMPLIANCE WITH ALL APPLICABLE LAWS.**
|
| 55 |
+
|
| 56 |
+
### Section 6: Violation and Termination
|
| 57 |
+
Violation of **any** provision of this ESOL v1.1 constitutes a **material breach** resulting in:
|
| 58 |
+
|
| 59 |
+
1. **Immediate Automatic Termination** of all rights under both the MIT License and this ESOL
|
| 60 |
+
2. **Obligation to Cease Use** and destroy all copies of the Work
|
| 61 |
+
3. **Liability for Damages:** The Licensee shall be liable for all damages, including but not limited to:
|
| 62 |
+
- Direct and consequential damages to affected third parties
|
| 63 |
+
- Licensor's legal costs in enforcement actions
|
| 64 |
+
- Reputational harm to the Licensor or the security research community
|
| 65 |
+
|
| 66 |
+
4. **Reporting to Authorities:** The Licensor reserves the right to report violations to appropriate law enforcement and cybersecurity authorities.
|
| 67 |
+
|
| 68 |
+
### Section 7: Severability and Precedence
|
| 69 |
+
If any provision of this License is held unenforceable, the remaining provisions remain in full effect. **In any conflict between the MIT License and ESOL v1.1, the ESOL v1.1 takes precedence regarding ethical use constraints.**
|
| 70 |
+
|
| 71 |
+
### Section 8: Jurisdiction and Dispute Resolution
|
| 72 |
+
This License shall be governed by the laws of **[Germany(Berlin)]**. Disputes shall be resolved through binding arbitration under **[Arbitration Rules]**, with the prevailing party entitled to attorney's fees.
|
| 73 |
+
|
| 74 |
+
### Section 9: License Acceptance Evidence
|
| 75 |
+
By downloading this software from any source (GitHub, mirrors, forks), you acknowledge that:
|
| 76 |
+
- This license was prominently displayed in the repository
|
| 77 |
+
- You had reasonable opportunity to review these terms
|
| 78 |
+
- Your use constitutes legally binding acceptance
|
| 79 |
+
---
|
| 80 |
+
|
| 81 |
+
## Final Licensing Statement
|
| 82 |
+
**This Work is dual-licensed under:**
|
| 83 |
+
1. **The MIT License** (for general software rights)
|
| 84 |
+
2. **The Ethical Security Operations License v1.1 (ESOL v1.1)** (for use constraints)
|
| 85 |
+
|
| 86 |
+
**The ESOL v1.1 is a mandatory, non-severable condition of use. Acceptance is automatic upon use of the Work.**
|
| 87 |
+
|
| 88 |
+
**By using this software, you acknowledge that misuse may result in criminal prosecution, civil liability, and permanent termination of license rights.**
|
| 89 |
+
|
| 90 |
+
> Checked & Updated, on 08.02.2026 @01700MEZ
|
| 91 |
+
## Ethical Security Operations License (ESOL v1.1) since 2021 against fraud and scam
|
LICENSE
ADDED
|
@@ -0,0 +1,201 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
Apache License
|
| 2 |
+
Version 2.0, January 2004
|
| 3 |
+
http://www.apache.org/licenses/
|
| 4 |
+
|
| 5 |
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
| 6 |
+
|
| 7 |
+
1. Definitions.
|
| 8 |
+
|
| 9 |
+
"License" shall mean the terms and conditions for use, reproduction,
|
| 10 |
+
and distribution as defined by Sections 1 through 9 of this document.
|
| 11 |
+
|
| 12 |
+
"Licensor" shall mean the copyright owner or entity authorized by
|
| 13 |
+
the copyright owner that is granting the License.
|
| 14 |
+
|
| 15 |
+
"Legal Entity" shall mean the union of the acting entity and all
|
| 16 |
+
other entities that control, are controlled by, or are under common
|
| 17 |
+
control with that entity. For the purposes of this definition,
|
| 18 |
+
"control" means (i) the power, direct or indirect, to cause the
|
| 19 |
+
direction or management of such entity, whether by contract or
|
| 20 |
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
| 21 |
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
| 22 |
+
|
| 23 |
+
"You" (or "Your") shall mean an individual or Legal Entity
|
| 24 |
+
exercising permissions granted by this License.
|
| 25 |
+
|
| 26 |
+
"Source" form shall mean the preferred form for making modifications,
|
| 27 |
+
including but not limited to software source code, documentation
|
| 28 |
+
source, and configuration files.
|
| 29 |
+
|
| 30 |
+
"Object" form shall mean any form resulting from mechanical
|
| 31 |
+
transformation or translation of a Source form, including but
|
| 32 |
+
not limited to compiled object code, generated documentation,
|
| 33 |
+
and conversions to other media types.
|
| 34 |
+
|
| 35 |
+
"Work" shall mean the work of authorship, whether in Source or
|
| 36 |
+
Object form, made available under the License, as indicated by a
|
| 37 |
+
copyright notice that is included in or attached to the work
|
| 38 |
+
(an example is provided in the Appendix below).
|
| 39 |
+
|
| 40 |
+
"Derivative Works" shall mean any work, whether in Source or Object
|
| 41 |
+
form, that is based on (or derived from) the Work and for which the
|
| 42 |
+
editorial revisions, annotations, elaborations, or other modifications
|
| 43 |
+
represent, as a whole, an original work of authorship. For the purposes
|
| 44 |
+
of this License, Derivative Works shall not include works that remain
|
| 45 |
+
separable from, or merely link (or bind by name) to the interfaces of,
|
| 46 |
+
the Work and Derivative Works thereof.
|
| 47 |
+
|
| 48 |
+
"Contribution" shall mean any work of authorship, including
|
| 49 |
+
the original version of the Work and any modifications or additions
|
| 50 |
+
to that Work or Derivative Works thereof, that is intentionally
|
| 51 |
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
| 52 |
+
or by an individual or Legal Entity authorized to submit on behalf of
|
| 53 |
+
the copyright owner. For the purposes of this definition, "submitted"
|
| 54 |
+
means any form of electronic, verbal, or written communication sent
|
| 55 |
+
to the Licensor or its representatives, including but not limited to
|
| 56 |
+
communication on electronic mailing lists, source code control systems,
|
| 57 |
+
and issue tracking systems that are managed by, or on behalf of, the
|
| 58 |
+
Licensor for the purpose of discussing and improving the Work, but
|
| 59 |
+
excluding communication that is conspicuously marked or otherwise
|
| 60 |
+
designated in writing by the copyright owner as "Not a Contribution."
|
| 61 |
+
|
| 62 |
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
| 63 |
+
on behalf of whom a Contribution has been received by Licensor and
|
| 64 |
+
subsequently incorporated within the Work.
|
| 65 |
+
|
| 66 |
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
| 67 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 68 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 69 |
+
copyright license to reproduce, prepare Derivative Works of,
|
| 70 |
+
publicly display, publicly perform, sublicense, and distribute the
|
| 71 |
+
Work and such Derivative Works in Source or Object form.
|
| 72 |
+
|
| 73 |
+
3. Grant of Patent License. Subject to the terms and conditions of
|
| 74 |
+
this License, each Contributor hereby grants to You a perpetual,
|
| 75 |
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
| 76 |
+
(except as stated in this section) patent license to make, have made,
|
| 77 |
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
| 78 |
+
where such license applies only to those patent claims licensable
|
| 79 |
+
by such Contributor that are necessarily infringed by their
|
| 80 |
+
Contribution(s) alone or by combination of their Contribution(s)
|
| 81 |
+
with the Work to which such Contribution(s) was submitted. If You
|
| 82 |
+
institute patent litigation against any entity (including a
|
| 83 |
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
| 84 |
+
or a Contribution incorporated within the Work constitutes direct
|
| 85 |
+
or contributory patent infringement, then any patent licenses
|
| 86 |
+
granted to You under this License for that Work shall terminate
|
| 87 |
+
as of the date such litigation is filed.
|
| 88 |
+
|
| 89 |
+
4. Redistribution. You may reproduce and distribute copies of the
|
| 90 |
+
Work or Derivative Works thereof in any medium, with or without
|
| 91 |
+
modifications, and in Source or Object form, provided that You
|
| 92 |
+
meet the following conditions:
|
| 93 |
+
|
| 94 |
+
(a) You must give any other recipients of the Work or
|
| 95 |
+
Derivative Works a copy of this License; and
|
| 96 |
+
|
| 97 |
+
(b) You must cause any modified files to carry prominent notices
|
| 98 |
+
stating that You changed the files; and
|
| 99 |
+
|
| 100 |
+
(c) You must retain, in the Source form of any Derivative Works
|
| 101 |
+
that You distribute, all copyright, patent, trademark, and
|
| 102 |
+
attribution notices from the Source form of the Work,
|
| 103 |
+
excluding those notices that do not pertain to any part of
|
| 104 |
+
the Derivative Works; and
|
| 105 |
+
|
| 106 |
+
(d) If the Work includes a "NOTICE" text file as part of its
|
| 107 |
+
distribution, then any Derivative Works that You distribute must
|
| 108 |
+
include a readable copy of the attribution notices contained
|
| 109 |
+
within such NOTICE file, excluding those notices that do not
|
| 110 |
+
pertain to any part of the Derivative Works, in at least one
|
| 111 |
+
of the following places: within a NOTICE text file distributed
|
| 112 |
+
as part of the Derivative Works; within the Source form or
|
| 113 |
+
documentation, if provided along with the Derivative Works; or,
|
| 114 |
+
within a display generated by the Derivative Works, if and
|
| 115 |
+
wherever such third-party notices normally appear. The contents
|
| 116 |
+
of the NOTICE file are for informational purposes only and
|
| 117 |
+
do not modify the License. You may add Your own attribution
|
| 118 |
+
notices within Derivative Works that You distribute, alongside
|
| 119 |
+
or as an addendum to the NOTICE text from the Work, provided
|
| 120 |
+
that such additional attribution notices cannot be construed
|
| 121 |
+
as modifying the License.
|
| 122 |
+
|
| 123 |
+
You may add Your own copyright statement to Your modifications and
|
| 124 |
+
may provide additional or different license terms and conditions
|
| 125 |
+
for use, reproduction, or distribution of Your modifications, or
|
| 126 |
+
for any such Derivative Works as a whole, provided Your use,
|
| 127 |
+
reproduction, and distribution of the Work otherwise complies with
|
| 128 |
+
the conditions stated in this License.
|
| 129 |
+
|
| 130 |
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
| 131 |
+
any Contribution intentionally submitted for inclusion in the Work
|
| 132 |
+
by You to the Licensor shall be under the terms and conditions of
|
| 133 |
+
this License, without any additional terms or conditions.
|
| 134 |
+
Notwithstanding the above, nothing herein shall supersede or modify
|
| 135 |
+
the terms of any separate license agreement you may have executed
|
| 136 |
+
with Licensor regarding such Contributions.
|
| 137 |
+
|
| 138 |
+
6. Trademarks. This License does not grant permission to use the trade
|
| 139 |
+
names, trademarks, service marks, or product names of the Licensor,
|
| 140 |
+
except as required for reasonable and customary use in describing the
|
| 141 |
+
origin of the Work and reproducing the content of the NOTICE file.
|
| 142 |
+
|
| 143 |
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
| 144 |
+
agreed to in writing, Licensor provides the Work (and each
|
| 145 |
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
| 146 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
| 147 |
+
implied, including, without limitation, any warranties or conditions
|
| 148 |
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
| 149 |
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
| 150 |
+
appropriateness of using or redistributing the Work and assume any
|
| 151 |
+
risks associated with Your exercise of permissions under this License.
|
| 152 |
+
|
| 153 |
+
8. Limitation of Liability. In no event and under no legal theory,
|
| 154 |
+
whether in tort (including negligence), contract, or otherwise,
|
| 155 |
+
unless required by applicable law (such as deliberate and grossly
|
| 156 |
+
negligent acts) or agreed to in writing, shall any Contributor be
|
| 157 |
+
liable to You for damages, including any direct, indirect, special,
|
| 158 |
+
incidental, or consequential damages of any character arising as a
|
| 159 |
+
result of this License or out of the use or inability to use the
|
| 160 |
+
Work (including but not limited to damages for loss of goodwill,
|
| 161 |
+
work stoppage, computer failure or malfunction, or any and all
|
| 162 |
+
other commercial damages or losses), even if such Contributor
|
| 163 |
+
has been advised of the possibility of such damages.
|
| 164 |
+
|
| 165 |
+
9. Accepting Warranty or Additional Liability. While redistributing
|
| 166 |
+
the Work or Derivative Works thereof, You may choose to offer,
|
| 167 |
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
| 168 |
+
or other liability obligations and/or rights consistent with this
|
| 169 |
+
License. However, in accepting such obligations, You may act only
|
| 170 |
+
on Your own behalf and on Your sole responsibility, not on behalf
|
| 171 |
+
of any other Contributor, and only if You agree to indemnify,
|
| 172 |
+
defend, and hold each Contributor harmless for any liability
|
| 173 |
+
incurred by, or claims asserted against, such Contributor by reason
|
| 174 |
+
of your accepting any such warranty or additional liability.
|
| 175 |
+
|
| 176 |
+
END OF TERMS AND CONDITIONS
|
| 177 |
+
|
| 178 |
+
APPENDIX: How to apply the Apache License to your work.
|
| 179 |
+
|
| 180 |
+
To apply the Apache License to your work, attach the following
|
| 181 |
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
| 182 |
+
replaced with your own identifying information. (Don't include
|
| 183 |
+
the brackets!) The text should be enclosed in the appropriate
|
| 184 |
+
comment syntax for the file format. We also recommend that a
|
| 185 |
+
file or class name and description of purpose be included on the
|
| 186 |
+
same "printed page" as the copyright notice for easier
|
| 187 |
+
identification within third-party archives.
|
| 188 |
+
|
| 189 |
+
Copyright [yyyy] [name of copyright owner]
|
| 190 |
+
|
| 191 |
+
Licensed under the Apache License, Version 2.0 (the "License");
|
| 192 |
+
you may not use this file except in compliance with the License.
|
| 193 |
+
You may obtain a copy of the License at
|
| 194 |
+
|
| 195 |
+
http://www.apache.org/licenses/LICENSE-2.0
|
| 196 |
+
|
| 197 |
+
Unless required by applicable law or agreed to in writing, software
|
| 198 |
+
distributed under the License is distributed on an "AS IS" BASIS,
|
| 199 |
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
| 200 |
+
See the License for the specific language governing permissions and
|
| 201 |
+
limitations under the License.
|
PyFundaments – Function Overview.md
ADDED
|
@@ -0,0 +1,145 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments – Function Overview
|
| 2 |
+
|
| 3 |
+
## `main.py`
|
| 4 |
+
|
| 5 |
+
| Function | Description |
|
| 6 |
+
| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
|
| 7 |
+
| `initialize_fundaments()` | Asynchronously initializes services based on available environment variables. Returns a dictionary of services (`None` if not initialized). |
|
| 8 |
+
| `main()` | Application entry point. Calls `initialize_fundaments()`, loads `app/app.py`, closes DB pool on shutdown. |
|
| 9 |
+
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
## `fundaments/config_handler.py` – `ConfigHandler`
|
| 13 |
+
|
| 14 |
+
| Function | Description |
|
| 15 |
+
| ------------------------ | ------------------------------------------------------------------ |
|
| 16 |
+
| `__init__()` | Loads `.env` via `python-dotenv` and system environment variables. |
|
| 17 |
+
| `load_all_config()` | Stores all non-empty environment variables in `self.config`. |
|
| 18 |
+
| `get(key)` | Returns value as string or `None`. |
|
| 19 |
+
| `get_bool(key, default)` | Parses boolean values (`true/1/yes/on`). |
|
| 20 |
+
| `get_int(key, default)` | Returns integer value or `default` on failure. |
|
| 21 |
+
| `has(key)` | Returns `True` if key exists and is not empty. |
|
| 22 |
+
| `get_all()` | Returns copy of full configuration dictionary. |
|
| 23 |
+
| `config_service` | Global singleton instance. |
|
| 24 |
+
|
| 25 |
+
---
|
| 26 |
+
|
| 27 |
+
## `fundaments/postgresql.py`
|
| 28 |
+
|
| 29 |
+
| Function | Description |
|
| 30 |
+
| ----------------------------------------------------- | ----------------------------------------------------------------------------------- |
|
| 31 |
+
| `enforce_cloud_security(dsn_url)` | Enforces `sslmode=require`, applies timeouts, removes incompatible DSN options. |
|
| 32 |
+
| `mask_dsn(dsn_url)` | Removes credentials from DSN for logging. |
|
| 33 |
+
| `ssl_runtime_check(conn)` | Verifies active SSL connection. |
|
| 34 |
+
| `init_db_pool(dsn_url)` | Creates asyncpg pool (min=1, max=10) and runs SSL check. |
|
| 35 |
+
| `close_db_pool()` | Gracefully closes connection pool. |
|
| 36 |
+
| `execute_secured_query(query, *params, fetch_method)` | Executes parameterized query (`fetch`, `fetchrow`, `execute`) with reconnect logic. |
|
| 37 |
+
|
| 38 |
+
---
|
| 39 |
+
|
| 40 |
+
## `fundaments/encryption.py` – `Encryption`
|
| 41 |
+
|
| 42 |
+
| Function | Description |
|
| 43 |
+
| -------------------------------------- | ---------------------------------------------------------------- |
|
| 44 |
+
| `generate_salt()` | Generates secure 16-byte hex salt. |
|
| 45 |
+
| `__init__(master_key, salt)` | Derives AES-256 key via PBKDF2-SHA256 (480k iterations). |
|
| 46 |
+
| `encrypt(data)` | Encrypts string using AES-256-GCM. Returns `{data, nonce, tag}`. |
|
| 47 |
+
| `decrypt(encrypted_data, nonce, tag)` | Decrypts data. Raises `InvalidTag` if tampered. |
|
| 48 |
+
| `encrypt_file(source_path, dest_path)` | Encrypts file in 8192-byte chunks. |
|
| 49 |
+
| `decrypt_file(source_path, dest_path)` | Decrypts file using stored nonce and tag. |
|
| 50 |
+
|
| 51 |
+
---
|
| 52 |
+
|
| 53 |
+
## `fundaments/access_control.py` – `AccessControl`
|
| 54 |
+
|
| 55 |
+
| Function | Description |
|
| 56 |
+
| -------------------------------------------------- | ------------------------------------ |
|
| 57 |
+
| `__init__(user_id)` | Initializes with optional user ID. |
|
| 58 |
+
| `has_permission(permission_name)` | Checks if user has permission. |
|
| 59 |
+
| `get_user_permissions()` | Returns all user permissions. |
|
| 60 |
+
| `get_user_roles()` | Returns assigned roles. |
|
| 61 |
+
| `assign_role(role_id)` | Assigns role to user. |
|
| 62 |
+
| `remove_role(role_id)` | Removes role from user. |
|
| 63 |
+
| `get_all_roles()` | Returns all roles. |
|
| 64 |
+
| `get_all_permissions()` | Returns all permissions. |
|
| 65 |
+
| `create_role(name, description)` | Creates new role and returns ID. |
|
| 66 |
+
| `update_role_permissions(role_id, permission_ids)` | Replaces all permissions for a role. |
|
| 67 |
+
| `get_role_permissions(role_id)` | Returns role permissions. |
|
| 68 |
+
|
| 69 |
+
---
|
| 70 |
+
|
| 71 |
+
## `fundaments/user_handler.py`
|
| 72 |
+
|
| 73 |
+
### `Database` (SQLite Wrapper)
|
| 74 |
+
|
| 75 |
+
| Function | Description |
|
| 76 |
+
| ------------------------- | ------------------------------------------------- |
|
| 77 |
+
| `execute(query, params)` | Executes query and commits. |
|
| 78 |
+
| `fetchone(query, params)` | Returns single row. |
|
| 79 |
+
| `fetchall(query, params)` | Returns all rows. |
|
| 80 |
+
| `close()` | Closes connection. |
|
| 81 |
+
| `setup_tables()` | Creates `users` and `sessions` tables if missing. |
|
| 82 |
+
|
| 83 |
+
---
|
| 84 |
+
|
| 85 |
+
### `Security` (Password Utilities)
|
| 86 |
+
|
| 87 |
+
| Function | Description |
|
| 88 |
+
| ----------------------------------- | -------------------------------------------- |
|
| 89 |
+
| `hash_password(password)` | Hashes password (PBKDF2-SHA256 via passlib). |
|
| 90 |
+
| `verify_password(password, hashed)` | Verifies password against hash. |
|
| 91 |
+
| `regenerate_session(session_id)` | Generates new UUID session ID. |
|
| 92 |
+
|
| 93 |
+
---
|
| 94 |
+
|
| 95 |
+
### `UserHandler`
|
| 96 |
+
|
| 97 |
+
| Function | Description |
|
| 98 |
+
| ----------------------------------------- | ------------------------------------------------------ |
|
| 99 |
+
| `login(username, password, request_data)` | Authenticates user, validates state, creates session. |
|
| 100 |
+
| `logout()` | Removes session from DB and memory. |
|
| 101 |
+
| `is_logged_in()` | Checks if active session exists. |
|
| 102 |
+
| `is_admin()` | Checks session `is_admin` flag. |
|
| 103 |
+
| `validate_session(request_data)` | Validates session against IP and User-Agent. |
|
| 104 |
+
| `lock_account(username)` | Locks user account. |
|
| 105 |
+
| `reset_failed_attempts(username)` | Resets failed login counter. |
|
| 106 |
+
| `increment_failed_attempts(username)` | Increments failed attempts and locks after 5 failures. |
|
| 107 |
+
|
| 108 |
+
---
|
| 109 |
+
|
| 110 |
+
## `fundaments/security.py` – `Security` (Orchestrator)
|
| 111 |
+
|
| 112 |
+
| Function | Description |
|
| 113 |
+
| ---------------------------------------------- | --------------------------------------------------------------------- |
|
| 114 |
+
| `__init__(services)` | Initializes with required services. Raises `RuntimeError` if missing. |
|
| 115 |
+
| `user_login(username, password, request_data)` | Performs login and session validation. |
|
| 116 |
+
| `check_permission(user_id, permission_name)` | Delegates permission check. |
|
| 117 |
+
| `encrypt_data(data)` | Encrypts data if encryption service is available. |
|
| 118 |
+
| `decrypt_data(encrypted_data, nonce, tag)` | Decrypts data or returns `None` on failure. |
|
| 119 |
+
|
| 120 |
+
---
|
| 121 |
+
|
| 122 |
+
## `fundaments/debug.py` – `PyFundamentsDebug`
|
| 123 |
+
|
| 124 |
+
| Function | Description |
|
| 125 |
+
| ----------------- | ------------------------------------------------------- |
|
| 126 |
+
| `__init__()` | Reads debug-related environment variables. |
|
| 127 |
+
| `_setup_logger()` | Configures logging handlers. |
|
| 128 |
+
| `run()` | Outputs runtime diagnostics when debug mode is enabled. |
|
| 129 |
+
|
| 130 |
+
---
|
| 131 |
+
|
| 132 |
+
## `app/app.py`
|
| 133 |
+
|
| 134 |
+
| Function | Description |
|
| 135 |
+
| ------------------------------- | --------------------------------------------------------------------- |
|
| 136 |
+
| `start_application(fundaments)` | Receives initialized service dictionary and starts application logic. |
|
| 137 |
+
|
| 138 |
+
---
|
| 139 |
+
|
| 140 |
+
## Architecture Notes
|
| 141 |
+
|
| 142 |
+
* `UserHandler` uses internal SQLite.
|
| 143 |
+
* `AccessControl` uses PostgreSQL via `execute_secured_query`.
|
| 144 |
+
* `security.py` `Security` is the orchestrator layer.
|
| 145 |
+
* All services are optional.
|
PyFundaments.md
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments: A Secure Python Architecture
|
| 2 |
+
##### v. 1.0.1 dev
|
| 3 |
+
|
| 4 |
+
## Description
|
| 5 |
+
|
| 6 |
+
This project, named **PyFundaments**, provides a robust and secure Python architecture. Its core mission is not to be a monolithic framework, but to establish a layered, modular foundation—or "fundament"—of essential services. This structure ensures that every application starts on a verified and secure base, with a focus on stability, code clarity, and a security-first mindset from the ground up.
|
| 7 |
+
|
| 8 |
+
> [!NOTE]
|
| 9 |
+
> you must create your app/bot in app/* or you get an error! Than uncoment
|
| 10 |
+
> ```
|
| 11 |
+
>from app.app import start_application
|
| 12 |
+
>await start_application(fundaments)
|
| 13 |
+
>```
|
| 14 |
+
> in main.py
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
|
| 18 |
+
-----
|
| 19 |
+
|
| 20 |
+
## Table of Contents
|
| 21 |
+
|
| 22 |
+
- [Project Structure](#project-structure)
|
| 23 |
+
- [The Role of `main.py`](#the-role-of-mainpy)
|
| 24 |
+
- [Configuration (`.env`)](#configuration-env)
|
| 25 |
+
- [Installation](#installation)
|
| 26 |
+
- [Getting Started](#getting-started)
|
| 27 |
+
- [Module Documentation](#module-documentation)
|
| 28 |
+
- [Notes](#notes)
|
| 29 |
+
|
| 30 |
+
-----
|
| 31 |
+
|
| 32 |
+
## Project Structure
|
| 33 |
+
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
├── main.py # run main!
|
| 37 |
+
├── README.md
|
| 38 |
+
├── requirements.txt
|
| 39 |
+
├── .gitignore
|
| 40 |
+
├── .env.example
|
| 41 |
+
├── app/
|
| 42 |
+
│ └── ...
|
| 43 |
+
│ └── app.py # sandboxed app run!
|
| 44 |
+
│ └── tools.py
|
| 45 |
+
│ └── provider.py
|
| 46 |
+
│ └── models.py
|
| 47 |
+
│ └── db_sync.py
|
| 48 |
+
|
| 49 |
+
├── fundaments/ # do not touch!
|
| 50 |
+
│ ├── access_control.py
|
| 51 |
+
│ ├── config_handler.py
|
| 52 |
+
│ ├── encryption.py
|
| 53 |
+
│ ├── postgresql.py
|
| 54 |
+
│ ├── security.py
|
| 55 |
+
│ └── user_handler.py
|
| 56 |
+
└── docs/
|
| 57 |
+
├── access_control.py.md
|
| 58 |
+
├── encryption.py.md
|
| 59 |
+
├── postgresql.py.md
|
| 60 |
+
├── security.py.md
|
| 61 |
+
└── user_handler.py.md
|
| 62 |
+
|
| 63 |
+
```
|
| 64 |
+
-----
|
| 65 |
+
|
| 66 |
+
## The Role of `main.py`
|
| 67 |
+
|
| 68 |
+
`main.py` is the **only entry point** and acts as the first line of defense for the application.
|
| 69 |
+
|
| 70 |
+
### Responsibilities
|
| 71 |
+
|
| 72 |
+
- [x] **Validate Dependencies**: It checks for all necessary core modules in `fundaments/` and exits immediately if any are missing.
|
| 73 |
+
- [x] **Conditional Environment Loading**: It uses the `config_handler` to load available configuration variables and initializes only the services for which configuration is present.
|
| 74 |
+
- [x] **Initialize Services**: It creates instances of available services (PostgreSQL, encryption, access control, user handling) based on environment configuration, collecting them into a single service dictionary.
|
| 75 |
+
- [x] **Graceful Degradation**: Services that cannot be initialized are skipped with warnings, allowing applications to run with partial functionality.
|
| 76 |
+
- [x] **Decouple App Logic**: It hands off the prepared and verified services to `app/app.py`, allowing the main application to focus purely on its business logic without worrying about low-level setup.
|
| 77 |
+
|
| 78 |
+
-----
|
| 79 |
+
|
| 80 |
+
## Configuration (`.env`)
|
| 81 |
+
|
| 82 |
+
Application settings are managed using a `.env` file. **Never commit this file to version control.**
|
| 83 |
+
|
| 84 |
+
The framework uses **conditional loading** - only services with available configuration are initialized. This allows different application types to use only what they need.
|
| 85 |
+
|
| 86 |
+
A `.env.example` file is provided to show available variables. Create a copy named `.env` and configure only what your application requires.
|
| 87 |
+
|
| 88 |
+
### Core Configuration (Optional based on app type)
|
| 89 |
+
|
| 90 |
+
```text
|
| 91 |
+
# Database connection (required only for database-using apps)
|
| 92 |
+
DATABASE_URL="postgresql://user:password@host:port/database?sslmode=require"
|
| 93 |
+
|
| 94 |
+
# Encryption keys (required only for apps using encryption)
|
| 95 |
+
MASTER_ENCRYPTION_KEY="your_256_bit_key_here"
|
| 96 |
+
PERSISTENT_ENCRYPTION_SALT="your_unique_salt_here"
|
| 97 |
+
|
| 98 |
+
# Logging configuration (optional)
|
| 99 |
+
LOG_LEVEL="INFO"
|
| 100 |
+
LOG_TO_TMP="false"
|
| 101 |
+
ENABLE_PUBLIC_LOGS="true"
|
| 102 |
+
```
|
| 103 |
+
|
| 104 |
+
### Application Examples
|
| 105 |
+
|
| 106 |
+
**Discord Bot:** Only needs `BOT_TOKEN`, no database or encryption required.
|
| 107 |
+
**ML Pipeline:** Only needs `DATABASE_URL` for data access.
|
| 108 |
+
**Web Application:** May need all services for full functionality.
|
| 109 |
+
|
| 110 |
+
-----
|
| 111 |
+
|
| 112 |
+
## Installation
|
| 113 |
+
|
| 114 |
+
```bash
|
| 115 |
+
pip install -r requirements.txt
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
-----
|
| 119 |
+
|
| 120 |
+
## Getting Started
|
| 121 |
+
|
| 122 |
+
1. **Configure**: Create your `.env` file from the `.env.example` and set only the variables your application type requires.
|
| 123 |
+
2. **Run**: Execute the `main.py` script to start the application.
|
| 124 |
+
|
| 125 |
+
```bash
|
| 126 |
+
python main.py
|
| 127 |
+
```
|
| 128 |
+
|
| 129 |
+
The framework will automatically detect available configuration and load only the necessary services.
|
| 130 |
+
|
| 131 |
+
-----
|
| 132 |
+
|
| 133 |
+
## Module Documentation
|
| 134 |
+
|
| 135 |
+
Each security-relevant core module is documented in the `docs/` directory:
|
| 136 |
+
|
| 137 |
+
| Module | Description | Documentation |
|
| 138 |
+
| ------------------- | ---------------------------------------- | -------------------------------------------------|
|
| 139 |
+
| `access_control.py` | Role-based access management | [access\_control.py.md](docs/access_control.py.md)|
|
| 140 |
+
| `config_handler.py` | Universal configuration loader | [config\_handler.py.md](docs/config_handler.py.md)|
|
| 141 |
+
| `encryption.py` | Cryptographic routines | [encryption.py.md](docs/encryption.py.md) |
|
| 142 |
+
| `postgresql.py` | Secure, asynchronous database access | [postgresql.py.md](docs/postgresql.py.md) |
|
| 143 |
+
| `user_handler.py` | Authentication and identity management | [user\_handler.py.md](docs/user_handler.py.md) |
|
| 144 |
+
| `security.py` | Central security orchestration layer | [security.py.md](docs/security.py.md) |
|
| 145 |
+
|
| 146 |
+
-----
|
| 147 |
+
|
| 148 |
+
## License
|
| 149 |
+
|
| 150 |
+
This project is licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0).
|
| 151 |
+
|
| 152 |
+
### Section 4: Additional Restrictions and Ethical Use Policy
|
| 153 |
+
|
| 154 |
+
This project is released under the permissive Apache 2.0 License, which is intended to provide broad freedom for use and modification.
|
| 155 |
+
However, in the interest of fostering a safe and responsible community, we include the following **mandatory ethical use restrictions**:
|
| 156 |
+
|
| 157 |
+
Use of this project (or derivatives) is **strictly prohibited** for purposes that:
|
| 158 |
+
|
| 159 |
+
- **Promote Hatred or Discrimination**
|
| 160 |
+
Includes hate speech, incitement to violence, or discrimination based on race, religion, gender, orientation, etc.
|
| 161 |
+
|
| 162 |
+
- **Facilitate Illegal Activities**
|
| 163 |
+
Use to conduct or support criminal activity is forbidden.
|
| 164 |
+
|
| 165 |
+
- **Spread Malicious Content**
|
| 166 |
+
This includes pornography, malware, spyware, or other harmful payloads.
|
| 167 |
+
|
| 168 |
+
We believe that freedom in development must be coupled with responsibility.
|
| 169 |
+
**Violation of these terms constitutes a breach of license and will trigger takedown actions** including legal and technical responses.
|
| 170 |
+
|
| 171 |
+
*Volkan Kücükbudak*
|
| 172 |
+
|
| 173 |
+
-----
|
| 174 |
+
|
| 175 |
+
## Credits
|
| 176 |
+
|
| 177 |
+
- Developed and maintained by the PyFundaments project.
|
| 178 |
+
- Core components inspired by best practices in modular app architecture and OWASP security principles.
|
| 179 |
+
- Third-party libraries are credited in their respective module docs and comply with open-source terms.
|
| 180 |
+
|
| 181 |
+
-----
|
| 182 |
+
|
| 183 |
+
## Notes
|
| 184 |
+
|
| 185 |
+
Security-first by design.
|
| 186 |
+
If one piece is missing or unsafe — the app does not run.
|
| 187 |
+
Zero tolerance for guesswork.
|
| 188 |
+
|
| 189 |
+
> Give a ⭐ if you find the structure helpful.
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
## Changelog
|
| 193 |
+
###### Version 1.0.0 -> 1.0.1
|
| 194 |
+
### PyFundaments Refactoring
|
| 195 |
+
|
| 196 |
+
#### Modified Files
|
| 197 |
+
|
| 198 |
+
**1. main.py - Conditional Service Loading**
|
| 199 |
+
- **Added:** Environment-based conditional service initialization
|
| 200 |
+
- **Added:** Graceful fallback when services can't be initialized (warning instead of crash)
|
| 201 |
+
- **Added:** Conditional logging configuration (LOG_LEVEL, LOG_TO_TMP, ENABLE_PUBLIC_LOGS)
|
| 202 |
+
- **Added:** Smart dependency management (access_control & user_handler only load if database available)
|
| 203 |
+
- **Added:** Safe shutdown (only close DB pool if it was initialized)
|
| 204 |
+
- **Result:** Framework now supports partial service loading for different app types
|
| 205 |
+
|
| 206 |
+
**2. app/app.py - Proper Service Injection**
|
| 207 |
+
- **Removed:** Direct service imports and instantiation
|
| 208 |
+
- **Added:** Services received as parameter from main.py
|
| 209 |
+
- **Modified:** start_application() now takes fundaments dictionary as parameter
|
| 210 |
+
- **Added:** Conditional service usage based on what main.py provides
|
| 211 |
+
- **Added:** Examples for different app types (database-only, database-free modes)
|
| 212 |
+
- **Result:** Clean separation - main.py handles initialization, app.py uses services
|
| 213 |
+
|
| 214 |
+
**3. fundaments/config_handler.py - Universal Configuration Loader**
|
| 215 |
+
- **Removed:** REQUIRED_KEYS validation (moved to main.py)
|
| 216 |
+
- **Modified:** Now loads ALL environment variables without validation
|
| 217 |
+
- **Added:** Helper methods: get_bool(), get_int(), has(), get_all()
|
| 218 |
+
- **Added:** Safe defaults and type conversion
|
| 219 |
+
- **Result:** Config handler never needs updates, works with any ENV variables
|
| 220 |
+
|
| 221 |
+
**4. .env.example - Conditional Configuration Template**
|
| 222 |
+
- **Added:** Clear separation between core and optional dependencies
|
| 223 |
+
- **Added:** Logging configuration options
|
| 224 |
+
- **Added:** App-specific configuration examples (Discord, ML, Web)
|
| 225 |
+
- **Added:** Comments explaining when to use each option
|
| 226 |
+
- **Result:** Users only configure what they need, no unused variables
|
| 227 |
+
|
| 228 |
+
#### Architecture Improvements
|
| 229 |
+
- **Conditional Loading:** Framework only loads needed services based on available ENV vars
|
| 230 |
+
- **Merge-Safe Structure:** User code (app/, config_handler.py) protected from framework updates
|
| 231 |
+
- **Zero Breaking Changes:** Updates only affect fundaments/ directory
|
| 232 |
+
- **Enterprise-Level Merging:** Community can safely pull framework updates</document_content></document>
|
README.md
CHANGED
|
@@ -1,15 +1,516 @@
|
|
| 1 |
---
|
| 2 |
-
title:
|
| 3 |
-
emoji:
|
| 4 |
-
colorFrom:
|
| 5 |
-
colorTo:
|
| 6 |
-
sdk:
|
| 7 |
-
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
|
| 14 |
-
|
| 15 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
---
|
| 2 |
+
title: Multi-LLM API Gateway
|
| 3 |
+
emoji: 🛡️
|
| 4 |
+
colorFrom: indigo
|
| 5 |
+
colorTo: red
|
| 6 |
+
sdk: docker
|
| 7 |
+
pinned: true
|
| 8 |
+
license: apache-2.0
|
| 9 |
+
short_description: Secure Multi-LLM Gateway — (Streamable HTTP / SSE)
|
| 10 |
+
---
|
| 11 |
+
|
| 12 |
+
# Multi-LLM API Gateway
|
| 13 |
+
|
| 14 |
+
— or Universal MCP Hub (Sandboxed)
|
| 15 |
+
— or secure AI wrapper with dual interface: REST + MCP
|
| 16 |
+
|
| 17 |
+
aka: a clean, secure starting point for your own projects.
|
| 18 |
+
Pick the description that fits your use case. They're all correct.
|
| 19 |
+
|
| 20 |
+
> A production-grade **the-thing** that actually thinks about security.
|
| 21 |
+
> Built on [PyFundaments](PyFundaments.md) — running on **simpleCity**.
|
| 22 |
+
|
| 23 |
+
```
|
| 24 |
+
No key → no tool → no crash → no exposed secrets
|
| 25 |
+
```
|
| 26 |
+
|
| 27 |
+
> [!WARNING]
|
| 28 |
+
> Most MCP servers are prompts dressed up as servers. This one has a real architecture.
|
| 29 |
+
|
| 30 |
+
---
|
| 31 |
+
|
| 32 |
+
> [!IMPORTANT]
|
| 33 |
+
> This project is under active development — always use the latest release from [Codey Lab](https://github.com/Codey-LAB/Multi-LLM-API-Gateway) *(more stable builds land here first)*.
|
| 34 |
+
> This repo ([DEV](https://github.com/VolkanSah/Multi-LLM-API-Gateway)) is where the chaos happens. 🔬 A ⭐ on the repos will be cool 😙
|
| 35 |
+
|
| 36 |
+
---
|
| 37 |
+
|
| 38 |
+
## What's new in v1.1.0 (Rebuild)
|
| 39 |
+
|
| 40 |
+
> Full changelog in [`REVIEW.md`](REVIEW.md). Highlights:
|
| 41 |
+
|
| 42 |
+
- **Fixed:** `/mcp` 404 (mount middleware stripped the prefix FastMCP expects) and the missing session-manager lifespan — MCP is fully functional again.
|
| 43 |
+
- **New:** **Auto model pricing** — costs are fetched from the public OpenRouter models API + the LiteLLM community price list, cached in `db_sync` (24h TTL). Your `.pyfun [MODELS]` values always win as override. Manual cost maintenance is over.
|
| 44 |
+
- **New:** Brave + Tavily search providers implemented with fallback chain — `web_search` registers as soon as `BRAVE_API_KEY` or `TAVILY_API_KEY` is set.
|
| 45 |
+
- **New:** `db_query` as MCP tool (SELECT-only, app/* tables).
|
| 46 |
+
- **Cleanup:** dead code removed, `requirements.txt` now pins `mcp>=1.10,<2` (not `fastmcp`), LF line endings.
|
| 47 |
+
|
| 48 |
+
## Why this exists
|
| 49 |
+
|
| 50 |
+
The AI ecosystem is full of servers with hardcoded keys, `os.environ` scattered everywhere, zero sandboxing. One misconfigured fork and your API keys are gone.
|
| 51 |
+
|
| 52 |
+
This is exactly the kind of negligence (and worse — outright fraud) that [Wall of Shames](https://github.com/Wall-of-Shames) documents: fake "AI tools" exploiting non-technical users — API wrappers dressed up as custom models, Telegram payment funnels, bought stars. If you build on open source, you should know this exists.
|
| 53 |
+
|
| 54 |
+
This hub is the antidote:
|
| 55 |
+
|
| 56 |
+
- **Structural sandboxing** — `app/*` can never touch `fundaments/` or `.env`. Not by convention. By design.
|
| 57 |
+
- **Guardian pattern** — `main.py` is the only process that reads secrets. It injects validated services as a dict. `app/*` never sees the raw environment.
|
| 58 |
+
- **Graceful degradation** — No key? Tool doesn't register. Server still starts. No crash, no error, no empty `None` floating around.
|
| 59 |
+
- **Single source of truth** — All tool/provider/model config lives in `app/.pyfun`. Adding a provider = edit one file. No code changes.
|
| 60 |
+
|
| 61 |
+
---
|
| 62 |
+
|
| 63 |
+
## Two Interfaces — One Server
|
| 64 |
+
|
| 65 |
+
This hub exposes **two completely independent interfaces** on the same hypercorn instance:
|
| 66 |
+
|
| 67 |
+
```
|
| 68 |
+
POST /api → REST interface — for custom clients, desktop apps, CMS plugins
|
| 69 |
+
GET+POST /mcp → MCP interface — for Claude Desktop, Cursor, Windsurf, any MCP client
|
| 70 |
+
GET / → Health check — uptime, status
|
| 71 |
+
```
|
| 72 |
+
|
| 73 |
+
They share the same tool registry, provider config, and fallback chain. Adding a tool once makes it available on both interfaces automatically.
|
| 74 |
+
|
| 75 |
+
### REST API (`/api`)
|
| 76 |
+
|
| 77 |
+
Simple JSON POST — no protocol overhead, works with any HTTP client:
|
| 78 |
+
|
| 79 |
+
```json
|
| 80 |
+
POST /api
|
| 81 |
+
{"tool": "llm_complete", "params": {"prompt": "Hello", "provider": "anthropic"}}
|
| 82 |
+
```
|
| 83 |
+
|
| 84 |
+
Used by: Desktop Client (`DESKTOP_CLIENT/hub.py`), WordPress plugin, any custom integration.
|
| 85 |
+
|
| 86 |
+
### MCP Interface (`/mcp`)
|
| 87 |
+
|
| 88 |
+
Full MCP protocol — tool discovery, structured calls, streaming responses.
|
| 89 |
+
|
| 90 |
+
**Primary transport: Streamable HTTP** (MCP spec 2025-11-25)
|
| 91 |
+
**Fallback transport: SSE** (legacy, configurable via `.pyfun`)
|
| 92 |
+
|
| 93 |
+
Configured via `HUB_TRANSPORT` in `app/.pyfun [HUB]`:
|
| 94 |
+
|
| 95 |
+
```ini
|
| 96 |
+
HUB_TRANSPORT = "streamable-http" # default — MCP spec 2025-11-25
|
| 97 |
+
# HUB_TRANSPORT = "sse" # legacy fallback for older clients
|
| 98 |
+
```
|
| 99 |
+
|
| 100 |
+
Used by: Claude Desktop, Cursor, Windsurf, any MCP-compatible client.
|
| 101 |
+
|
| 102 |
+
---
|
| 103 |
+
|
| 104 |
+
## Architecture
|
| 105 |
+
|
| 106 |
+
```
|
| 107 |
+
main.py (Guardian)
|
| 108 |
+
│
|
| 109 |
+
│ reads .env / HF Secrets
|
| 110 |
+
│ initializes fundaments/* conditionally
|
| 111 |
+
│ injects validated services as dict
|
| 112 |
+
│
|
| 113 |
+
└──► app/app.py (Orchestrator, sandboxed)
|
| 114 |
+
│
|
| 115 |
+
│ unpacks fundaments ONCE, at startup, never stores globally
|
| 116 |
+
│ starts hypercorn (async ASGI)
|
| 117 |
+
│ routes: GET / | POST /api | /mcp (transport-dependent)
|
| 118 |
+
│
|
| 119 |
+
├── app/mcp.py ← FastMCP + transport handler (Streamable HTTP / SSE)
|
| 120 |
+
├── app/tools.py ← Tool registry (key-gated)
|
| 121 |
+
├── app/providers.py ← LLM + Search execution + fallback chain
|
| 122 |
+
├── app/models.py ← Model limits, costs, capabilities
|
| 123 |
+
├── app/config.py ← .pyfun parser (single source of truth)
|
| 124 |
+
└── app/db_sync.py ← Internal SQLite IPC (app/* state only)
|
| 125 |
+
≠ fundaments/postgresql.py (Guardian-only)
|
| 126 |
+
```
|
| 127 |
+
|
| 128 |
+
**The sandbox is structural:**
|
| 129 |
+
|
| 130 |
+
```python
|
| 131 |
+
# app/app.py — fundaments unpacked ONCE, NEVER stored globally
|
| 132 |
+
async def start_application(fundaments: Dict[str, Any]) -> None:
|
| 133 |
+
config_service = fundaments["config"]
|
| 134 |
+
db_service = fundaments["db"] # None if not configured
|
| 135 |
+
encryption_service = fundaments["encryption"] # None if keys missing
|
| 136 |
+
access_control_service = fundaments["access_control"]
|
| 137 |
+
...
|
| 138 |
+
# From here: app/* reads its own config from app/.pyfun only.
|
| 139 |
+
# fundaments are never passed into other app/* modules.
|
| 140 |
+
```
|
| 141 |
+
|
| 142 |
+
`app/app.py` never calls `os.environ`. Never imports from `fundaments/`. Never reads `.env`.
|
| 143 |
+
This isn't documentation. It's enforced by the import structure.
|
| 144 |
+
|
| 145 |
+
### Why Quart + hypercorn?
|
| 146 |
+
|
| 147 |
+
**Quart** is async Flask — fully `async/await` native. FastMCP's handlers are async; mixing sync Flask would require thread hacks. With Quart, `/mcp` hands off directly to FastMCP — no bridging, no blocking.
|
| 148 |
+
|
| 149 |
+
**hypercorn** is an ASGI server (vs. waitress/gunicorn which are WSGI). WSGI servers handle one request per thread — wrong for long-lived MCP connections. hypercorn handles both Streamable HTTP and SSE natively, and runs without extra config on HuggingFace Spaces. HTTP/2 support (`config.h2 = True`) is built-in — relevant for Streamable HTTP performance at scale.
|
| 150 |
+
|
| 151 |
+
The `/mcp` route in `app.py` remains the natural interception point regardless of transport — auth checks, rate limiting, and logging can all be added there before the request reaches FastMCP.
|
| 152 |
+
|
| 153 |
+
---
|
| 154 |
+
|
| 155 |
+
## Two Databases — One Architecture
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
```
|
| 159 |
+
┌─────────────────────────────────────────────────────────────┐
|
| 160 |
+
│ Guardian Layer (fundaments/*) │
|
| 161 |
+
│ │
|
| 162 |
+
│ postgresql.py → Cloud DB (e.g. Neon, Supabase) │
|
| 163 |
+
│ asyncpg pool, SSL enforced │
|
| 164 |
+
│ │
|
| 165 |
+
│ user_handler.py → SQLite (users + sessions tables) │
|
| 166 |
+
│ PBKDF2-SHA256 password hashing │
|
| 167 |
+
│ Session validation incl. IP + UserAgent │
|
| 168 |
+
│ Account lockout after 5 failed attempts │
|
| 169 |
+
│ │
|
| 170 |
+
└──────────────────────┬──────────────────────────────────────┘
|
| 171 |
+
│ inject as fundaments dict
|
| 172 |
+
▼
|
| 173 |
+
┌─────────────────────────────────────────────────────────────┐
|
| 174 |
+
│ App Layer (app/*) │
|
| 175 |
+
│ │
|
| 176 |
+
│ db_sync.py → SQLite (hub_state + tool_cache tables) │
|
| 177 |
+
│ aiosqlite (async, non-blocking) │
|
| 178 |
+
│ NEVER touches users/sessions tables │
|
| 179 |
+
│ Relocated to /tmp/ on HF Spaces auto │
|
| 180 |
+
│ │
|
| 181 |
+
└─────────────────────────────────────────────────────────────┘
|
| 182 |
+
```
|
| 183 |
+
|
| 184 |
+
**Table ownership — hard rule:**
|
| 185 |
+
|
| 186 |
+
| Table | Owner | Access |
|
| 187 |
+
| :--- | :--- | :--- |
|
| 188 |
+
| `users` | `fundaments/user_handler.py` | Guardian only |
|
| 189 |
+
| `sessions` | `fundaments/user_handler.py` | Guardian only |
|
| 190 |
+
| `hub_state` | `app/db_sync.py` | app/* only |
|
| 191 |
+
| `tool_cache` | `app/db_sync.py` | app/* only |
|
| 192 |
+
| `hub_results` | PostgreSQL / Guardian | via `persist_result` tool |
|
| 193 |
+
|
| 194 |
+
---
|
| 195 |
+
|
| 196 |
+
## Tools
|
| 197 |
+
|
| 198 |
+
Tools register at startup — only if the required API key exists. No key, no tool. Server always starts.
|
| 199 |
+
|
| 200 |
+
| ENV Secret | Tool | Notes |
|
| 201 |
+
| :--- | :--- | :--- |
|
| 202 |
+
| `ANTHROPIC_API_KEY` | `llm_complete` | Claude Haiku / Sonnet / Opus |
|
| 203 |
+
| `GEMINI_API_KEY` | `llm_complete` | Gemini 2.0 / 2.5 / 3.x Flash & Pro |
|
| 204 |
+
| `OPENROUTER_API_KEY` | `llm_complete` | 100+ models via OpenRouter |
|
| 205 |
+
| `HF_TOKEN` | `llm_complete` | HuggingFace Inference API |
|
| 206 |
+
| `BRAVE_API_KEY` | `web_search` | Independent web index |
|
| 207 |
+
| `TAVILY_API_KEY` | `web_search` | AI-optimized search with synthesized answers |
|
| 208 |
+
| `DATABASE_URL` | `cloud DB` | e.g. Neon, Supabase |
|
| 209 |
+
| `DATABASE_URL` | `db_query`, `persist_result` | SQLite read + PostgreSQL write |
|
| 210 |
+
| *(always)* | `list_active_tools` | Shows key names only — never values |
|
| 211 |
+
| *(always)* | `health_check` | Status + uptime + active transport |
|
| 212 |
+
| *(always)* | `get_model_info` | Limits, costs, capabilities per model (auto-merged) |
|
| 213 |
+
| *(always)* | `get_model_pricing` | Auto-fetched costs — `.pyfun` values win as override |
|
| 214 |
+
| *(always)* | `refresh_pricing` | Force refresh from OpenRouter + LiteLLM |
|
| 215 |
+
| *(always)* | `db_query` | Read-only SELECT on internal hub state |
|
| 216 |
+
|
| 217 |
+
For all key names see [`app/.pyfun`](app/.pyfun).
|
| 218 |
+
|
| 219 |
+
**Tools are configured in `.pyfun` — including system prompts:**
|
| 220 |
+
|
| 221 |
+
```ini
|
| 222 |
+
[TOOL.code_review]
|
| 223 |
+
active = "true"
|
| 224 |
+
description = "Review code for bugs, security issues and improvements"
|
| 225 |
+
provider_type = "llm"
|
| 226 |
+
default_provider = "anthropic"
|
| 227 |
+
timeout_sec = "60"
|
| 228 |
+
system_prompt = "You are an expert code reviewer. Analyze the given code for bugs, security issues, and improvements. Be specific and concise."
|
| 229 |
+
[TOOL.code_review_END]
|
| 230 |
+
```
|
| 231 |
+
|
| 232 |
+
Current built-in tools: `llm_complete`, `code_review`, `summarize`, `translate`, `web_search`, `db_query`
|
| 233 |
+
Future hooks (commented, ready): `image_gen`, `code_exec`, `shellmaster_2.0`, Discord, GitHub webhooks
|
| 234 |
+
|
| 235 |
+
---
|
| 236 |
+
|
| 237 |
+
## LLM Fallback Chain
|
| 238 |
+
|
| 239 |
+
All LLM providers share one `llm_complete` tool. If a provider fails, the hub walks the fallback chain from `.pyfun`:
|
| 240 |
+
|
| 241 |
+
```
|
| 242 |
+
e.g. anthropic → gemini → openrouter → huggingface
|
| 243 |
+
```
|
| 244 |
+
|
| 245 |
+
```ini
|
| 246 |
+
[LLM_PROVIDER.anthropic]
|
| 247 |
+
fallback_to = "gemini"
|
| 248 |
+
[LLM_PROVIDER.anthropic_END]
|
| 249 |
+
|
| 250 |
+
[LLM_PROVIDER.gemini]
|
| 251 |
+
fallback_to = "openrouter"
|
| 252 |
+
[LLM_PROVIDER.gemini_END]
|
| 253 |
+
```
|
| 254 |
+
|
| 255 |
+
Same pattern applies to search providers (`brave → tavily`).
|
| 256 |
+
|
| 257 |
+
---
|
| 258 |
+
|
| 259 |
+
## Quick Start
|
| 260 |
+
|
| 261 |
+
### HuggingFace Spaces (recommended)
|
| 262 |
+
|
| 263 |
+
1. Fork / duplicate this Space
|
| 264 |
+
2. Go to **Settings → Variables and secrets**
|
| 265 |
+
3. Add the API keys you have (any subset works)
|
| 266 |
+
4. Space starts automatically — only tools with valid keys register
|
| 267 |
+
|
| 268 |
+
[→ Live Demo Space](https://huggingface.co/spaces/codey-lab/Multi-LLM-API-Gateway) (no LLM keys set)
|
| 269 |
+
|
| 270 |
+
### Local / Docker
|
| 271 |
+
|
| 272 |
+
```bash
|
| 273 |
+
git clone https://github.com/VolkanSah/Multi-LLM-API-Gateway
|
| 274 |
+
cd Multi-LLM-API-Gateway
|
| 275 |
+
cp example-mcp___.env .env
|
| 276 |
+
# fill in your keys
|
| 277 |
+
pip install -r requirements.txt
|
| 278 |
+
python main.py
|
| 279 |
+
```
|
| 280 |
+
|
| 281 |
+
Minimum required ENV vars (everything else is optional):
|
| 282 |
+
|
| 283 |
+
```env
|
| 284 |
+
PYFUNDAMENTS_DEBUG=""
|
| 285 |
+
LOG_LEVEL="INFO"
|
| 286 |
+
LOG_TO_TMP=""
|
| 287 |
+
ENABLE_PUBLIC_LOGS="true"
|
| 288 |
+
HF_TOKEN=""
|
| 289 |
+
HUB_SPACE_URL=""
|
| 290 |
+
```
|
| 291 |
+
|
| 292 |
+
Transport is configured in `app/.pyfun [HUB]` — not via ENV.
|
| 293 |
+
|
| 294 |
+
---
|
| 295 |
+
|
| 296 |
+
## Connect an MCP Client
|
| 297 |
+
|
| 298 |
+
### Streamable HTTP (default — MCP spec 2025-11-25)
|
| 299 |
+
|
| 300 |
+
```json
|
| 301 |
+
{
|
| 302 |
+
"mcpServers": {
|
| 303 |
+
"universal-mcp-hub": {
|
| 304 |
+
"url": "https://YOUR_USERNAME-universal-mcp-hub.hf.space/mcp"
|
| 305 |
+
}
|
| 306 |
+
}
|
| 307 |
+
}
|
| 308 |
+
```
|
| 309 |
+
|
| 310 |
+
### Streamable HTTP — Private Space (with HF token)
|
| 311 |
+
|
| 312 |
+
```json
|
| 313 |
+
{
|
| 314 |
+
"mcpServers": {
|
| 315 |
+
"universal-mcp-hub": {
|
| 316 |
+
"url": "https://YOUR_USERNAME-universal-mcp-hub.hf.space/mcp",
|
| 317 |
+
"headers": {
|
| 318 |
+
"Authorization": "Bearer hf_..."
|
| 319 |
+
}
|
| 320 |
+
}
|
| 321 |
+
}
|
| 322 |
+
}
|
| 323 |
+
```
|
| 324 |
+
|
| 325 |
+
### SSE legacy fallback (set `HUB_TRANSPORT = "sse"` in `.pyfun`)
|
| 326 |
+
|
| 327 |
+
```json
|
| 328 |
+
{
|
| 329 |
+
"mcpServers": {
|
| 330 |
+
"universal-mcp-hub": {
|
| 331 |
+
"url": "https://YOUR_USERNAME-universal-mcp-hub.hf.space/mcp"
|
| 332 |
+
}
|
| 333 |
+
}
|
| 334 |
+
}
|
| 335 |
+
```
|
| 336 |
+
|
| 337 |
+
> Same URL (`/mcp`) for both transports — the protocol is negotiated automatically.
|
| 338 |
+
> SSE fallback is for older clients that don't support Streamable HTTP yet.
|
| 339 |
+
|
| 340 |
+
---
|
| 341 |
+
|
| 342 |
+
## Desktop Client
|
| 343 |
+
###### (experimental — ~80% AI generated)
|
| 344 |
+
|
| 345 |
+
A full PySide6 desktop client is included in `DESKTOP_CLIENT/hub.py`.
|
| 346 |
+
Communicates via the REST `/api` endpoint — no MCP protocol overhead.
|
| 347 |
+
Ideal for private or non-public Spaces.
|
| 348 |
+
|
| 349 |
+
```bash
|
| 350 |
+
pip install PySide6 httpx
|
| 351 |
+
# optional file handling:
|
| 352 |
+
pip install Pillow PyPDF2 pandas openpyxl
|
| 353 |
+
python DESKTOP_CLIENT/hub.py
|
| 354 |
+
```
|
| 355 |
+
|
| 356 |
+
**Features:**
|
| 357 |
+
- Multi-chat with persistent history
|
| 358 |
+
- Tool / Provider / Model selector loaded live from your Hub
|
| 359 |
+
- File attachments: images, PDF, CSV, Excel, ZIP, source code
|
| 360 |
+
- Connect tab with health check + auto-load
|
| 361 |
+
- Settings: HF Token + Hub URL saved locally, never sent anywhere except your own Hub
|
| 362 |
+
- Full request/response log with timestamps
|
| 363 |
+
- Runs on Windows, Linux, macOS
|
| 364 |
+
|
| 365 |
+
[→ Desktop Client docs](DESKTOP_CLIENT/README.md)
|
| 366 |
+
|
| 367 |
+
---
|
| 368 |
+
|
| 369 |
+
## CMS & Custom Clients
|
| 370 |
+
|
| 371 |
+
| Client | Interface used | Notes |
|
| 372 |
+
| :--- | :--- | :--- |
|
| 373 |
+
| [Desktop Client](DESKTOP_CLIENT/hub.py) | REST `/api` | PySide6, local |
|
| 374 |
+
| [WP AI Hub](https://github.com/VolkanSah/WP-AI-HUB/) | REST `/api` | WordPress plugin |
|
| 375 |
+
| TYPO3 (soon) | REST `/api` | — |
|
| 376 |
+
| Claude Desktop | MCP `/mcp` | Streamable HTTP |
|
| 377 |
+
| Cursor / Windsurf | MCP `/mcp` | Streamable HTTP |
|
| 378 |
+
|
| 379 |
+
---
|
| 380 |
+
|
| 381 |
+
## Configuration (.pyfun)
|
| 382 |
+
|
| 383 |
+
`app/.pyfun` is the single source of truth for all app behavior. Three tiers:
|
| 384 |
+
|
| 385 |
+
```
|
| 386 |
+
LAZY: [HUB] + one [LLM_PROVIDER.*] → works
|
| 387 |
+
NORMAL: + [SEARCH_PROVIDER.*] + [MODELS.*] → works better
|
| 388 |
+
PRODUCTIVE: + [TOOLS] + [HUB_LIMITS] + [DB_SYNC] → full power
|
| 389 |
+
```
|
| 390 |
+
|
| 391 |
+
Key settings in `[HUB]`:
|
| 392 |
+
|
| 393 |
+
```ini
|
| 394 |
+
[HUB]
|
| 395 |
+
HUB_TRANSPORT = "streamable-http" # streamable-http | sse
|
| 396 |
+
HUB_STATELESS = "true" # true = HF Spaces safe, no session state
|
| 397 |
+
HUB_PORT = "7860"
|
| 398 |
+
[HUB_END]
|
| 399 |
+
```
|
| 400 |
+
|
| 401 |
+
Adding a new LLM provider — two steps:
|
| 402 |
+
|
| 403 |
+
```ini
|
| 404 |
+
# 1. app/.pyfun
|
| 405 |
+
[LLM_PROVIDER.mistral]
|
| 406 |
+
active = "true"
|
| 407 |
+
base_url = "https://api.mistral.ai/v1"
|
| 408 |
+
env_key = "MISTRAL_API_KEY"
|
| 409 |
+
default_model = "mistral-large-latest"
|
| 410 |
+
models = "mistral-large-latest, mistral-small-latest"
|
| 411 |
+
fallback_to = ""
|
| 412 |
+
[LLM_PROVIDER.mistral_END]
|
| 413 |
+
```
|
| 414 |
+
|
| 415 |
+
```python
|
| 416 |
+
# 2. app/providers.py — uncomment the dummy
|
| 417 |
+
_PROVIDER_CLASSES = {
|
| 418 |
+
...
|
| 419 |
+
"mistral": MistralProvider, # ← uncomment to activate
|
| 420 |
+
}
|
| 421 |
+
```
|
| 422 |
+
|
| 423 |
+
---
|
| 424 |
+
|
| 425 |
+
## Dependencies
|
| 426 |
+
|
| 427 |
+
```
|
| 428 |
+
# PyFundaments Core (always required)
|
| 429 |
+
asyncpg — async PostgreSQL pool (Guardian/cloud DB)
|
| 430 |
+
python-dotenv — .env loading
|
| 431 |
+
passlib — PBKDF2 password hashing in user_handler.py
|
| 432 |
+
cryptography — encryption layer in fundaments/
|
| 433 |
+
|
| 434 |
+
# MCP Hub
|
| 435 |
+
mcp — MCP protocol + FastMCP (Streamable HTTP + SSE)
|
| 436 |
+
httpx — async HTTP for all provider API calls
|
| 437 |
+
quart — async Flask (ASGI) — needed for MCP + hypercorn
|
| 438 |
+
hypercorn — ASGI server — Streamable HTTP + SSE, HF Spaces native
|
| 439 |
+
requests — sync HTTP for tool workers
|
| 440 |
+
|
| 441 |
+
# Optional (uncomment in requirements.txt as needed)
|
| 442 |
+
# aiofiles — async file ops (ML pipelines, file uploads)
|
| 443 |
+
# discord.py — Discord bot integration (planned)
|
| 444 |
+
# PyNaCl — Discord signature verification
|
| 445 |
+
# psycopg2-binary — alternative PostgreSQL driver
|
| 446 |
+
```
|
| 447 |
+
|
| 448 |
+
> **Note:** The package is `mcp` (not `fastmcp`) — `FastMCP` is imported from `mcp.server.fastmcp`.
|
| 449 |
+
> Streamable HTTP support requires `mcp >= 1.6.0`.
|
| 450 |
+
|
| 451 |
+
---
|
| 452 |
+
|
| 453 |
+
## Security Design
|
| 454 |
+
|
| 455 |
+
- API keys live in HF Secrets / `.env` — never in `.pyfun`, never in code
|
| 456 |
+
- `list_active_tools` returns key **names** only — never values
|
| 457 |
+
- `db_query` is SELECT-only, enforced at application level (not just docs)
|
| 458 |
+
- `app/*` has zero import access to `fundaments/` internals
|
| 459 |
+
- Direct execution of `app/app.py` blocked by design — warning + null-fundaments fallback
|
| 460 |
+
- `fundaments/` initialized conditionally — missing services degrade gracefully, never crash
|
| 461 |
+
- Streamable HTTP uses standard Bearer headers — no token-in-URL (unlike SSE)
|
| 462 |
+
|
| 463 |
+
> PyFundaments is not perfect. But it's more secure than most of what runs in production today.
|
| 464 |
+
|
| 465 |
+
[→ Full Security Policy](SECURITY.md)
|
| 466 |
+
|
| 467 |
+
---
|
| 468 |
+
|
| 469 |
+
## Foundation
|
| 470 |
+
|
| 471 |
+
Built on [PyFundaments](PyFundaments.md) — a security-first Python boilerplate:
|
| 472 |
+
|
| 473 |
+
- `config_handler.py` — env loading with validation
|
| 474 |
+
- `postgresql.py` — async DB pool (Guardian-only)
|
| 475 |
+
- `encryption.py` — key-based encryption layer
|
| 476 |
+
- `access_control.py` — role/permission management
|
| 477 |
+
- `user_handler.py` — user lifecycle management
|
| 478 |
+
- `security.py` — unified security manager composing the above
|
| 479 |
+
|
| 480 |
+
None accessible from `app/*`. Injected as a validated dict by `main.py`.
|
| 481 |
+
|
| 482 |
+
[→ PyFundaments Function Overview](PyFundaments%20–%20Function%20Overview.md)
|
| 483 |
+
[→ Module Docs](docs/app/)
|
| 484 |
+
[→ Source Repo](https://github.com/VolkanSah/Multi-LLM-API-Gateway)
|
| 485 |
+
|
| 486 |
+
---
|
| 487 |
+
|
| 488 |
+
## Related Projects
|
| 489 |
+
|
| 490 |
+
- [Customs LLMs for free — Build Your Own LLM Service](https://github.com/VolkanSah/SmolLM2-customs/)
|
| 491 |
+
- [WP AI Hub (WordPress Client)](https://github.com/VolkanSah/WP-AI-HUB/)
|
| 492 |
+
- [ShellMaster (2023 precursor)](https://github.com/VolkanSah/ChatGPT-ShellMaster)
|
| 493 |
+
|
| 494 |
+
---
|
| 495 |
+
|
| 496 |
+
## History
|
| 497 |
+
|
| 498 |
+
[ShellMaster](https://github.com/VolkanSah/ChatGPT-ShellMaster) (2023, MIT) was the precursor — browser-accessible shell for ChatGPT with session memory, built before MCP was a concept. Universal MCP Hub is its natural evolution: same idea, proper architecture, dual interface.
|
| 499 |
+
|
| 500 |
+
---
|
| 501 |
+
|
| 502 |
+
## License
|
| 503 |
+
|
| 504 |
+
Dual-licensed:
|
| 505 |
+
|
| 506 |
+
- [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0)
|
| 507 |
+
- [Ethical Security Operations License v1.1 (ESOL)](ESOL) — mandatory, non-severable
|
| 508 |
+
|
| 509 |
+
By using this software you agree to all ethical constraints defined in ESOL v1.1.
|
| 510 |
+
|
| 511 |
+
---
|
| 512 |
+
|
| 513 |
+
*Architecture, security decisions, and PyFundaments by Volkan Kücükbudak.*
|
| 514 |
+
*Built with Claude (Anthropic) as a typing assistant for docs (and the occasional bug).*
|
| 515 |
+
|
| 516 |
+
> crafted with passion — just wanted to understand how it works, don't actually need it, have a CLI 😄
|
REVIEW.md
ADDED
|
@@ -0,0 +1,250 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Code-Review & Rebuild — Multi-LLM-API-Gateway v1.1.0
|
| 2 |
+
|
| 3 |
+
**Datum:** 10.11.2026 · **Basis:** `Multi-LLM-API-Gateway_brocken.7z` + `PyFundaments-Quart-Boilerplate.zip`
|
| 4 |
+
**Getestet:** Live-Boot, MCP-Session-Flow, /api-Endpunkte, echtes Pricing-Fetch (OpenRouter + LiteLLM), Fehlerpfade
|
| 5 |
+
|
| 6 |
+
---
|
| 7 |
+
|
| 8 |
+
## 1. Kurzfassung
|
| 9 |
+
|
| 10 |
+
Der Gateway **startete**, aber **MCP war komplett tot** (`POST /mcp` → 404). Zwei verlorene Boilerplate-Konzepte waren die Ursache. Nach dem Fix wurde der Hub um Auto-Model-Pricing erweitert (dein Schmerzpunkt: Kosten nie wieder von Hand pflegen) und um echte Search-Provider.
|
| 11 |
+
|
| 12 |
+
**Finale Tool-Liste (live getestet):**
|
| 13 |
+
`llm_complete` · `web_search` · `list_active_tools` · `health_check` · `get_model_info` · `get_model_pricing` · `refresh_pricing` · `db_query`
|
| 14 |
+
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
## 2. Was kaputt war (mit Beweis)
|
| 18 |
+
|
| 19 |
+
| # | Befund | Beweis / Test |
|
| 20 |
+
|---|--------|---------------|
|
| 21 |
+
| **A1** | `/mcp` → **404**. Die Mount-Middleware stripte den `/mcp`-Prefix, aber FastMCP registriert seine Route **intern selbst** unter `/mcp` (`settings.streamable_http_path`). Der gestripte Pfad `/` existiert in der inneren Starlette-App nicht. | Minimal-Repro: gestript → 404; Pfad unangetastet → erreichbar. Das Boilerplate hat genau das mit `strip_prefix=False` (Default) abgefangen. |
|
| 22 |
+
| **A2** | **Session-Manager lief nie.** `mcp.lifespan()` fehlte komplett, `serve()` lag nicht im `async with`. | Ohne lifespan: `RuntimeError: Task group is not initialized` bei jedem Request (nachgewiesen). |
|
| 23 |
+
| **A3** | **SSE-Fallback tot.** `mcp.handle_request()` rief `_mcp.handle_sse(request)` — diese Methode existiert im MCP-SDK 1.27 nicht. | SDK-Inspektion. |
|
| 24 |
+
| **A4** | `tools.run()` verlangte `prompt` positional → `/api` ohne `prompt` = roher 500er. | Live-Test. |
|
| 25 |
+
| **A5** | `providers.search()` war ein Stub („not yet implemented") — Brave/Tavily standen in der `.pyfun`, `web_search` registrierte sich nie. | Boot-Log: „No active search providers". |
|
| 26 |
+
| **A6** | `requirements.txt` pinnte `fastmcp` — dein Code importiert aber `mcp.server.fastmcp` (das **mcp**-SDK). Funktionierte nur zufällig, weil fastmcp mcp als Dep nachzieht. | `pip show fastmcp` → `mcp<2.0,>=1.24.0`. |
|
| 27 |
+
| **A7** | Leichen: `mcp_old.py` (379 Zeilen), leere `example.py`, `app/web/`, `app/tmp/`, und `app/web/tmp/__init__py` **ohne Punkt** im Dateinamen. | Datei-Inspektion. |
|
| 28 |
+
| **A8** | `.pyfun`: `[MODEL.gemini-2.5-flash-lite]` wurde mit `[MODEL.gemini-2.0-flash-lite_END]` geschlossen (Parser toleriertes es, war aber falsch); doppelter Header-Kommentar. | Parser-Review. |
|
| 29 |
+
| **A9** | `shellmaster.py` war nie verdrahtet — `initialize()` wurde von niemandem gerufen. | Import-Graph. |
|
| 30 |
+
| **A10** | CRLF-Zeilenenden überall (Windows-Checkout). | `file`-Check: „with CRLF line terminators". |
|
| 31 |
+
|
| 32 |
+
## 3. Fundament-Befunde (pyfun) — nur relevant mit `DATABASE_URL`
|
| 33 |
+
|
| 34 |
+
| # | Befund | Status |
|
| 35 |
+
|---|--------|--------|
|
| 36 |
+
| **D1** | `UserHandler` erwartet ein SQLite-`Database`-Objekt (`fetchone`/`execute`, synchron) — `main.py` übergab aber den **asyncpg-Pool** → `AttributeError` beim ersten Login. | **Gefixt:** `main.py` baut jetzt `UserHandler(Database())` mit `setup_tables()` — exakt wie im `__main__`-Demo deines eigenen `user_handler.py`. |
|
| 37 |
+
| **D2** | `Security.user_login()` awaited eine **synchrone** Methode → `TypeError: bool can't be awaited`. | **Gefixt:** `await` entfernt (UserHandler bleibt synchron/SQLite). |
|
| 38 |
+
| **D3** | `Security.check_permission(user_id, name)` rief `has_permission(name)` mit 2 Argumenten auf, obwohl die Signatur nur 1 nimmt → `TypeError`. | **Gefixt:** `has_permission(permission_name, user_id=None)` — optionaler Override, rückwärtskompatibel. |
|
| 39 |
+
| **D4** | `AccessControl()` wurde ohne `user_id` gebaut → Permission-Check wäre immer `False` gewesen. | **Mit D3 gelöst** (user_id kann pro Call übergeben werden). |
|
| 40 |
+
| **D5** | Gemischte Stores: users/sessions in **SQLite**, Rollen/Permissions in **PostgreSQL**. | **Dokumentiert, nicht geändert** — das ist ein Design-Decision, die dir gehört. |
|
| 41 |
+
| **D6** | `main.py` loggte „Database service initialized." obwohl `init_db_pool()` `None` zurückgab. | **Gefixt:** Rückgabewert wird geprüft, Warnung stattdessen. |
|
| 42 |
+
|
| 43 |
+
> **Hinweis fürs PyFundaments-Repo:** D1–D4 sind im Boilerplate genauso drin. Die Fixes (`main.py`-Wiring, `security.py`-Signaturen, `access_control.py`-Override) kannst du 1:1 übernehmen.
|
| 44 |
+
|
| 45 |
+
---
|
| 46 |
+
|
| 47 |
+
## 4. Was im Rebuild passiert ist (Datei für Datei)
|
| 48 |
+
|
| 49 |
+
### `app/app.py`
|
| 50 |
+
- Mount-Middleware zurück auf Boilerplate-Logik: `prefixes`-Tuple, **kein Strip**, `strip_prefix` nur noch bewusster Opt-in.
|
| 51 |
+
- `serve()` läuft jetzt in `async with mcp.lifespan():` — der FastMCP-Session-Manager lebt.
|
| 52 |
+
- `mcp.mount_prefixes()` bestimmt die Prefixe (streamable-http → `/mcp`; sse → `/sse` + `/messages`).
|
| 53 |
+
- `HUB_HOST` wird aus `.pyfun` gelesen (stand vorher auf `0.0.0.0` hardcoded).
|
| 54 |
+
- `/api`: saubere Fehlercodes — `ValueError` → 400 (fehlender Prompt, unbekanntes Tool), `KeyError` → 404, `RuntimeError` → 502 (Provider-Kette erschöpft, Message upstream sanitisiert), kaputtes JSON crasht nicht.
|
| 55 |
+
- `get_model_pricing` als REST-Tool direkt daneben.
|
| 56 |
+
- `shellmaster.initialize()` + `pricing.initialize()` in die Startup-Sequenz.
|
| 57 |
+
- Tote SSE-Quart-Route entfernt, ungenutzte Imports (`threading`, `requests`, `time`) raus, `datetime.utcnow()` (deprecated) → `datetime.now(timezone.utc)`.
|
| 58 |
+
|
| 59 |
+
### `app/mcp.py`
|
| 60 |
+
- `lifespan()` + `mount_prefixes()` zurück (mit dem WARUM-Kommentar aus dem Boilerplate).
|
| 61 |
+
- Tote `handle_request()`/`handle_sse()` entfernt.
|
| 62 |
+
- **Neu:** `_register_pricing_tools()` → `get_model_pricing` + `refresh_pricing`.
|
| 63 |
+
- `get_model_info` ist jetzt async und liefert **gemergte** Kosten (.pyfun gewinnt, Auto-Werte füllen).
|
| 64 |
+
- `_register_db_tools()` aktiviert — alle drei im Kommentar genannten Bedingungen sind erfüllt.
|
| 65 |
+
- `list_active_tools` zeigt jetzt `pricing_stats`.
|
| 66 |
+
|
| 67 |
+
### `app/pricing.py` — **NEU** (dein Schmerzpunkt)
|
| 68 |
+
- **Das Problem:** Nicht jeder Anbieter hat eine Pricing-API (Anthropic & Google: keine). Kosten von Hand in `[MODELS]` pflegen nervt.
|
| 69 |
+
- **Die Lösung:** Zwei freie, öffentliche Quellen + deine `.pyfun` als Override:
|
| 70 |
+
|
| 71 |
+
| Priorität | Quelle | Deckt | Getestet |
|
| 72 |
+
|-----------|--------|-------|----------|
|
| 73 |
+
| 1 | `.pyfun [MODELS]` | Deine manuellen Werte — **gewinnen immer** | ✅ `claude-opus-4-6` → 0.015/0.075 $/1k aus deiner Config |
|
| 74 |
+
| 2 | OpenRouter `/api/v1/models` (öffentlich, kein Key) | Alle OR-Modelle, per-Token-Preise, Context-Längen | ✅ 465 Modelle live geholt |
|
| 75 |
+
| 3 | LiteLLM `model_prices_and_context_window.json` (Community-Standard) | Anthropic, Gemini, OpenAI, Mistral, … | ✅ 4615 Modelle live geholt |
|
| 76 |
+
|
| 77 |
+
- **Merge-Logik:** `.pyfun` > OpenRouter > LiteLLM. Fehlt beides → `source: "unknown"`, kein Crash.
|
| 78 |
+
- **Matching:** exakt normalisiert (lowercase, Vendor-Prefixe wie `anthropic/`, `gemini/` gestrippt) + Loose-Match gegen Punkt/Strich-Differenzen (OR schreibt `claude-opus-4.6`, du `claude-opus-4-6` — beides findet sich).
|
| 79 |
+
- **Caching:** landet in `db_sync` (`hub_state["pricing.cache"]`), TTL aus `.pyfun` (Default 24 h). **Restart innerhalb der TTL = null externe Calls** (live getestet).
|
| 80 |
+
- **Nicht-blockierend:** Fetch läuft als Background-Task — der Server startet sofort, auch wenn beide Quellen down sind.
|
| 81 |
+
- **MCP/REST:** `get_model_pricing(model_name)` liefert alles gemergt; `refresh_pricing()` erzwingt den Refresh.
|
| 82 |
+
|
| 83 |
+
### `app/providers.py`
|
| 84 |
+
- **Neu:** `BaseSearchProvider` + `BraveSearchProvider` (Auth via `X-Subscription-Token`, **nicht** Bearer) + `TavilySearchProvider` (Key im Payload, `include_answer`-Support).
|
| 85 |
+
- `_initialize_search()` mit demselben Key-Gating wie die LLMs: kein Key = Provider übersprungen = kein Crash.
|
| 86 |
+
- `search()` mit Fallback-Kette via `fallback_to` (brave → tavily), mit Visited-Set gegen Endlosschleifen.
|
| 87 |
+
- Fehler sanitisiert wie bei LLMs: nur HTTP-Status + safe_url, niemals Keys/Bodies.
|
| 88 |
+
|
| 89 |
+
### `app/tools.py`
|
| 90 |
+
- `prompt` jetzt optional + `**params`-Fänger → unbekannte Parameter crashen nicht mehr.
|
| 91 |
+
- Fehlender Prompt → sauberer `ValueError` → 400.
|
| 92 |
+
|
| 93 |
+
### `app/config.py`, `app/models.py`
|
| 94 |
+
- `get_pricing()` für die neue `[PRICING]`-Sektion.
|
| 95 |
+
- `models.get()` fällt auf die rohe `.pyfun` zurück, falls das Registry noch nicht initialisiert ist.
|
| 96 |
+
|
| 97 |
+
### `app/.pyfun`
|
| 98 |
+
- **Neu:** `[PRICING]`-Sektion (Enabled, TTL, beide Quell-URLs).
|
| 99 |
+
- END-Marker-Fix + Doppel-Header entfernt.
|
| 100 |
+
- `HUB_VERSION` → `1.1.0`.
|
| 101 |
+
|
| 102 |
+
### `app/db_sync.py`, `app/shellmaster.py`
|
| 103 |
+
- Unverändert bzw. nur verdrahtet — beide waren solide gebaut.
|
| 104 |
+
- ShellMaster bleibt **inactive** in `.pyfun` bis Token + `shellmaster_commands.jsonl` + Agent-URL da sind.
|
| 105 |
+
|
| 106 |
+
### `fundaments/` (minimal)
|
| 107 |
+
- `security.py`: D2 + D3 (siehe oben).
|
| 108 |
+
- `access_control.py`: D3-Override.
|
| 109 |
+
- `main.py` (Guardian): D1-Wiring + D6-Log.
|
| 110 |
+
- Sonst **keine** Änderungen — dein Fundament bleibt deins.
|
| 111 |
+
|
| 112 |
+
### Projekt
|
| 113 |
+
- Entfernt: `mcp_old.py`, `example.py`, `app/web/`, `app/tmp/`.
|
| 114 |
+
- `requirements.txt`: `fastmcp` → `mcp>=1.10,<2`.
|
| 115 |
+
- `Dockerfile`: `EXPOSE 7860`.
|
| 116 |
+
- `example-mcp___.env`: aufgeräumt (stale `APP_MODE`/`MCP_TRANSPORT` raus, alle Provider-Keys dokumentiert).
|
| 117 |
+
- Alle Dateien auf LF normalisiert (40 Dateien, CRLF kam aus dem Windows-Checkout).
|
| 118 |
+
|
| 119 |
+
---
|
| 120 |
+
|
| 121 |
+
## 5. Live-Test-Ergebnisse (alle bestanden)
|
| 122 |
+
|
| 123 |
+
```
|
| 124 |
+
✅ Boot ohne ENV-Keys → Server startet, 3 System-Tools, kein Crash
|
| 125 |
+
✅ Boot mit Dummy-Keys → anthropic + brave registrieren sich, 8 Tools
|
| 126 |
+
✅ MCP initialize → JSON-RPC Response mit serverInfo
|
| 127 |
+
✅ MCP tools/list → 8 Tools
|
| 128 |
+
✅ MCP tools/call (pricing) → gemini-2.5-flash: .pyfun-Werte gewinnen
|
| 129 |
+
✅ MCP tools/call (db_query) → SELECT auf hub_state funktioniert
|
| 130 |
+
✅ Pricing-Quellen → OpenRouter 465 + LiteLLM 4615 Modelle live
|
| 131 |
+
✅ Pricing-Lookup → .pyfun-Override ✅ / Auto-Fill ✅ / Unknown ✅
|
| 132 |
+
✅ Cache-TTL → Restart: „cache fresh — no external fetch needed"
|
| 133 |
+
✅ Fallback-Kette → anthropic 403 → Kette durch → RuntimeError, KEIN Key im Log
|
| 134 |
+
✅ /api Fehlerpffade → fehlender Prompt 400, unbekanntes Tool 400,
|
| 135 |
+
Schreib-SQL blockiert, kaputtes JSON 400
|
| 136 |
+
```
|
| 137 |
+
|
| 138 |
+
---
|
| 139 |
+
|
| 140 |
+
## 6. Wie du weitermachst
|
| 141 |
+
|
| 142 |
+
```bash
|
| 143 |
+
# 1. .env anlegen
|
| 144 |
+
cp example-mcp___.env .env # dann Keys eintragen
|
| 145 |
+
|
| 146 |
+
# 2. Starten
|
| 147 |
+
python main.py
|
| 148 |
+
|
| 149 |
+
# 3. Kosten abfragen (macht jetzt der Hub selbst)
|
| 150 |
+
curl -X POST localhost:7860/api -H "Content-Type: application/json" \
|
| 151 |
+
-d '{"tool":"get_model_pricing","params":{"model_name":"gemini-2.5-flash"}}'
|
| 152 |
+
|
| 153 |
+
# 4. Preise erzwingen zu aktualisieren
|
| 154 |
+
curl -X POST localhost:7860/api -H "Content-Type: application/json" \
|
| 155 |
+
-d '{"tool":"refresh_pricing"}'
|
| 156 |
+
```
|
| 157 |
+
|
| 158 |
+
**Du kannst in `.pyfun [MODELS]` jetzt alle `cost_*`-Zeilen löschen** — sie werden automatisch gefüllt. Einträge, die du setzt, gewinnen weiterhin als Override (z. B. wenn du Sonderkonditionen hast).
|
| 159 |
+
|
| 160 |
+
**Nächste sinnvolle Schritte:** echte Keys testen · `persist_result` mit echter PostgreSQL-Tabelle · ShellMaster aktivieren (Token + Registry-Datei + Agent) · Auth-Layer vor `/api` und `/mcp` legen.
|
| 161 |
+
|
| 162 |
+
---
|
| 163 |
+
|
| 164 |
+
## 7. Debug- & Secrets-Audit (nachgetragen am 10.11.2026)
|
| 165 |
+
|
| 166 |
+
**Anlass:** Nutzer-Hinweis auf die Debug-Einstellungen in `.pyfun` und `.env` — die fehlte im ersten Review. Nachgetragen, mit Beweis.
|
| 167 |
+
|
| 168 |
+
### Secrets — wo auch immer gesucht, nichts gefunden
|
| 169 |
+
|
| 170 |
+
| Prüfung | Ergebnis | Beweis |
|
| 171 |
+
|---------|----------|--------|
|
| 172 |
+
| Echte API-Keys im Work-Tree | ✅ keine | `.pyfun` enthält nur ENV-**Namen** (`env_key = "ANTHROPIC_API_KEY"`), `shellmaster_token = ""` |
|
| 173 |
+
| Echte API-Keys in **gesamter Git-History** (~30 Commits) | ✅ keine | Alle Muster-Treffer sind `...`-Platzhalter in Doku-Kommentaren (`sk-ant-...`, `hf_...`, `BSA...`, `tvly-...`) |
|
| 174 |
+
| `.env` je committed | ✅ nein | `git log --all --diff-filter=A` — nur `example-mcp___.env` (Platzhalter) |
|
| 175 |
+
| Logs über HTTP ausgeliefert | ✅ nein | `app.py` hat nur `/`, `/api`, `/crypto` — keine Log-Route |
|
| 176 |
+
| Prompts/Responses/Bodies in Logs | ✅ nein | `grep` über alle `logger.*`-Aufrufe in `app/` — nur Status, Registrierungen, sanitized URLs |
|
| 177 |
+
|
| 178 |
+
### Debug-Schalter — wer liest sie WIRKLICH?
|
| 179 |
+
|
| 180 |
+
| Schalter | Wo gelesen | Default | Tatsächliche Wirkung |
|
| 181 |
+
|----------|-----------|---------|---------------------|
|
| 182 |
+
| `PYFUNDAMENTS_DEBUG` | `.env` → `main.py` + `fundaments/debug.py` | `false` | Startup-Diagnose (Python-Version, `sys.path`, Datei-Checks) |
|
| 183 |
+
| `LOG_LEVEL` | `.env` | `INFO` | Log-Tiefe |
|
| 184 |
+
| `LOG_TO_TMP` | `.env` | `false` | Logs nach `/tmp` (nur Host-lokal, nicht öffentlich) |
|
| 185 |
+
| `ENABLE_PUBLIC_LOGS` | `.env` | **`true`** | `false` = nur CRITICAL. **Achtung HF Spaces:** stdout-Logs sind dort im Log-Tab **öffentlich einsehbar** — für öffentliche Spaces `false` setzen |
|
| 186 |
+
| `.pyfun [DEBUG]`-Sektion | **niemand** | — | **TOTE CONFIG** — kein `app/*`-Code liest sie (Stand v1.1.0, per `grep` über `app/` bewiesen) |
|
| 187 |
+
|
| 188 |
+
### Was daraus bereinigt wurde
|
| 189 |
+
|
| 190 |
+
- **`.pyfun [DEBUG]`**: Stand auf `DEBUG="ON"`, `DEBUG_LEVEL="FULL"`, `LOG_RESPONSES="true"` — wirkungslos heute, aber eine **Landmine**: wäre die Sektion später verdrahtet worden, hätte sie sofort sensitiv geloggt. Jetzt: `OFF`/`false`-Defaults + „not wired yet"-Hinweis im Header.
|
| 191 |
+
- **`example-mcp___.env`**: shippte Dev-Defaults (`PYFUNDAMENTS_DEBUG="true"`, `LOG_LEVEL="DEBUG"`, `ENABLE_PUBLIC_LOGS="true"`). Template jetzt produktions-sicher (`false`/`INFO`/`false`) + Warnhinweis zum öffentlichen HF-Spaces-Log-Tab. Für die Testphase einfach in deiner lokalen `.env` umschalten.
|
| 192 |
+
- **Nicht geändert (Empfehlung):** Der Code-Default `ENABLE_PUBLIC_LOGS=true` in `main.py`/`fundaments/debug.py`. Für öffentliche HF Spaces wäre `false` der sicherere Default — aber das ist dein Fundament, deine Entscheidung. Eindeutig dokumentiert in der `.env`-Vorlage.
|
| 193 |
+
|
| 194 |
+
> **Wichtig fürs Verständnis:** Runtime-Verhalten hat sich durch die Bereinigung **nicht** geändert — die `[DEBUG]`-Sektion war tot, und die `.env`-Vorlage wirkt erst, wenn du sie kopierst. Geändert haben sich nur die Defaults, mit denen das Projekt ausgeliefert wird.
|
| 195 |
+
|
| 196 |
+
---
|
| 197 |
+
|
| 198 |
+
## 8. Update 2 (v1.2.0): Auto-Pricing konsequent, Gedächtnis-Orchestrierung, ShellMaster ROT
|
| 199 |
+
|
| 200 |
+
### 8.1 Die Pricing-Frage: Brauchen wir `[MODELS]`-Kosten noch?
|
| 201 |
+
|
| 202 |
+
**Antwort auf die OpenRouter-Frage (belegt, OpenRouter-FAQ live):**
|
| 203 |
+
> *„We charge a small fee when purchasing credits. We never mark-up the pricing of the underlying providers, and you'll always pay the same as the provider's listed price."*
|
| 204 |
+
|
| 205 |
+
- **Per-Token = Original-Listenpreis.** Kein Markup. Du irrst dich NICHT.
|
| 206 |
+
- **ABER:** ~5,5 % Gebühr beim **Kredit-Kauf** (Stripe, $0,80 Min; Coinbase 5 %) — effektiv also Listenpreis + Kaufgebühr, nicht pro Token.
|
| 207 |
+
- Wer mit eigenem `ANTHROPIC_API_KEY` direkt callt, zahlt ohnehin Vendor-Listenpreis → gleiche Zahlen.
|
| 208 |
+
|
| 209 |
+
**Konsequenz umgesetzt:** 34 `cost_*`-Zeilen aus `[MODELS]` entfernt. Übrig bleiben nur die 4 Zeilen der Custom-Modelle (`smollm2-360m`, `codey-lab/...` = eigenes Deployment, 0.0 ist dort die Wahrheit — keine externe Quelle kennt sie). **Die Sektion selbst bleibt** — `requests_per_min/day`, `context_tokens`, `capabilities` und das Provider-Mapping gibt es in keiner Preisquelle.
|
| 210 |
+
|
| 211 |
+
**Live bewiesen (26/26 Checks):** Alle 17 übrigen Modelle füllen Kosten automatisch:
|
| 212 |
+
`claude-*` → litellm (z. B. opus 0.005/0.025 $/1k) · `gemini-3.*-preview` → openrouter · `mistral-7b` → litellm (**OpenRouter hat es aus dem Katalog genommen** — der Fallback greift genau richtig) · `smollm2-360m` → pyfun/0.0 · unbekannt → unknown.
|
| 213 |
+
|
| 214 |
+
> ⚠️ Beachte: `claude-opus-4-6` zeigt jetzt **0.005/0.025** (LiteLLM) statt deiner alten 0.015/0.075. Prüfe das gegen deine echte Rechnung — falls du andere Konditionen hast, einfach die `cost_*`-Zeilen für DIESES Modell wieder eintragen (Override gewinnt immer).
|
| 215 |
+
|
| 216 |
+
**Kleiner Fix nebenbei:** `.pyfun`-Kosten kamen als String (`"0.00000"`), Auto-Werte als Float — `get_pricing()` normalisiert jetzt alles auf Float. Und `/api list_active_tools` liefert jetzt auch `pricing_stats` (vorher nur die MCP-Version).
|
| 217 |
+
|
| 218 |
+
### 8.2 Gedächtnis-Orchestrierung — dein Architektur-Wunsch, umgesetzt
|
| 219 |
+
|
| 220 |
+
**Dein Design:** SQLite = Kurzzeit-Gedächtnis (alle KI-Tools lesen/schreiben) · Wichtiges → PostgreSQL = Dauer-Gedächtnis · Auswahl was wichtig ist = **dein** Bullshit-Filter (deine Sache).
|
| 221 |
+
|
| 222 |
+
```
|
| 223 |
+
persist_result (MCP + /api)
|
| 224 |
+
↓ enqueue_persist() ← sofort, IMMER, kein Crash
|
| 225 |
+
persist_queue (SQLite) ← überlebt Restarts (BEWIESEN: 3 vor, 3 nach Reboot)
|
| 226 |
+
↓ flush_persist_queue() ← Background-Flusher, Retry alle 60 s (config)
|
| 227 |
+
PostgreSQL hub_results ← CREATE TABLE IF NOT EXISTS + INSERT
|
| 228 |
+
```
|
| 229 |
+
|
| 230 |
+
| Eigenschaft | Umsetzung |
|
| 231 |
+
|---|---|
|
| 232 |
+
| Ohne `DATABASE_URL` | `status: "queued"` statt RuntimeError/502 — **kein Crash mehr** (vorher Befund!) |
|
| 233 |
+
| PG temporär weg | Eintrag bleibt, `attempts`+1, sanitisierter `last_error` (nur Exception-Typ) |
|
| 234 |
+
| PG wieder da | Flusher (5 s nach Boot, dann Intervall) holt alles nach |
|
| 235 |
+
| Schema | `hub_results(id, created_at, source, model, score, payload JSONB)` |
|
| 236 |
+
| **Score-Slot** | `score REAL` — Spalte + Parameter durchgereicht bis PostgreSQL, **uninterpretiert**. Dein Filter schreibt später einfach mit `score` mit oder filtert beim Lesen. Kein Hub-Code rührt ihn an. |
|
| 237 |
+
| Observability | `persist_status` (MCP + /api): queue_depth, ältester Eintrag, Fehlversuche, Bridge-Status |
|
| 238 |
+
| Grenzen | Queue max. 500 (älteste fliegen raus, wie `tool_cache`) · at-least-once (Duplikat statt Verlust) · HF Spaces: `/tmp`-SQLite überlebt Container-Reset nicht — dort ist PG das einzige Dauer-Gedächtnis |
|
| 239 |
+
|
| 240 |
+
**Live bewiesen:** queued ohne PG ✅ · score 0.85 + source + model landen korrekt ✅ · `db_query` liest die Queue ✅ · MCP registriert `persist_result` + `persist_status` (8 Tools) ✅ · **Restart: 3 Einträge überleben** ✅
|
| 241 |
+
|
| 242 |
+
### 8.3 ShellMaster = dein CLI-Commander — SECURITY LEVEL ROT
|
| 243 |
+
|
| 244 |
+
Bestätigt, was du erkannt hast: Er ist für den **lokalen Quart-Agenten** gebaut (`localhost:5004`, Bearer-Token, Befehle auf DEINEM Rechner). Schutzmuster im Code: Hardblock-Liste (rm -rf, mkfs, …) + JSONL-Registry + **explizite Bestätigung** vor jeder Ausführung + Recovery-Plan im Gedächtnis.
|
| 245 |
+
|
| 246 |
+
Trotzdem gilt: **Stufe rot, nur lokal.** Dokumentiert in `.pyfun` (Kommentar-Block über `[TOOL.shellmaster]`). Zusätzlich entschärft: Token-Field in `.pyfun` war eine Einladung, ein Secret in die Config zu legen — jetzt gibt es `shellmaster_env_key = "SHELLMASTER_TOKEN"` (→ `.env`, exakt wie bei allen Providern), das `.pyfun`-Feld ist nur noch lokaler Test-Fallback. `active` bleibt `"false"`.
|
| 247 |
+
|
| 248 |
+
---
|
| 249 |
+
|
| 250 |
+
*Rebuild nach PyFundaments-Architektur: Guardian → Orchestrator → sandboxes app/* · No key = no tool = no crash.*
|
app/.pyfun
ADDED
|
@@ -0,0 +1,561 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# .pyfun — PyFundaments App Configuration
|
| 3 |
+
# Single source of truth for app/* modules (provider, models, tools, hub)
|
| 4 |
+
# Part of: Universal MCP Hub on PyFundaments
|
| 5 |
+
# =============================================================================
|
| 6 |
+
# RULES:
|
| 7 |
+
# - All values in double quotes "value"
|
| 8 |
+
# - NO secrets here! Keys stay in .env → only ENV-VAR NAMES referenced here
|
| 9 |
+
# - Comment-out unused sections with # → keep structure, parsers need it!
|
| 10 |
+
# - DO NOT DELETE headers or [X_END] → parsers rely on these markers
|
| 11 |
+
# - Empty/unused values: "" → never leave bare =
|
| 12 |
+
# =============================================================================
|
| 13 |
+
# TIERS:
|
| 14 |
+
# LAZY: fill [HUB] + one [LLM_PROVIDER.*] only → works
|
| 15 |
+
# NORMAL: + [SEARCH_PROVIDER.*] + [MODELS.*] → works better
|
| 16 |
+
# PRODUCTIVE: + [TOOLS] + [FALLBACK] + [HUB_LIMITS] → full power
|
| 17 |
+
# =============================================================================
|
| 18 |
+
# DO NOT DELETE — file identifier used by all parsers
|
| 19 |
+
[PYFUN_FILE = .pyfun]
|
| 20 |
+
|
| 21 |
+
# =============================================================================
|
| 22 |
+
# HUB — Core identity & transport config
|
| 23 |
+
# =============================================================================
|
| 24 |
+
[HUB]
|
| 25 |
+
HUB_NAME = "Multi-LLM-API-Gateway"
|
| 26 |
+
HUB_VERSION = "1.2.0"
|
| 27 |
+
HUB_DESCRIPTION = "Universal Hub built on PyFundaments"
|
| 28 |
+
|
| 29 |
+
# Transport: sse (legacy/HF Spaces fallback) | streamable-http (MCP spec 2025-11-25)
|
| 30 |
+
HUB_TRANSPORT = "streamable-http"
|
| 31 |
+
|
| 32 |
+
# Stateless: true = HF Spaces + horizontal scaling safe
|
| 33 |
+
# false = nur wenn du Server-initiated Notifications brauchst
|
| 34 |
+
HUB_STATELESS = "true"
|
| 35 |
+
|
| 36 |
+
HUB_HOST = "0.0.0.0"
|
| 37 |
+
HUB_PORT = "7860"
|
| 38 |
+
HUB_MODE = "mcp"
|
| 39 |
+
HUB_SPACE_URL = ""
|
| 40 |
+
[HUB_END]
|
| 41 |
+
|
| 42 |
+
# =============================================================================
|
| 43 |
+
# HUB_LIMITS — Request & retry behavior
|
| 44 |
+
# =============================================================================
|
| 45 |
+
[HUB_LIMITS]
|
| 46 |
+
MAX_PARALLEL_REQUESTS = "5"
|
| 47 |
+
RETRY_COUNT = "3"
|
| 48 |
+
RETRY_DELAY_SEC = "2"
|
| 49 |
+
REQUEST_TIMEOUT_SEC = "60"
|
| 50 |
+
SEARCH_TIMEOUT_SEC = "30"
|
| 51 |
+
[HUB_LIMITS_END]
|
| 52 |
+
|
| 53 |
+
# =============================================================================
|
| 54 |
+
# PROVIDERS — All external API providers
|
| 55 |
+
# Secrets stay in .env! Only ENV-VAR NAMES are referenced here.
|
| 56 |
+
# =============================================================================
|
| 57 |
+
[PROVIDERS]
|
| 58 |
+
|
| 59 |
+
# ── LLM Providers ─────────────────────────────────────────────────────────────
|
| 60 |
+
[LLM_PROVIDERS]
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
# customs LLM
|
| 64 |
+
[LLM_PROVIDER.smollm]
|
| 65 |
+
active = "true"
|
| 66 |
+
base_url = "https://codey-lab-smollm2-customs.hf.space/v1"
|
| 67 |
+
env_key = "SMOLLM_API_KEY"
|
| 68 |
+
default_model = "smollm2-360m"
|
| 69 |
+
models = "smollm2-360m, codey-lab/model.universal-mcp-hub"
|
| 70 |
+
fallback_to = "gemini"
|
| 71 |
+
[LLM_PROVIDER.smollm_END]
|
| 72 |
+
|
| 73 |
+
# AI Wrapper
|
| 74 |
+
|
| 75 |
+
[LLM_PROVIDER.anthropic]
|
| 76 |
+
active = "true"
|
| 77 |
+
base_url = "https://api.anthropic.com/v1"
|
| 78 |
+
env_key = "ANTHROPIC_API_KEY"
|
| 79 |
+
api_version_header = "2023-06-01"
|
| 80 |
+
default_model = "claude-haiku-4-5-20251001"
|
| 81 |
+
models = "claude-opus-4-6, claude-sonnet-4-6, claude-haiku-4-5-20251001"
|
| 82 |
+
supports_cache = "true"
|
| 83 |
+
fallback_to = "gemini"
|
| 84 |
+
[LLM_PROVIDER.anthropic_END]
|
| 85 |
+
|
| 86 |
+
[LLM_PROVIDER.gemini]
|
| 87 |
+
active = "true"
|
| 88 |
+
base_url = "https://generativelanguage.googleapis.com/v1beta"
|
| 89 |
+
env_key = "GEMINI_API_KEY"
|
| 90 |
+
default_model = "gemini-2.5-flash"
|
| 91 |
+
models = "gemini-2.0-flash, gemini-2.5-flash, gemini-2.5-flash-lite, gemini-3.1-flash-lite-preview, gemini-3-flash-preview, gemini-3.1-pro-preview"
|
| 92 |
+
supports_cache = "true"
|
| 93 |
+
fallback_to = "openrouter"
|
| 94 |
+
[LLM_PROVIDER.gemini_END]
|
| 95 |
+
|
| 96 |
+
[LLM_PROVIDER.openrouter]
|
| 97 |
+
active = "true"
|
| 98 |
+
base_url = "https://openrouter.ai/api/v1"
|
| 99 |
+
env_key = "OPENROUTER_API_KEY" # → .env: OPENROUTER_API_KEY=sk-or-...
|
| 100 |
+
default_model = "nvidia/nemotron-nano-9b-v2:free"
|
| 101 |
+
models = "meta-llama/llama-3-8b-instruct, mistralai/mistral-7b-instruct, nvidia/nemotron-nano-9b-v2:free, google/gemma-3-27b-it:free, openai/gpt-oss-20b:free, mistralai/Mistral-7B-Instruct-v0.3, meta-llama/Llama-3.3-70B-Instruct"
|
| 102 |
+
fallback_to = "huggingface"
|
| 103 |
+
[LLM_PROVIDER.openrouter_END]
|
| 104 |
+
|
| 105 |
+
[LLM_PROVIDER.huggingface]
|
| 106 |
+
active = "true"
|
| 107 |
+
base_url = "https://api-inference.huggingface.co/v1"
|
| 108 |
+
env_key = "HF_TOKEN" # → .env: HF_TOKEN=hf_...
|
| 109 |
+
default_model = "Qwen/Qwen2.5-Coder-32B-Instruct"
|
| 110 |
+
models = "Qwen/Qwen2.5-Coder-32B-Instruct, meta-llama/Llama-3.1-8B-Instruct, mistralai/Mistral-Nemo-Instruct-2407, Qwen/Qwen2.5-72B-Instruct"
|
| 111 |
+
|
| 112 |
+
fallback_to = "smollm" # ← last in chain, no further fallback
|
| 113 |
+
[LLM_PROVIDER.huggingface_END]
|
| 114 |
+
|
| 115 |
+
# ── Add more LLM providers below ──────────────────────────────────────────
|
| 116 |
+
# [LLM_PROVIDER.mistral]
|
| 117 |
+
# active = "false"
|
| 118 |
+
# base_url = "https://api.mistral.ai/v1"
|
| 119 |
+
# env_key = "MISTRAL_API_KEY"
|
| 120 |
+
# default_model = "mistral-large-latest"
|
| 121 |
+
# models = "mistral-large-latest, mistral-small-latest"
|
| 122 |
+
# fallback_to = ""
|
| 123 |
+
# [LLM_PROVIDER.mistral_END]
|
| 124 |
+
|
| 125 |
+
# [LLM_PROVIDER.openai]
|
| 126 |
+
# active = "false"
|
| 127 |
+
# base_url = "https://api.openai.com/v1"
|
| 128 |
+
# env_key = "OPENAI_API_KEY"
|
| 129 |
+
# default_model = "gpt-4o"
|
| 130 |
+
# models = "gpt-4o, gpt-4o-mini, gpt-3.5-turbo"
|
| 131 |
+
# fallback_to = ""
|
| 132 |
+
# [LLM_PROVIDER.openai_END]
|
| 133 |
+
|
| 134 |
+
[LLM_PROVIDERS_END]
|
| 135 |
+
|
| 136 |
+
# ── Search Providers ───────────────────────────────────────────────────────────
|
| 137 |
+
[SEARCH_PROVIDERS]
|
| 138 |
+
|
| 139 |
+
[SEARCH_PROVIDER.brave]
|
| 140 |
+
active = "true"
|
| 141 |
+
base_url = "https://api.search.brave.com/res/v1/web/search"
|
| 142 |
+
env_key = "BRAVE_API_KEY" # → .env: BRAVE_API_KEY=BSA...
|
| 143 |
+
default_results = "5"
|
| 144 |
+
max_results = "20"
|
| 145 |
+
fallback_to = "tavily"
|
| 146 |
+
[SEARCH_PROVIDER.brave_END]
|
| 147 |
+
|
| 148 |
+
[SEARCH_PROVIDER.tavily]
|
| 149 |
+
active = "true"
|
| 150 |
+
base_url = "https://api.tavily.com/search"
|
| 151 |
+
env_key = "TAVILY_API_KEY" # → .env: TAVILY_API_KEY=tvly-...
|
| 152 |
+
default_results = "5"
|
| 153 |
+
max_results = "10"
|
| 154 |
+
include_answer = "true" # AI-synthesized answer
|
| 155 |
+
fallback_to = ""
|
| 156 |
+
[SEARCH_PROVIDER.tavily_END]
|
| 157 |
+
|
| 158 |
+
# ── Add more search providers below ───────────────────────────────────────
|
| 159 |
+
# [SEARCH_PROVIDER.serper]
|
| 160 |
+
# active = "false"
|
| 161 |
+
# base_url = "https://google.serper.dev/search"
|
| 162 |
+
# env_key = "SERPER_API_KEY"
|
| 163 |
+
# fallback_to = ""
|
| 164 |
+
# [SEARCH_PROVIDER.serper_END]
|
| 165 |
+
|
| 166 |
+
[SEARCH_PROVIDERS_END]
|
| 167 |
+
|
| 168 |
+
# ── Web / Action Providers (Webhooks, Bots, Social) ───────────────────────────
|
| 169 |
+
# [WEB_PROVIDERS]
|
| 170 |
+
|
| 171 |
+
# [WEB_PROVIDER.discord]
|
| 172 |
+
# active = "false"
|
| 173 |
+
# base_url = "https://discord.com/api/v10"
|
| 174 |
+
# env_key = "BOT_TOKEN"
|
| 175 |
+
# [WEB_PROVIDER.discord_END]
|
| 176 |
+
|
| 177 |
+
# [WEB_PROVIDER.github]
|
| 178 |
+
# active = "false"
|
| 179 |
+
# base_url = "https://api.github.com"
|
| 180 |
+
# env_key = "GITHUB_TOKEN"
|
| 181 |
+
# [WEB_PROVIDER.github_END]
|
| 182 |
+
|
| 183 |
+
# [WEB_PROVIDERS_END]
|
| 184 |
+
|
| 185 |
+
[PROVIDERS_END]
|
| 186 |
+
|
| 187 |
+
# =============================================================================
|
| 188 |
+
# MODELS — Token & rate limits per model
|
| 189 |
+
# Parser builds: MODELS[provider][model_name] → limits dict
|
| 190 |
+
# COSTS: cost_*-Zeilen sind seit v1.1 OPTIONAL — [PRICING] füllt sie automatisch
|
| 191 |
+
# aus OpenRouter + LiteLLM nach (OR markiert nie auf: per-Token = Vendor-
|
| 192 |
+
# Listenpreis; ~5.5% Gebühr fällt nur beim Kredit-Kauf an, nicht pro Token).
|
| 193 |
+
# Deine manuellen cost_* gewinnen IMMER als Override — sinnvoll für eigene
|
| 194 |
+
# Deployments (smollm = 0.0, weil eigener Server) und Sonderkonditionen.
|
| 195 |
+
# =============================================================================
|
| 196 |
+
[MODELS]
|
| 197 |
+
|
| 198 |
+
[MODEL.claude-opus-4-6]
|
| 199 |
+
provider = "anthropic"
|
| 200 |
+
context_tokens = "200000"
|
| 201 |
+
max_output_tokens = "32000"
|
| 202 |
+
requests_per_min = "5"
|
| 203 |
+
requests_per_day = "300"
|
| 204 |
+
capabilities = "text, code, analysis, vision"
|
| 205 |
+
[MODEL.claude-opus-4-6_END]
|
| 206 |
+
|
| 207 |
+
[MODEL.claude-sonnet-4-6]
|
| 208 |
+
provider = "anthropic"
|
| 209 |
+
context_tokens = "200000"
|
| 210 |
+
max_output_tokens = "16000"
|
| 211 |
+
requests_per_min = "50"
|
| 212 |
+
requests_per_day = "1000"
|
| 213 |
+
capabilities = "text, code, analysis, vision"
|
| 214 |
+
[MODEL.claude-sonnet-4-6_END]
|
| 215 |
+
|
| 216 |
+
[MODEL.claude-haiku-4-5-20251001]
|
| 217 |
+
provider = "anthropic"
|
| 218 |
+
context_tokens = "200000"
|
| 219 |
+
max_output_tokens = "8000"
|
| 220 |
+
requests_per_min = "50"
|
| 221 |
+
requests_per_day = "2000"
|
| 222 |
+
capabilities = "text, code, fast"
|
| 223 |
+
[MODEL.claude-haiku-4-5-20251001_END]
|
| 224 |
+
|
| 225 |
+
[MODEL.gemini-3.1-flash-lite-preview]
|
| 226 |
+
provider = "gemini"
|
| 227 |
+
context_tokens = "1000000"
|
| 228 |
+
max_output_tokens = "8192"
|
| 229 |
+
requests_per_min = "15"
|
| 230 |
+
requests_per_day = "1000"
|
| 231 |
+
capabilities = "text, code, vision, fast, cheap"
|
| 232 |
+
[MODEL.gemini-3.1-flash-lite-preview_END]
|
| 233 |
+
|
| 234 |
+
[MODEL.gemini-3-flash-preview]
|
| 235 |
+
provider = "gemini"
|
| 236 |
+
context_tokens = "1000000"
|
| 237 |
+
max_output_tokens = "8192"
|
| 238 |
+
requests_per_min = "10"
|
| 239 |
+
requests_per_day = "1000"
|
| 240 |
+
capabilities = "text, code, vision, audio, frontier"
|
| 241 |
+
[MODEL.gemini-3-flash-preview_END]
|
| 242 |
+
|
| 243 |
+
[MODEL.gemini-3.1-pro-preview]
|
| 244 |
+
provider = "gemini"
|
| 245 |
+
context_tokens = "1000000"
|
| 246 |
+
max_output_tokens = "64000"
|
| 247 |
+
requests_per_min = "5"
|
| 248 |
+
requests_per_day = "1000"
|
| 249 |
+
capabilities = "text, code, vision, audio, reasoning, agents"
|
| 250 |
+
[MODEL.gemini-3.1-pro-preview_END]
|
| 251 |
+
|
| 252 |
+
[MODEL.gemini-2.5-flash]
|
| 253 |
+
provider = "gemini"
|
| 254 |
+
context_tokens = "1000000"
|
| 255 |
+
max_output_tokens = "8192"
|
| 256 |
+
requests_per_min = "15"
|
| 257 |
+
requests_per_day = "1500"
|
| 258 |
+
capabilities = "text, code, vision, audio"
|
| 259 |
+
[MODEL.gemini-2.5-flash_END]
|
| 260 |
+
|
| 261 |
+
[MODEL.gemini-2.5-flash-lite]
|
| 262 |
+
provider = "gemini"
|
| 263 |
+
context_tokens = "1000000"
|
| 264 |
+
max_output_tokens = "8192"
|
| 265 |
+
requests_per_min = "15"
|
| 266 |
+
requests_per_day = "1500"
|
| 267 |
+
capabilities = "text, code, vision, audio"
|
| 268 |
+
[MODEL.gemini-2.5-flash-lite_END]
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
[MODEL.gemini-2.0-flash]
|
| 272 |
+
provider = "gemini"
|
| 273 |
+
context_tokens = "1000000"
|
| 274 |
+
max_output_tokens = "8192"
|
| 275 |
+
requests_per_min = "15"
|
| 276 |
+
requests_per_day = "1500"
|
| 277 |
+
capabilities = "text, code, vision, audio"
|
| 278 |
+
[MODEL.gemini-2.0-flash_END]
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
[MODEL.mistral-7b-instruct]
|
| 282 |
+
provider = "openrouter"
|
| 283 |
+
context_tokens = "32000"
|
| 284 |
+
max_output_tokens = "4096"
|
| 285 |
+
requests_per_min = "60"
|
| 286 |
+
requests_per_day = "10000"
|
| 287 |
+
capabilities = "text, code, fast, cheap"
|
| 288 |
+
[MODEL.mistral-7b-instruct_END]
|
| 289 |
+
|
| 290 |
+
[MODEL.dolphin-mistral-24b-venice-edition]
|
| 291 |
+
provider = "openrouter"
|
| 292 |
+
context_tokens = "32768"
|
| 293 |
+
max_output_tokens = "4096"
|
| 294 |
+
requests_per_min = ""
|
| 295 |
+
requests_per_day = ""
|
| 296 |
+
capabilities = "uncensored, text, code, fast, very cheap"
|
| 297 |
+
[MODEL.dolphin-mistral-24b-venice-edition_END]
|
| 298 |
+
|
| 299 |
+
[MODEL.nvidia-nemotron-nano-9b-v2]
|
| 300 |
+
provider = "openrouter"
|
| 301 |
+
context_tokens = "131072"
|
| 302 |
+
max_output_tokens = "4096"
|
| 303 |
+
requests_per_min = ""
|
| 304 |
+
requests_per_day = ""
|
| 305 |
+
capabilities = "text, code, reasoning, fast, free"
|
| 306 |
+
[MODEL.nvidia-nemotron-nano-9b-v2_END]
|
| 307 |
+
|
| 308 |
+
[MODEL.gemma-3-27b-it]
|
| 309 |
+
provider = "openrouter"
|
| 310 |
+
context_tokens = "131072"
|
| 311 |
+
max_output_tokens = "8192"
|
| 312 |
+
requests_per_min = ""
|
| 313 |
+
requests_per_day = ""
|
| 314 |
+
capabilities = "text, code, vision, multilingual, free"
|
| 315 |
+
[MODEL.gemma-3-27b-it_END]
|
| 316 |
+
|
| 317 |
+
[MODEL.gpt-oss-20b]
|
| 318 |
+
provider = "openrouter"
|
| 319 |
+
context_tokens = "128000"
|
| 320 |
+
max_output_tokens = "4096"
|
| 321 |
+
requests_per_min = ""
|
| 322 |
+
requests_per_day = ""
|
| 323 |
+
capabilities = "text, code, free"
|
| 324 |
+
[MODEL.gpt-oss-20b_END]
|
| 325 |
+
|
| 326 |
+
[MODEL.deephermes-3-llama-3-8b-preview]
|
| 327 |
+
provider = "openrouter"
|
| 328 |
+
context_tokens = "131072"
|
| 329 |
+
max_output_tokens = "4096"
|
| 330 |
+
requests_per_min = ""
|
| 331 |
+
requests_per_day = ""
|
| 332 |
+
capabilities = "text, code, reasoning, uncensored, free"
|
| 333 |
+
[MODEL.deephermes-3-llama-3-8b-preview_END]
|
| 334 |
+
|
| 335 |
+
[MODEL.Mistral-7B-Instruct-v0.3]
|
| 336 |
+
provider = "openrouter"
|
| 337 |
+
context_tokens = "32768"
|
| 338 |
+
max_output_tokens = "4096"
|
| 339 |
+
requests_per_min = "60"
|
| 340 |
+
requests_per_day = "10000"
|
| 341 |
+
capabilities = "text, code, fast, cheap"
|
| 342 |
+
[MODEL.Mistral-7B-Instruct-v0.3_END]
|
| 343 |
+
|
| 344 |
+
[MODEL.Llama-3.3-70B-Instruct]
|
| 345 |
+
provider = "openrouter"
|
| 346 |
+
context_tokens = "131072"
|
| 347 |
+
max_output_tokens = "8192"
|
| 348 |
+
requests_per_min = "60"
|
| 349 |
+
requests_per_day = "10000"
|
| 350 |
+
capabilities = "text, code, reasoning, multilingual"
|
| 351 |
+
[MODEL.Llama-3.3-70B-Instruct_END]
|
| 352 |
+
|
| 353 |
+
# customs llms
|
| 354 |
+
[MODEL.smollm2-360m]
|
| 355 |
+
provider = "smollm"
|
| 356 |
+
context_tokens = "8192"
|
| 357 |
+
max_output_tokens = "300"
|
| 358 |
+
requests_per_min = "60"
|
| 359 |
+
requests_per_day = "10000"
|
| 360 |
+
cost_input_per_1k = "0.00000"
|
| 361 |
+
cost_output_per_1k = "0.00000"
|
| 362 |
+
capabilities = "text, assistant, navigation, free, local"
|
| 363 |
+
[MODEL.smollm2-360m_END]
|
| 364 |
+
|
| 365 |
+
[MODEL.codey-lab/model.universal-mcp-hub]
|
| 366 |
+
provider = "smollm"
|
| 367 |
+
context_tokens = "8192"
|
| 368 |
+
max_output_tokens = "300"
|
| 369 |
+
requests_per_min = "60"
|
| 370 |
+
requests_per_day = "10000"
|
| 371 |
+
cost_input_per_1k = "0.00000"
|
| 372 |
+
cost_output_per_1k = "0.00000"
|
| 373 |
+
capabilities = "text, assistant, navigation, free, custom, finetuned"
|
| 374 |
+
[MODEL.codey-lab/model.universal-mcp-hub_END]
|
| 375 |
+
|
| 376 |
+
[MODELS_END]
|
| 377 |
+
|
| 378 |
+
# =============================================================================
|
| 379 |
+
# TOOLS — Tool definitions + provider mapping
|
| 380 |
+
# Tools are registered in mcp.py only if their provider ENV key exists!
|
| 381 |
+
# =============================================================================
|
| 382 |
+
[TOOLS]
|
| 383 |
+
|
| 384 |
+
[TOOL.llm_complete]
|
| 385 |
+
active = "true"
|
| 386 |
+
description = "Send prompt to any configured LLM provider"
|
| 387 |
+
provider_type = "llm"
|
| 388 |
+
default_provider = "anthropic"
|
| 389 |
+
timeout_sec = "60"
|
| 390 |
+
system_prompt = "You are a helpful assistant integrated into the Universal MCP Hub. Answer concisely and only what is asked."
|
| 391 |
+
[TOOL.llm_complete_END]
|
| 392 |
+
|
| 393 |
+
[TOOL.code_review]
|
| 394 |
+
active = "true"
|
| 395 |
+
description = "Review code for bugs, security issues and improvements"
|
| 396 |
+
provider_type = "llm"
|
| 397 |
+
default_provider = "anthropic"
|
| 398 |
+
timeout_sec = "60"
|
| 399 |
+
system_prompt = "You are an expert code reviewer. Analyze the given code for bugs, security issues, and improvements. Be specific and concise."
|
| 400 |
+
[TOOL.code_review_END]
|
| 401 |
+
|
| 402 |
+
[TOOL.summarize]
|
| 403 |
+
active = "true"
|
| 404 |
+
description = "Summarize text into concise bullet points"
|
| 405 |
+
provider_type = "llm"
|
| 406 |
+
default_provider = "gemini"
|
| 407 |
+
timeout_sec = "30"
|
| 408 |
+
system_prompt = "You are a summarization expert. Summarize the given text in 3-5 bullet points. Be concise and accurate."
|
| 409 |
+
[TOOL.summarize_END]
|
| 410 |
+
|
| 411 |
+
[TOOL.translate]
|
| 412 |
+
active = "true"
|
| 413 |
+
description = "Translate text — auto-detects source language"
|
| 414 |
+
provider_type = "llm"
|
| 415 |
+
default_provider = "gemini"
|
| 416 |
+
timeout_sec = "30"
|
| 417 |
+
system_prompt = "You are a professional translator. Translate the given text accurately. Auto-detect the source language. Output only the translation, no explanation."
|
| 418 |
+
[TOOL.translate_END]
|
| 419 |
+
|
| 420 |
+
[TOOL.web_search]
|
| 421 |
+
active = "true"
|
| 422 |
+
description = "Search the web via configured search provider"
|
| 423 |
+
provider_type = "search"
|
| 424 |
+
default_provider = "brave"
|
| 425 |
+
timeout_sec = "30"
|
| 426 |
+
[TOOL.web_search_END]
|
| 427 |
+
|
| 428 |
+
[TOOL.db_query]
|
| 429 |
+
active = "true"
|
| 430 |
+
description = "Execute SELECT queries on connected database (read-only)"
|
| 431 |
+
provider_type = "db"
|
| 432 |
+
readonly = "true"
|
| 433 |
+
timeout_sec = "10"
|
| 434 |
+
[TOOL.db_query_END]
|
| 435 |
+
|
| 436 |
+
[TOOL.persist_result]
|
| 437 |
+
active = "true"
|
| 438 |
+
description = "Persist important results to long-term memory (PostgreSQL) — restart-safe queue in SQLite, flushes automatically"
|
| 439 |
+
provider_type = "persist"
|
| 440 |
+
state_read_key = "" # welcher hub_state Key → leer = prompt direkt
|
| 441 |
+
target_table = "hub_results"
|
| 442 |
+
timeout_sec = "10"
|
| 443 |
+
# score-Parameter (optional, float) = reservierter Slot fuer den
|
| 444 |
+
# Bullshit-Filter — wird unverarbeitet durchgereicht und landet als
|
| 445 |
+
# eigene Spalte in PostgreSQL. Filter/Scoring baust DU, nicht der Hub.
|
| 446 |
+
[TOOL.persist_result_END]
|
| 447 |
+
|
| 448 |
+
# ── Future tools ──────────────────────────────────────────────────────────
|
| 449 |
+
# [TOOL.image_gen]
|
| 450 |
+
# active = "false"
|
| 451 |
+
# description = "Generate images via configured provider"
|
| 452 |
+
# provider_type = "image"
|
| 453 |
+
# default_provider = ""
|
| 454 |
+
# timeout_sec = "120"
|
| 455 |
+
# [TOOL.image_gen_END]
|
| 456 |
+
|
| 457 |
+
# [TOOL.code_exec]
|
| 458 |
+
# active = "false"
|
| 459 |
+
# description = "Execute sandboxed code snippets"
|
| 460 |
+
# provider_type = "sandbox"
|
| 461 |
+
# timeout_sec = "30"
|
| 462 |
+
# [TOOL.code_exec_END]
|
| 463 |
+
|
| 464 |
+
# ── Shellmaster 2.0 ──────────────────────────────────────────────────────────
|
| 465 |
+
# SECURITY LEVEL: ROT — dein CLI-Commander. LLM generiert → Hardblock+
|
| 466 |
+
# Registry-Check → DEINE Bestätigung → Ausführung auf DEINEM lokalen
|
| 467 |
+
# Rechner (Agent :5004). NUR lokal — NIEMALS auf öffentlichen HF Spaces
|
| 468 |
+
# aktivieren! Token gehört in .env (shellmaster_env_key), nicht hierher.
|
| 469 |
+
[TOOL.shellmaster]
|
| 470 |
+
active = "false"
|
| 471 |
+
shellmaster_agent_url = "http://localhost:5004"
|
| 472 |
+
shellmaster_env_key = "SHELLMASTER_TOKEN" # → .env, wie bei allen Providern
|
| 473 |
+
shellmaster_token = "" # Fallback nur für lokale Tests
|
| 474 |
+
description = "AI-powered shell assistant — generates and executes safe shell commands"
|
| 475 |
+
provider_type = "llm"
|
| 476 |
+
default_provider = "smollm"
|
| 477 |
+
timeout_sec = "30"
|
| 478 |
+
shellmaster_commands_file = "shellmaster_commands.jsonl"
|
| 479 |
+
shellmaster_commands_dataset = ""
|
| 480 |
+
shellmaster_customs_model = ""
|
| 481 |
+
system_prompt = "You are ShellMaster. Generate safe shell commands for the requested task. Output ONLY valid JSON: {\"command\": \"cmd here\", \"backup\": \"backup cmd or empty\", \"recovery\": \"recovery cmd or empty\", \"risk\": \"low|medium|high\"}"
|
| 482 |
+
[TOOL.shellmaster_END]
|
| 483 |
+
|
| 484 |
+
[TOOL.shellmaster_confirm]
|
| 485 |
+
active = "false"
|
| 486 |
+
description = "Confirm and execute the last ShellMaster command — runs backup first"
|
| 487 |
+
[TOOL.shellmaster_confirm_END]
|
| 488 |
+
|
| 489 |
+
[TOOL.shellmaster_recover]
|
| 490 |
+
active = "false"
|
| 491 |
+
description = "Execute recovery plan for last ShellMaster command"
|
| 492 |
+
[TOOL.shellmaster_recover_END]
|
| 493 |
+
|
| 494 |
+
[TOOL.shellmaster_commands]
|
| 495 |
+
active = "false"
|
| 496 |
+
description = "List allowed ShellMaster commands from registry"
|
| 497 |
+
[TOOL.shellmaster_commands_END]
|
| 498 |
+
|
| 499 |
+
[TOOLS_END]
|
| 500 |
+
|
| 501 |
+
# =============================================================================
|
| 502 |
+
# DB_SYNC — Internal SQLite config for app/* IPC
|
| 503 |
+
# This is NOT the cloud DB — that lives in .env → DATABASE_URL
|
| 504 |
+
# =============================================================================
|
| 505 |
+
[DB_SYNC]
|
| 506 |
+
SQLITE_PATH = "app/.hub_state.db" # internal state, never commit!
|
| 507 |
+
SYNC_INTERVAL_SEC = "30" # how often to flush to SQLite
|
| 508 |
+
MAX_CACHE_ENTRIES = "1000"
|
| 509 |
+
PERSIST_QUEUE_MAX = "500" # restart-safe queue: SQLite → PostgreSQL
|
| 510 |
+
PERSIST_FLUSH_INTERVAL_SEC = "60" # retry interval for the PG flusher
|
| 511 |
+
[DB_SYNC_END]
|
| 512 |
+
|
| 513 |
+
# =============================================================================
|
| 514 |
+
# PRICING — Auto model costs (OpenRouter + LiteLLM)
|
| 515 |
+
# WHY: not every provider has a pricing API (Anthropic/Google: none).
|
| 516 |
+
# These two public sources fill cost_input_per_1k / cost_output_per_1k
|
| 517 |
+
# automatically. Your [MODELS] values ALWAYS win as override.
|
| 518 |
+
# Cache lives in db_sync hub_state, TTL-gated (default 24h).
|
| 519 |
+
# =============================================================================
|
| 520 |
+
[PRICING]
|
| 521 |
+
PRICING_ENABLED = "true"
|
| 522 |
+
PRICING_TTL_HOURS = "24"
|
| 523 |
+
OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models"
|
| 524 |
+
LITELLM_PRICES_URL = "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"
|
| 525 |
+
[PRICING_END]
|
| 526 |
+
|
| 527 |
+
# =============================================================================
|
| 528 |
+
# DEBUG — app/* debug behavior (fundaments debug stays in .env)
|
| 529 |
+
# NOTE (v1.1.0): This section is NOT YET WIRED — no app/* code reads it.
|
| 530 |
+
# Real debug switches live in .env: PYFUNDAMENTS_DEBUG / LOG_LEVEL /
|
| 531 |
+
# LOG_TO_TMP / ENABLE_PUBLIC_LOGS (see example-mcp___.env).
|
| 532 |
+
# Defaults below are SAFE (OFF) so wiring this later cannot leak data.
|
| 533 |
+
# =============================================================================
|
| 534 |
+
[DEBUG]
|
| 535 |
+
DEBUG = "OFF" # ON | OFF (not wired yet)
|
| 536 |
+
DEBUG_LEVEL = "WARN" # FULL | WARN | ERROR (not wired yet)
|
| 537 |
+
LOG_FILE = "hub_debug.log" # (not wired yet)
|
| 538 |
+
LOG_REQUESTS = "false" # (not wired yet)
|
| 539 |
+
LOG_RESPONSES = "false" # (not wired yet — would log sensitive data!)
|
| 540 |
+
[DEBUG_END]
|
| 541 |
+
|
| 542 |
+
# =============================================================================
|
| 543 |
+
# PARSER_PROVIDER FREE ENDPOINTS — soon
|
| 544 |
+
# =============================================================================
|
| 545 |
+
[PARSER_PROVIDER]
|
| 546 |
+
#[PARSER_PROVIDER.metaculus]
|
| 547 |
+
#active = "true"
|
| 548 |
+
#base_url = "https://www.metaculus.com/api2"
|
| 549 |
+
#env_key = "" # public API, kein key!
|
| 550 |
+
#category = "research"
|
| 551 |
+
#legal_de = "true" # ← Deutschland-Check direkt in config!
|
| 552 |
+
#[PARSER_PROVIDER.metaculus_END]
|
| 553 |
+
|
| 554 |
+
#[PARSER_PROVIDER.polymarket]
|
| 555 |
+
#active = "false"
|
| 556 |
+
#base_url = "https://gamma-api.polymarket.com"
|
| 557 |
+
#env_key = ""
|
| 558 |
+
#legal_de = "false"
|
| 559 |
+
#[PARSER_PROVIDER.polymarket_END]
|
| 560 |
+
[PARSER_PROVIDER_END]
|
| 561 |
+
|
app/README.md
ADDED
|
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
|
|
|
| 1 |
+
> Haha! Sandboxed MCP server! (No nelson image cause big "D")
|
| 2 |
+
|
app/__init__.py
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# app/__init__.py
|
| 2 |
+
#from . import config
|
| 3 |
+
#from . import providers
|
| 4 |
+
#from . import models
|
| 5 |
+
#from . import tools
|
| 6 |
+
#from . import db_sync
|
| 7 |
+
#from . import mcp
|
app/app.py
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# root/app/app.py
|
| 3 |
+
# Universal MCP Hub (Sandboxed) - based on PyFundaments Architecture
|
| 4 |
+
# Copyright 2026 - Volkan Kücükbudak
|
| 5 |
+
# Apache License V. 2 + ESOL 1.1
|
| 6 |
+
# Repo: https://github.com/VolkanSah/Universal-MCP-Hub-sandboxed
|
| 7 |
+
# =============================================================================
|
| 8 |
+
# ARCHITECTURE NOTE:
|
| 9 |
+
# This file is the Orchestrator of the sandboxed app/* layer.
|
| 10 |
+
# It is ONLY started by main.py (the "Guardian").
|
| 11 |
+
# All fundament services are injected via the `fundaments` dictionary.
|
| 12 |
+
# Direct execution is blocked by design.
|
| 13 |
+
#
|
| 14 |
+
# SANDBOX RULES:
|
| 15 |
+
# - fundaments dict is ONLY unpacked inside start_application()
|
| 16 |
+
# - fundaments are NEVER stored globally or passed to other app/* modules
|
| 17 |
+
# - app/* modules read their own config from app/.pyfun
|
| 18 |
+
# - app/* internal state/IPC uses app/db_sync.py (SQLite) — NOT postgresql.py
|
| 19 |
+
# - Secrets stay in .env → Guardian reads them → never touched by app/*
|
| 20 |
+
# =============================================================================
|
| 21 |
+
from quart import Quart, request, jsonify # async Flask — ASGI compatible
|
| 22 |
+
import logging
|
| 23 |
+
from hypercorn.asyncio import serve # ASGI server — async native, replaces waitress
|
| 24 |
+
from hypercorn.config import Config # hypercorn config
|
| 25 |
+
from datetime import datetime, timezone
|
| 26 |
+
import asyncio
|
| 27 |
+
from typing import Dict, Any
|
| 28 |
+
# =============================================================================
|
| 29 |
+
# Import app/* modules — MINIMAL BUILD
|
| 30 |
+
# Each module reads its own config from app/.pyfun independently.
|
| 31 |
+
# NO fundaments passed into these modules!
|
| 32 |
+
# =============================================================================
|
| 33 |
+
from . import mcp # MCP transport layer (Streamable HTTP / SSE via ASGI mount)
|
| 34 |
+
from . import config as app_config # app/.pyfun parser — used only in app/*
|
| 35 |
+
from . import providers # API provider registry — reads app/.pyfun
|
| 36 |
+
from . import models # Model config + token/rate limits — reads app/.pyfun
|
| 37 |
+
from . import pricing # Auto model costs (OpenRouter + LiteLLM) — reads app/.pyfun
|
| 38 |
+
from . import tools # MCP tool definitions + provider mapping — reads app/.pyfun
|
| 39 |
+
from . import db_sync # Internal SQLite IPC — app/* state & communication
|
| 40 |
+
from . import shellmaster # ShellMaster 2.0 — safe shell assistant (inactive by default)
|
| 41 |
+
# # db_sync ≠ postgresql.py! Cloud DB is Guardian-only.
|
| 42 |
+
# Future modules (will uncomment when ready):
|
| 43 |
+
# from . import discord_api # Discord bot integration
|
| 44 |
+
# from . import hf_hooks # HuggingFace Space hooks
|
| 45 |
+
# from . import git_hooks # GitHub/GitLab webhook handler
|
| 46 |
+
# from . import web_api # Generic REST API handler
|
| 47 |
+
# =============================================================================
|
| 48 |
+
# Loggers — one per module for clean log filtering
|
| 49 |
+
# =============================================================================
|
| 50 |
+
logger = logging.getLogger('application')
|
| 51 |
+
logger_mcp = logging.getLogger('mcp')
|
| 52 |
+
logger_config = logging.getLogger('config')
|
| 53 |
+
logger_tools = logging.getLogger('tools')
|
| 54 |
+
logger_providers = logging.getLogger('providers')
|
| 55 |
+
logger_models = logging.getLogger('models')
|
| 56 |
+
logger_db_sync = logging.getLogger('db_sync')
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
|
| 60 |
+
# ── NEU: nach den Imports, vor app = Quart(__name__) ──────────────────────────
|
| 61 |
+
|
| 62 |
+
def _make_mount_middleware(outer_app, prefixes, inner_app, strip_prefix: bool = False):
|
| 63 |
+
"""
|
| 64 |
+
ASGI-Middleware: leitet Requests mit passendem Prefix an inner_app
|
| 65 |
+
(FastMCP Streamable HTTP / SSE) weiter, alles andere geht an outer_app (Quart).
|
| 66 |
+
Lifespan-Scopes gehen immer an outer_app (Quart).
|
| 67 |
+
|
| 68 |
+
WICHTIG: Prefix wird standardmaessig NICHT gestript - FastMCP registriert
|
| 69 |
+
seine Route intern selbst (settings.streamable_http_path = "/mcp").
|
| 70 |
+
Wer stript, erzeugt 404s. Genau das war der Break im alten Build.
|
| 71 |
+
"""
|
| 72 |
+
async def middleware(scope, receive, send):
|
| 73 |
+
path = scope.get("path", "")
|
| 74 |
+
for prefix in prefixes:
|
| 75 |
+
if path == prefix or path.startswith(prefix + "/"):
|
| 76 |
+
if strip_prefix:
|
| 77 |
+
scope = dict(scope)
|
| 78 |
+
stripped = path[len(prefix):] or "/"
|
| 79 |
+
scope["path"] = stripped
|
| 80 |
+
scope["raw_path"] = stripped.encode()
|
| 81 |
+
await inner_app(scope, receive, send)
|
| 82 |
+
return
|
| 83 |
+
await outer_app(scope, receive, send)
|
| 84 |
+
return middleware
|
| 85 |
+
|
| 86 |
+
# =============================================================================
|
| 87 |
+
# Quart app instance
|
| 88 |
+
# =============================================================================
|
| 89 |
+
app = Quart(__name__)
|
| 90 |
+
START_TIME = datetime.now(timezone.utc)
|
| 91 |
+
# =============================================================================
|
| 92 |
+
# Quart Routes
|
| 93 |
+
# =============================================================================
|
| 94 |
+
@app.route("/", methods=["GET"])
|
| 95 |
+
async def health_check():
|
| 96 |
+
"""
|
| 97 |
+
Health check endpoint.
|
| 98 |
+
Used by HuggingFace Spaces and monitoring systems to verify the app is running.
|
| 99 |
+
"""
|
| 100 |
+
uptime = datetime.now(timezone.utc) - START_TIME
|
| 101 |
+
return jsonify({
|
| 102 |
+
"status": "running",
|
| 103 |
+
"service": "Universal MCP Hub",
|
| 104 |
+
"uptime_seconds": int(uptime.total_seconds()),
|
| 105 |
+
})
|
| 106 |
+
|
| 107 |
+
|
| 108 |
+
@app.route("/api", methods=["POST"])
|
| 109 |
+
async def api_endpoint():
|
| 110 |
+
"""
|
| 111 |
+
REST tool gateway: POST {"tool": "name", "params": {...}}
|
| 112 |
+
Hook point: add auth / rate limiting here before dispatching.
|
| 113 |
+
"""
|
| 114 |
+
try:
|
| 115 |
+
data = await request.get_json(silent=True) or {}
|
| 116 |
+
tool_name = data.get("tool")
|
| 117 |
+
params = data.get("params", {}) or {}
|
| 118 |
+
|
| 119 |
+
if not tool_name:
|
| 120 |
+
return jsonify({"error": "Missing 'tool'"}), 400
|
| 121 |
+
|
| 122 |
+
# System tools — handle directly, no prompt needed!
|
| 123 |
+
if tool_name == "list_active_tools":
|
| 124 |
+
return jsonify({"result": {
|
| 125 |
+
"active_tools": tools.list_all(),
|
| 126 |
+
"active_llm_providers": providers.list_active_llm(),
|
| 127 |
+
"active_search_providers": providers.list_active_search(),
|
| 128 |
+
"available_models": models.list_all(),
|
| 129 |
+
"pricing_stats": pricing.stats(),
|
| 130 |
+
}})
|
| 131 |
+
|
| 132 |
+
if tool_name == "health_check":
|
| 133 |
+
return jsonify({"result": {"status": "ok"}})
|
| 134 |
+
|
| 135 |
+
# db_query — handled by db_sync directly, not tools.run()
|
| 136 |
+
if tool_name == "db_query":
|
| 137 |
+
sql = params.get("sql", "") or params.get("prompt", "")
|
| 138 |
+
if not sql:
|
| 139 |
+
return jsonify({"error": "db_query requires 'sql' (or 'prompt') parameter"}), 400
|
| 140 |
+
result = await db_sync.query(sql)
|
| 141 |
+
return jsonify({"result": result})
|
| 142 |
+
|
| 143 |
+
# persist_status — memory orchestration observability (no prompt needed)
|
| 144 |
+
if tool_name == "persist_status":
|
| 145 |
+
return jsonify({"result": await db_sync.persist_queue_stats()})
|
| 146 |
+
|
| 147 |
+
# get_model_pricing — auto-fetched costs, .pyfun values win as override
|
| 148 |
+
if tool_name == "get_model_pricing":
|
| 149 |
+
model_name = params.get("model_name", "") or params.get("model", "")
|
| 150 |
+
if not model_name:
|
| 151 |
+
return jsonify({"error": "get_model_pricing requires 'model_name'"}), 400
|
| 152 |
+
result = await pricing.get_pricing(model_name)
|
| 153 |
+
return jsonify({"result": result})
|
| 154 |
+
|
| 155 |
+
# rename 'provider' → 'provider_name' for tools.run()
|
| 156 |
+
if "provider" in params:
|
| 157 |
+
params["provider_name"] = params.pop("provider")
|
| 158 |
+
|
| 159 |
+
result = await tools.run(tool_name, **params)
|
| 160 |
+
return jsonify({"result": result})
|
| 161 |
+
|
| 162 |
+
except ValueError as e: # unknown tool / missing prompt → 400
|
| 163 |
+
return jsonify({"error": str(e)}), 400
|
| 164 |
+
except KeyError as e:
|
| 165 |
+
return jsonify({"error": str(e.args[0])}), 404
|
| 166 |
+
except RuntimeError as e:
|
| 167 |
+
# Provider-Kette erschöpft — Message ist upstream sanitisiert
|
| 168 |
+
# (nur HTTP-Status + safe_url, niemals Keys/Bodies)
|
| 169 |
+
return jsonify({"error": str(e)}), 502
|
| 170 |
+
except Exception as e:
|
| 171 |
+
logger.error(f"API error: {type(e).__name__}: {e}")
|
| 172 |
+
return jsonify({"error": "Internal error"}), 500
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
@app.route("/crypto", methods=["POST"])
|
| 176 |
+
async def crypto_endpoint():
|
| 177 |
+
"""
|
| 178 |
+
Encrypted API endpoint.
|
| 179 |
+
Encryption handled by app/* layer — no direct fundaments access here.
|
| 180 |
+
"""
|
| 181 |
+
# TODO: implement via app/* encryption wrapper
|
| 182 |
+
data = await request.get_json()
|
| 183 |
+
return jsonify({"status": "not_implemented"}), 501
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
# Future routes (uncomment when ready):
|
| 187 |
+
# @app.route("/discord", methods=["POST"])
|
| 188 |
+
# async def discord_interactions():
|
| 189 |
+
# """Discord interactions endpoint — signature verification via discord_api module."""
|
| 190 |
+
# pass
|
| 191 |
+
|
| 192 |
+
# @app.route("/webhook/hf", methods=["POST"])
|
| 193 |
+
# async def hf_webhook():
|
| 194 |
+
# """HuggingFace Space event hooks."""
|
| 195 |
+
# pass
|
| 196 |
+
|
| 197 |
+
# @app.route("/webhook/git", methods=["POST"])
|
| 198 |
+
# async def git_webhook():
|
| 199 |
+
# """GitHub / GitLab webhook handler."""
|
| 200 |
+
# pass
|
| 201 |
+
# =============================================================================
|
| 202 |
+
# Main entry point — called exclusively by Guardian (main.py)
|
| 203 |
+
# =============================================================================
|
| 204 |
+
async def start_application(fundaments: Dict[str, Any]) -> None:
|
| 205 |
+
"""
|
| 206 |
+
Main entry point for the sandboxed app layer.
|
| 207 |
+
Called exclusively by main.py after all fundament services are initialized.
|
| 208 |
+
|
| 209 |
+
Args:
|
| 210 |
+
fundaments: Dictionary of initialized services from Guardian (main.py).
|
| 211 |
+
Services are unpacked here and NEVER stored globally or
|
| 212 |
+
passed into other app/* modules.
|
| 213 |
+
"""
|
| 214 |
+
logger.info("Application starting...")
|
| 215 |
+
|
| 216 |
+
# =========================================================================
|
| 217 |
+
# Unpack fundaments — ONLY here, NEVER elsewhere in app/*
|
| 218 |
+
# These are the 6 fundament services from fundaments/*
|
| 219 |
+
# =========================================================================
|
| 220 |
+
config_service = fundaments["config"] # fundaments/config_handler.py
|
| 221 |
+
db_service = fundaments["db"] # fundaments/postgresql.py — None if not configured
|
| 222 |
+
encryption_service = fundaments["encryption"] # fundaments/encryption.py — None if keys not set
|
| 223 |
+
access_control_service = fundaments["access_control"] # fundaments/access_control.py — None if no DB
|
| 224 |
+
user_handler_service = fundaments["user_handler"] # fundaments/user_handler.py — None if no DB
|
| 225 |
+
security_service = fundaments["security"] # fundaments/security.py — None if deps missing
|
| 226 |
+
|
| 227 |
+
# --- Log active fundament services ---
|
| 228 |
+
if encryption_service:
|
| 229 |
+
logger.info("Encryption service active.")
|
| 230 |
+
|
| 231 |
+
if user_handler_service and security_service:
|
| 232 |
+
logger.info("Auth services active (user_handler + security).")
|
| 233 |
+
|
| 234 |
+
if access_control_service and security_service:
|
| 235 |
+
logger.info("Access control active.")
|
| 236 |
+
|
| 237 |
+
if db_service and not user_handler_service:
|
| 238 |
+
logger.info("Database-only mode active (e.g. ML pipeline).")
|
| 239 |
+
|
| 240 |
+
if not db_service:
|
| 241 |
+
logger.info("Database-free mode active (e.g. Discord bot, API client).")
|
| 242 |
+
|
| 243 |
+
# =========================================================================
|
| 244 |
+
# Initialize app/* internal services — MINIMAL BUILD
|
| 245 |
+
# Each module reads its own config from app/.pyfun — no fundaments passed in.
|
| 246 |
+
# =========================================================================
|
| 247 |
+
|
| 248 |
+
# db_sync only if cloud_DB used to!
|
| 249 |
+
# PSQL bridge — nur wenn Guardian DB-Service injiziert hat
|
| 250 |
+
await db_sync.initialize()
|
| 251 |
+
|
| 252 |
+
if db_service:
|
| 253 |
+
# asyncpg Pool direkt nutzen — kein execute_secured_query nötig
|
| 254 |
+
db_sync.set_psql_writer(db_service.execute)
|
| 255 |
+
logger.info("PostgreSQL bridge active.")
|
| 256 |
+
else:
|
| 257 |
+
logger.info("PostgreSQL bridge inactive — no DATABASE_URL configured.")
|
| 258 |
+
|
| 259 |
+
# Memory-Orchestrierung: Retry-Flusher starten — läuft MIT und OHNE PG.
|
| 260 |
+
# Ohne Writer bleibt alles restart-safe in der SQLite-Queue liegen
|
| 261 |
+
# (persist_result ohne DATABASE_URL = queued, kein Crash).
|
| 262 |
+
db_sync.start_persist_flusher()
|
| 263 |
+
|
| 264 |
+
# ShellMaster — bleibt inaktiv, bis .pyfun [TOOL.shellmaster] active="true"
|
| 265 |
+
# UND token + commands-Datei vorhanden sind (siehe shellmaster.initialize)
|
| 266 |
+
await shellmaster.initialize()
|
| 267 |
+
|
| 268 |
+
# Auto-Pricing — füllt Kosten aus OpenRouter/LiteLLM nach, .pyfun gewinnt.
|
| 269 |
+
# Läuft im Hintergrund — blockiert den Start NICHT (no fetch = no crash).
|
| 270 |
+
await pricing.initialize()
|
| 271 |
+
|
| 272 |
+
await mcp.initialize()
|
| 273 |
+
|
| 274 |
+
# =========================================================================
|
| 275 |
+
# MCP: ASGI-App holen, DANACH mounten, DANACH lifespan starten
|
| 276 |
+
# (get_asgi_app() muss VOR mcp.lifespan() kommen — siehe mcp.py)
|
| 277 |
+
# =========================================================================
|
| 278 |
+
hub_cfg = app_config.get_hub()
|
| 279 |
+
mcp_asgi = mcp.get_asgi_app()
|
| 280 |
+
app.asgi_app = _make_mount_middleware(app.asgi_app, mcp.mount_prefixes(), mcp_asgi)
|
| 281 |
+
logger.info(f"MCP mounted: {', '.join(mcp.mount_prefixes())}")
|
| 282 |
+
|
| 283 |
+
# --- Read HOST/PORT from app/.pyfun [HUB] ---
|
| 284 |
+
host = hub_cfg.get("HUB_HOST", "0.0.0.0")
|
| 285 |
+
port = int(hub_cfg.get("HUB_PORT", "7860")) # hub_cfg bereits gelesen
|
| 286 |
+
|
| 287 |
+
# --- Configure hypercorn ---
|
| 288 |
+
config = Config()
|
| 289 |
+
config.bind = [f"{host}:{port}"]
|
| 290 |
+
|
| 291 |
+
logger.info(f"Starting hypercorn on {host}:{port}...")
|
| 292 |
+
|
| 293 |
+
# --- Run hypercorn — blocks until shutdown ---
|
| 294 |
+
# mcp.lifespan() ist PFLICHT: startet den FastMCP Session-Manager.
|
| 295 |
+
# Ohne ihn schlägt jeder /mcp-Request mit
|
| 296 |
+
# 'Task group is not initialized' fehl.
|
| 297 |
+
async with mcp.lifespan():
|
| 298 |
+
logger.info("All services running.")
|
| 299 |
+
await serve(app, config)
|
| 300 |
+
|
| 301 |
+
# =============================================================================
|
| 302 |
+
# Direct execution guard
|
| 303 |
+
# =============================================================================
|
| 304 |
+
if __name__ == '__main__':
|
| 305 |
+
print("WARNING: Running app.py directly. Fundament modules might not be correctly initialized.")
|
| 306 |
+
print("Please run 'python main.py' instead for proper initialization.")
|
| 307 |
+
|
| 308 |
+
test_fundaments = {
|
| 309 |
+
"config": None,
|
| 310 |
+
"db": None,
|
| 311 |
+
"encryption": None,
|
| 312 |
+
"access_control": None,
|
| 313 |
+
"user_handler": None,
|
| 314 |
+
"security": None,
|
| 315 |
+
}
|
| 316 |
+
|
| 317 |
+
asyncio.run(start_application(test_fundaments))
|
app/config.py
ADDED
|
@@ -0,0 +1,298 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# app/config.py
|
| 3 |
+
# .pyfun parser for app/* modules
|
| 4 |
+
# Universal MCP Hub (Sandboxed) - based on PyFundaments Architecture
|
| 5 |
+
# Copyright 2026 - Volkan Kücükbudak
|
| 6 |
+
# Apache License V. 2 + ESOL 1.1
|
| 7 |
+
# =============================================================================
|
| 8 |
+
# USAGE in any app/* module:
|
| 9 |
+
# from . import config
|
| 10 |
+
# cfg = config.get()
|
| 11 |
+
# providers = cfg["LLM_PROVIDERS"]
|
| 12 |
+
# =============================================================================
|
| 13 |
+
# USAGE
|
| 14 |
+
# in providers.py
|
| 15 |
+
# from . import config
|
| 16 |
+
|
| 17 |
+
# active = config.get_active_llm_providers()
|
| 18 |
+
# → { "anthropic": { "base_url": "...", "env_key": "ANTHROPIC_API_KEY", ... }, ... }
|
| 19 |
+
# =============================================================================
|
| 20 |
+
# in models.py
|
| 21 |
+
# from . import config
|
| 22 |
+
|
| 23 |
+
# anthropic_models = config.get_models_for_provider("anthropic")
|
| 24 |
+
# =============================================================================
|
| 25 |
+
# in tools.py
|
| 26 |
+
# from . import config
|
| 27 |
+
|
| 28 |
+
# active_tools = config.get_active_tools()
|
| 29 |
+
# =============================================================================
|
| 30 |
+
import os
|
| 31 |
+
import logging
|
| 32 |
+
from typing import Dict, Any, Optional
|
| 33 |
+
|
| 34 |
+
logger = logging.getLogger('app.config')
|
| 35 |
+
|
| 36 |
+
# Path to .pyfun — lives in app/ next to this file
|
| 37 |
+
PYFUN_PATH = os.path.join(os.path.dirname(__file__), ".pyfun")
|
| 38 |
+
|
| 39 |
+
# Internal cache — loaded once at first get()
|
| 40 |
+
_cache: Optional[Dict[str, Any]] = None
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
def _parse_value(value: str) -> str:
|
| 44 |
+
"""Strip quotes and inline comments from a value."""
|
| 45 |
+
value = value.strip()
|
| 46 |
+
# Remove inline comment
|
| 47 |
+
if " #" in value:
|
| 48 |
+
value = value[:value.index(" #")].strip()
|
| 49 |
+
# Strip surrounding quotes
|
| 50 |
+
if value.startswith('"') and value.endswith('"'):
|
| 51 |
+
value = value[1:-1]
|
| 52 |
+
return value
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _parse() -> Dict[str, Any]:
|
| 56 |
+
"""
|
| 57 |
+
Parses the app/.pyfun file into a nested dictionary.
|
| 58 |
+
|
| 59 |
+
Structure:
|
| 60 |
+
[SECTION]
|
| 61 |
+
[SUBSECTION]
|
| 62 |
+
[BLOCK.name]
|
| 63 |
+
key = "value"
|
| 64 |
+
[BLOCK.name_END]
|
| 65 |
+
[SUBSECTION_END]
|
| 66 |
+
[SECTION_END]
|
| 67 |
+
|
| 68 |
+
Returns nested dict:
|
| 69 |
+
{
|
| 70 |
+
"HUB": { "HUB_NAME": "...", ... },
|
| 71 |
+
"LLM_PROVIDERS": {
|
| 72 |
+
"anthropic": { "active": "true", "base_url": "...", ... },
|
| 73 |
+
"gemini": { ... },
|
| 74 |
+
},
|
| 75 |
+
"MODELS": {
|
| 76 |
+
"claude-opus-4-6": { "provider": "anthropic", ... },
|
| 77 |
+
},
|
| 78 |
+
...
|
| 79 |
+
}
|
| 80 |
+
"""
|
| 81 |
+
if not os.path.isfile(PYFUN_PATH):
|
| 82 |
+
logger.critical(f".pyfun not found at: {PYFUN_PATH}")
|
| 83 |
+
raise FileNotFoundError(f".pyfun not found at: {PYFUN_PATH}")
|
| 84 |
+
|
| 85 |
+
result: Dict[str, Any] = {}
|
| 86 |
+
|
| 87 |
+
# Parser state
|
| 88 |
+
section: Optional[str] = None # e.g. "HUB", "PROVIDERS"
|
| 89 |
+
subsection: Optional[str] = None # e.g. "LLM_PROVIDERS"
|
| 90 |
+
block_type: Optional[str] = None # e.g. "LLM_PROVIDER", "MODEL", "TOOL"
|
| 91 |
+
block_name: Optional[str] = None # e.g. "anthropic", "claude-opus-4-6"
|
| 92 |
+
|
| 93 |
+
with open(PYFUN_PATH, "r", encoding="utf-8") as f:
|
| 94 |
+
for raw_line in f:
|
| 95 |
+
line = raw_line.strip()
|
| 96 |
+
|
| 97 |
+
# Skip empty lines and full-line comments
|
| 98 |
+
if not line or line.startswith("#"):
|
| 99 |
+
continue
|
| 100 |
+
|
| 101 |
+
# Skip file identifier
|
| 102 |
+
if line.startswith("[PYFUN_FILE"):
|
| 103 |
+
continue
|
| 104 |
+
|
| 105 |
+
# --- Block END markers (most specific first) ---
|
| 106 |
+
if line.endswith("_END]") and "." in line:
|
| 107 |
+
# e.g. [LLM_PROVIDER.anthropic_END] or [MODEL.claude-opus-4-6_END]
|
| 108 |
+
block_type = None
|
| 109 |
+
block_name = None
|
| 110 |
+
continue
|
| 111 |
+
|
| 112 |
+
if line.endswith("_END]") and not "." in line:
|
| 113 |
+
# e.g. [LLM_PROVIDERS_END], [HUB_END], [MODELS_END]
|
| 114 |
+
inner = line[1:-1].replace("_END", "")
|
| 115 |
+
if subsection and inner == subsection:
|
| 116 |
+
subsection = None
|
| 117 |
+
elif section and inner == section:
|
| 118 |
+
section = None
|
| 119 |
+
continue
|
| 120 |
+
|
| 121 |
+
# --- Block START markers ---
|
| 122 |
+
if line.startswith("[") and line.endswith("]"):
|
| 123 |
+
inner = line[1:-1]
|
| 124 |
+
|
| 125 |
+
# Named block: [LLM_PROVIDER.anthropic] or [MODEL.claude-opus-4-6]
|
| 126 |
+
if "." in inner:
|
| 127 |
+
parts = inner.split(".", 1)
|
| 128 |
+
block_type = parts[0] # e.g. LLM_PROVIDER, MODEL, TOOL
|
| 129 |
+
block_name = parts[1] # e.g. anthropic, claude-opus-4-6
|
| 130 |
+
|
| 131 |
+
# Determine which top-level key to store under
|
| 132 |
+
if block_type == "LLM_PROVIDER":
|
| 133 |
+
result.setdefault("LLM_PROVIDERS", {})
|
| 134 |
+
result["LLM_PROVIDERS"].setdefault(block_name, {})
|
| 135 |
+
elif block_type == "SEARCH_PROVIDER":
|
| 136 |
+
result.setdefault("SEARCH_PROVIDERS", {})
|
| 137 |
+
result["SEARCH_PROVIDERS"].setdefault(block_name, {})
|
| 138 |
+
elif block_type == "WEB_PROVIDER":
|
| 139 |
+
result.setdefault("WEB_PROVIDERS", {})
|
| 140 |
+
result["WEB_PROVIDERS"].setdefault(block_name, {})
|
| 141 |
+
elif block_type == "MODEL":
|
| 142 |
+
result.setdefault("MODELS", {})
|
| 143 |
+
result["MODELS"].setdefault(block_name, {})
|
| 144 |
+
elif block_type == "TOOL":
|
| 145 |
+
result.setdefault("TOOLS", {})
|
| 146 |
+
result["TOOLS"].setdefault(block_name, {})
|
| 147 |
+
continue
|
| 148 |
+
|
| 149 |
+
# Subsection: [LLM_PROVIDERS], [SEARCH_PROVIDERS] etc.
|
| 150 |
+
if section and not subsection:
|
| 151 |
+
subsection = inner
|
| 152 |
+
result.setdefault(inner, {})
|
| 153 |
+
continue
|
| 154 |
+
|
| 155 |
+
# Top-level section: [HUB], [PROVIDERS], [MODELS] etc.
|
| 156 |
+
section = inner
|
| 157 |
+
result.setdefault(inner, {})
|
| 158 |
+
continue
|
| 159 |
+
|
| 160 |
+
# --- Key = Value ---
|
| 161 |
+
if "=" in line:
|
| 162 |
+
key, _, val = line.partition("=")
|
| 163 |
+
key = key.strip()
|
| 164 |
+
val = _parse_value(val)
|
| 165 |
+
|
| 166 |
+
# Strip provider prefix from key (e.g. "anthropic.base_url" → "base_url")
|
| 167 |
+
if block_name and key.startswith(f"{block_name}."):
|
| 168 |
+
key = key[len(block_name) + 1:]
|
| 169 |
+
|
| 170 |
+
# Store in correct location
|
| 171 |
+
if block_type and block_name:
|
| 172 |
+
if block_type == "LLM_PROVIDER":
|
| 173 |
+
result["LLM_PROVIDERS"][block_name][key] = val
|
| 174 |
+
elif block_type == "SEARCH_PROVIDER":
|
| 175 |
+
result["SEARCH_PROVIDERS"][block_name][key] = val
|
| 176 |
+
elif block_type == "WEB_PROVIDER":
|
| 177 |
+
result["WEB_PROVIDERS"][block_name][key] = val
|
| 178 |
+
elif block_type == "MODEL":
|
| 179 |
+
result["MODELS"][block_name][key] = val
|
| 180 |
+
elif block_type == "TOOL":
|
| 181 |
+
result["TOOLS"][block_name][key] = val
|
| 182 |
+
elif section:
|
| 183 |
+
result[section][key] = val
|
| 184 |
+
|
| 185 |
+
logger.info(f".pyfun loaded. Sections: {list(result.keys())}")
|
| 186 |
+
return result
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def load() -> Dict[str, Any]:
|
| 190 |
+
"""Force (re)load of .pyfun — clears cache."""
|
| 191 |
+
global _cache
|
| 192 |
+
_cache = _parse()
|
| 193 |
+
return _cache
|
| 194 |
+
|
| 195 |
+
|
| 196 |
+
def get() -> Dict[str, Any]:
|
| 197 |
+
"""
|
| 198 |
+
Returns parsed .pyfun config as nested dict.
|
| 199 |
+
Loads and caches on first call — subsequent calls return cache.
|
| 200 |
+
"""
|
| 201 |
+
global _cache
|
| 202 |
+
if _cache is None:
|
| 203 |
+
_cache = _parse()
|
| 204 |
+
return _cache
|
| 205 |
+
|
| 206 |
+
|
| 207 |
+
def get_section(section: str) -> Dict[str, Any]:
|
| 208 |
+
"""
|
| 209 |
+
Returns a specific top-level section.
|
| 210 |
+
Returns empty dict if section not found.
|
| 211 |
+
"""
|
| 212 |
+
return get().get(section, {})
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def get_llm_providers() -> Dict[str, Any]:
|
| 216 |
+
"""Returns all LLM providers (active and inactive)."""
|
| 217 |
+
return get().get("LLM_PROVIDERS", {})
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def get_active_llm_providers() -> Dict[str, Any]:
|
| 221 |
+
"""Returns only LLM providers where active = 'true'."""
|
| 222 |
+
return {
|
| 223 |
+
name: cfg
|
| 224 |
+
for name, cfg in get_llm_providers().items()
|
| 225 |
+
if cfg.get("active", "false").lower() == "true"
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
|
| 229 |
+
def get_search_providers() -> Dict[str, Any]:
|
| 230 |
+
"""Returns all search providers."""
|
| 231 |
+
return get().get("SEARCH_PROVIDERS", {})
|
| 232 |
+
|
| 233 |
+
|
| 234 |
+
def get_active_search_providers() -> Dict[str, Any]:
|
| 235 |
+
"""Returns only search providers where active = 'true'."""
|
| 236 |
+
return {
|
| 237 |
+
name: cfg
|
| 238 |
+
for name, cfg in get_search_providers().items()
|
| 239 |
+
if cfg.get("active", "false").lower() == "true"
|
| 240 |
+
}
|
| 241 |
+
|
| 242 |
+
|
| 243 |
+
def get_models() -> Dict[str, Any]:
|
| 244 |
+
"""Returns all model definitions."""
|
| 245 |
+
return get().get("MODELS", {})
|
| 246 |
+
|
| 247 |
+
|
| 248 |
+
def get_models_for_provider(provider_name: str) -> Dict[str, Any]:
|
| 249 |
+
"""Returns all models for a specific provider."""
|
| 250 |
+
return {
|
| 251 |
+
name: cfg
|
| 252 |
+
for name, cfg in get_models().items()
|
| 253 |
+
if cfg.get("provider", "") == provider_name
|
| 254 |
+
}
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def get_tools() -> Dict[str, Any]:
|
| 258 |
+
"""Returns all tool definitions."""
|
| 259 |
+
return get().get("TOOLS", {})
|
| 260 |
+
|
| 261 |
+
|
| 262 |
+
def get_active_tools() -> Dict[str, Any]:
|
| 263 |
+
"""Returns only tools where active = 'true'."""
|
| 264 |
+
return {
|
| 265 |
+
name: cfg
|
| 266 |
+
for name, cfg in get_tools().items()
|
| 267 |
+
if cfg.get("active", "false").lower() == "true"
|
| 268 |
+
}
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
def get_hub() -> Dict[str, Any]:
|
| 272 |
+
"""Returns [HUB] section."""
|
| 273 |
+
return get_section("HUB")
|
| 274 |
+
|
| 275 |
+
|
| 276 |
+
def get_limits() -> Dict[str, Any]:
|
| 277 |
+
"""Returns [HUB_LIMITS] section."""
|
| 278 |
+
return get_section("HUB_LIMITS")
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def get_db_sync() -> Dict[str, Any]:
|
| 282 |
+
"""Returns [DB_SYNC] section."""
|
| 283 |
+
return get_section("DB_SYNC")
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def get_debug() -> Dict[str, Any]:
|
| 287 |
+
"""Returns [DEBUG] section."""
|
| 288 |
+
return get_section("DEBUG")
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
def is_debug() -> bool:
|
| 292 |
+
"""Returns True if DEBUG = 'ON' in .pyfun."""
|
| 293 |
+
return get_debug().get("DEBUG", "OFF").upper() == "ON"
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
def get_pricing() -> Dict[str, Any]:
|
| 297 |
+
"""Returns [PRICING] section — auto model cost fetching."""
|
| 298 |
+
return get_section("PRICING")
|
app/db_sync.py
ADDED
|
@@ -0,0 +1,554 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
# =============================================================================
|
| 3 |
+
# app/db_sync.py
|
| 4 |
+
# Internal SQLite IPC — app/* state & communication
|
| 5 |
+
# Universal MCP Hub (Sandboxed) - based on PyFundaments Architecture
|
| 6 |
+
# Copyright 2026 - Volkan Kücükbudak
|
| 7 |
+
# Apache License V. 2 + ESOL 1.1
|
| 8 |
+
# Repo: https://github.com/VolkanSah/Universal-MCP-Hub-sandboxed
|
| 9 |
+
# =============================================================================
|
| 10 |
+
# ARCHITECTURE NOTE:
|
| 11 |
+
# This file lives exclusively in app/ and is ONLY started by app/app.py.
|
| 12 |
+
# NO direct access to fundaments/*, .env, or Guardian (main.py).
|
| 13 |
+
# DB path comes from app/.pyfun [DB_SYNC] → SQLITE_PATH via app/config.py.
|
| 14 |
+
#
|
| 15 |
+
# CRITICAL RULES:
|
| 16 |
+
# - This is NOT postgresql.py — cloud DB is Guardian-only!
|
| 17 |
+
# - db_sync ONLY manages its own tables (hub_state, tool_cache)
|
| 18 |
+
# - NEVER touch Guardian tables (users, sessions) — those belong to user_handler.py
|
| 19 |
+
# - SQLite path is shared with user_handler.py via SQLITE_PATH
|
| 20 |
+
# - app/* modules call db_sync.write() / db_sync.read() — never aiosqlite directly
|
| 21 |
+
#
|
| 22 |
+
# TABLE OWNERSHIP:
|
| 23 |
+
# users, sessions → Guardian (fundaments/user_handler.py) — DO NOT TOUCH!
|
| 24 |
+
# hub_state → db_sync (app/* internal state)
|
| 25 |
+
# tool_cache → db_sync (app/* tool response cache)
|
| 26 |
+
# persist_queue → db_sync (restart-safe queue → PostgreSQL long-term memory)
|
| 27 |
+
# =============================================================================
|
| 28 |
+
|
| 29 |
+
import asyncio
|
| 30 |
+
import re
|
| 31 |
+
import aiosqlite
|
| 32 |
+
import logging
|
| 33 |
+
import json
|
| 34 |
+
from datetime import datetime, timezone
|
| 35 |
+
from typing import Any, Dict, List, Optional
|
| 36 |
+
|
| 37 |
+
from . import config
|
| 38 |
+
|
| 39 |
+
logger = logging.getLogger("db_sync")
|
| 40 |
+
|
| 41 |
+
# =============================================================================
|
| 42 |
+
# Internal State
|
| 43 |
+
# =============================================================================
|
| 44 |
+
_db_path: Optional[str] = None
|
| 45 |
+
_initialized: bool = False
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
# =============================================================================
|
| 49 |
+
# Initialization — called by app/app.py (parameterless, sandboxed)
|
| 50 |
+
# =============================================================================
|
| 51 |
+
|
| 52 |
+
async def initialize() -> None:
|
| 53 |
+
global _db_path, _initialized
|
| 54 |
+
|
| 55 |
+
if _initialized:
|
| 56 |
+
return
|
| 57 |
+
|
| 58 |
+
db_cfg = config.get_db_sync()
|
| 59 |
+
raw_path = db_cfg.get("SQLITE_PATH", "app/.hub_state.db")
|
| 60 |
+
|
| 61 |
+
# HF Spaces: SPACE_ID is set → filesystem is read-only except /tmp/
|
| 62 |
+
import os
|
| 63 |
+
if os.getenv("SPACE_ID"):
|
| 64 |
+
filename = os.path.basename(raw_path)
|
| 65 |
+
_db_path = f"/tmp/{filename}"
|
| 66 |
+
logger.info(f"HF Space detected — SQLite relocated to {_db_path}")
|
| 67 |
+
else:
|
| 68 |
+
_db_path = raw_path
|
| 69 |
+
|
| 70 |
+
await _init_tables()
|
| 71 |
+
|
| 72 |
+
_initialized = True
|
| 73 |
+
logger.info(f"db_sync initialized — path: {_db_path}")
|
| 74 |
+
|
| 75 |
+
|
| 76 |
+
# =============================================================================
|
| 77 |
+
# SECTION 1 — Table Setup (app/* tables only!)
|
| 78 |
+
# =============================================================================
|
| 79 |
+
|
| 80 |
+
async def _init_tables() -> None:
|
| 81 |
+
"""
|
| 82 |
+
Creates app/* internal tables if they don't exist.
|
| 83 |
+
NEVER modifies Guardian tables (users, sessions).
|
| 84 |
+
"""
|
| 85 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 86 |
+
|
| 87 |
+
# hub_state — generic key/value store for app/* modules
|
| 88 |
+
await db.execute("""
|
| 89 |
+
CREATE TABLE IF NOT EXISTS hub_state (
|
| 90 |
+
key TEXT PRIMARY KEY,
|
| 91 |
+
value TEXT,
|
| 92 |
+
updated_at TEXT
|
| 93 |
+
)
|
| 94 |
+
""")
|
| 95 |
+
|
| 96 |
+
# tool_cache — cached tool responses to reduce API calls
|
| 97 |
+
await db.execute("""
|
| 98 |
+
CREATE TABLE IF NOT EXISTS tool_cache (
|
| 99 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 100 |
+
tool_name TEXT NOT NULL,
|
| 101 |
+
prompt TEXT NOT NULL,
|
| 102 |
+
response TEXT NOT NULL,
|
| 103 |
+
provider TEXT,
|
| 104 |
+
model TEXT,
|
| 105 |
+
created_at TEXT
|
| 106 |
+
)
|
| 107 |
+
""")
|
| 108 |
+
|
| 109 |
+
await db.execute("""
|
| 110 |
+
CREATE INDEX IF NOT EXISTS idx_tool_cache_tool
|
| 111 |
+
ON tool_cache(tool_name)
|
| 112 |
+
""")
|
| 113 |
+
|
| 114 |
+
# persist_queue — Dauer-Gedächtnis-Queue: überlebt Restarts,
|
| 115 |
+
# bis der Eintrag erfolgreich nach PostgreSQL geflusht wurde.
|
| 116 |
+
# score-Spalte = reservierter Slot für den Bullshit-Filter (Nutzer-eigenes
|
| 117 |
+
# Scoring, später) — wird von db_sync NICHT interpretiert.
|
| 118 |
+
await db.execute("""
|
| 119 |
+
CREATE TABLE IF NOT EXISTS persist_queue (
|
| 120 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 121 |
+
target_table TEXT NOT NULL,
|
| 122 |
+
payload TEXT NOT NULL,
|
| 123 |
+
source TEXT,
|
| 124 |
+
model TEXT,
|
| 125 |
+
score REAL,
|
| 126 |
+
queued_at TEXT NOT NULL,
|
| 127 |
+
attempts INTEGER NOT NULL DEFAULT 0,
|
| 128 |
+
last_error TEXT
|
| 129 |
+
)
|
| 130 |
+
""")
|
| 131 |
+
|
| 132 |
+
await db.commit()
|
| 133 |
+
|
| 134 |
+
logger.info("db_sync tables ready.")
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
# =============================================================================
|
| 138 |
+
# SECTION 2 — Key/Value Store (hub_state table)
|
| 139 |
+
# =============================================================================
|
| 140 |
+
|
| 141 |
+
async def write(key: str, value: Any) -> None:
|
| 142 |
+
"""
|
| 143 |
+
Write a value to hub_state key/value store.
|
| 144 |
+
Value is JSON-serialized — supports dicts, lists, strings, numbers.
|
| 145 |
+
|
| 146 |
+
Args:
|
| 147 |
+
key: Unique key string (e.g. 'scheduler.last_run').
|
| 148 |
+
value: Any JSON-serializable value.
|
| 149 |
+
"""
|
| 150 |
+
_check_init()
|
| 151 |
+
now = datetime.now(timezone.utc).isoformat()
|
| 152 |
+
|
| 153 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 154 |
+
await db.execute("""
|
| 155 |
+
INSERT OR REPLACE INTO hub_state (key, value, updated_at)
|
| 156 |
+
VALUES (?, ?, ?)
|
| 157 |
+
""", (key, json.dumps(value), now))
|
| 158 |
+
await db.commit()
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
async def read(key: str, default: Any = None) -> Any:
|
| 162 |
+
"""
|
| 163 |
+
Read a value from hub_state key/value store.
|
| 164 |
+
Returns default if key does not exist.
|
| 165 |
+
|
| 166 |
+
Args:
|
| 167 |
+
key: Key string to look up.
|
| 168 |
+
default: Default value if key not found. Default: None.
|
| 169 |
+
|
| 170 |
+
Returns:
|
| 171 |
+
Deserialized value, or default if not found.
|
| 172 |
+
"""
|
| 173 |
+
_check_init()
|
| 174 |
+
|
| 175 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 176 |
+
cursor = await db.execute(
|
| 177 |
+
"SELECT value FROM hub_state WHERE key = ?", (key,)
|
| 178 |
+
)
|
| 179 |
+
row = await cursor.fetchone()
|
| 180 |
+
|
| 181 |
+
if row is None:
|
| 182 |
+
return default
|
| 183 |
+
|
| 184 |
+
try:
|
| 185 |
+
return json.loads(row[0])
|
| 186 |
+
except (json.JSONDecodeError, TypeError):
|
| 187 |
+
return row[0]
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
async def delete(key: str) -> None:
|
| 191 |
+
"""
|
| 192 |
+
Delete a key from hub_state.
|
| 193 |
+
|
| 194 |
+
Args:
|
| 195 |
+
key: Key string to delete.
|
| 196 |
+
"""
|
| 197 |
+
_check_init()
|
| 198 |
+
|
| 199 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 200 |
+
await db.execute("DELETE FROM hub_state WHERE key = ?", (key,))
|
| 201 |
+
await db.commit()
|
| 202 |
+
|
| 203 |
+
|
| 204 |
+
# =============================================================================
|
| 205 |
+
# SECTION 3 — Tool Cache (tool_cache table)
|
| 206 |
+
# =============================================================================
|
| 207 |
+
|
| 208 |
+
async def cache_write(
|
| 209 |
+
tool_name: str,
|
| 210 |
+
prompt: str,
|
| 211 |
+
response: str,
|
| 212 |
+
provider: str = None,
|
| 213 |
+
model: str = None,
|
| 214 |
+
) -> None:
|
| 215 |
+
"""
|
| 216 |
+
Cache a tool response to reduce redundant API calls.
|
| 217 |
+
|
| 218 |
+
Args:
|
| 219 |
+
tool_name: Tool name (e.g. 'llm_complete', 'web_search').
|
| 220 |
+
prompt: The input prompt/query that was used.
|
| 221 |
+
response: The response to cache.
|
| 222 |
+
provider: Provider name used (optional).
|
| 223 |
+
model: Model name used (optional).
|
| 224 |
+
"""
|
| 225 |
+
_check_init()
|
| 226 |
+
|
| 227 |
+
db_cfg = config.get_db_sync()
|
| 228 |
+
max_entries = int(db_cfg.get("MAX_CACHE_ENTRIES", "1000"))
|
| 229 |
+
now = datetime.now(timezone.utc).isoformat()
|
| 230 |
+
|
| 231 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 232 |
+
await db.execute("""
|
| 233 |
+
INSERT INTO tool_cache (tool_name, prompt, response, provider, model, created_at)
|
| 234 |
+
VALUES (?, ?, ?, ?, ?, ?)
|
| 235 |
+
""", (tool_name, prompt, response, provider, model, now))
|
| 236 |
+
|
| 237 |
+
# Enforce MAX_CACHE_ENTRIES — delete oldest if exceeded
|
| 238 |
+
await db.execute("""
|
| 239 |
+
DELETE FROM tool_cache WHERE id NOT IN (
|
| 240 |
+
SELECT id FROM tool_cache ORDER BY created_at DESC LIMIT ?
|
| 241 |
+
)
|
| 242 |
+
""", (max_entries,))
|
| 243 |
+
|
| 244 |
+
await db.commit()
|
| 245 |
+
|
| 246 |
+
|
| 247 |
+
async def cache_read(tool_name: str, prompt: str) -> Optional[str]:
|
| 248 |
+
"""
|
| 249 |
+
Read a cached tool response.
|
| 250 |
+
Returns None if no cache entry exists.
|
| 251 |
+
|
| 252 |
+
Args:
|
| 253 |
+
tool_name: Tool name to look up.
|
| 254 |
+
prompt: The exact prompt/query to match.
|
| 255 |
+
|
| 256 |
+
Returns:
|
| 257 |
+
Cached response string, or None if not found.
|
| 258 |
+
"""
|
| 259 |
+
_check_init()
|
| 260 |
+
|
| 261 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 262 |
+
cursor = await db.execute("""
|
| 263 |
+
SELECT response FROM tool_cache
|
| 264 |
+
WHERE tool_name = ? AND prompt = ?
|
| 265 |
+
ORDER BY created_at DESC LIMIT 1
|
| 266 |
+
""", (tool_name, prompt))
|
| 267 |
+
row = await cursor.fetchone()
|
| 268 |
+
|
| 269 |
+
return row[0] if row else None
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
# =============================================================================
|
| 273 |
+
# SECTION 4 — Read-Only Query (for mcp.py db_query tool)
|
| 274 |
+
# =============================================================================
|
| 275 |
+
|
| 276 |
+
async def query(sql: str) -> List[Dict]:
|
| 277 |
+
"""
|
| 278 |
+
Execute a read-only SELECT query on the internal hub state database.
|
| 279 |
+
Only SELECT statements are permitted — write operations are blocked.
|
| 280 |
+
Called by mcp.py db_query tool when db_sync.py is active.
|
| 281 |
+
|
| 282 |
+
Args:
|
| 283 |
+
sql: SQL SELECT statement to execute.
|
| 284 |
+
|
| 285 |
+
Returns:
|
| 286 |
+
List of result rows as dicts.
|
| 287 |
+
|
| 288 |
+
Raises:
|
| 289 |
+
ValueError: If the query is not a SELECT statement.
|
| 290 |
+
"""
|
| 291 |
+
_check_init()
|
| 292 |
+
|
| 293 |
+
if not sql.strip().upper().startswith("SELECT"):
|
| 294 |
+
raise ValueError("Only SELECT queries are permitted in db_query tool.")
|
| 295 |
+
|
| 296 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 297 |
+
db.row_factory = aiosqlite.Row
|
| 298 |
+
cursor = await db.execute(sql)
|
| 299 |
+
rows = await cursor.fetchall()
|
| 300 |
+
return [dict(r) for r in rows]
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
# =============================================================================
|
| 304 |
+
# SECTION 5 — Helpers
|
| 305 |
+
# =============================================================================
|
| 306 |
+
|
| 307 |
+
def _check_init() -> None:
|
| 308 |
+
"""Raise RuntimeError if db_sync was not initialized."""
|
| 309 |
+
if not _initialized or not _db_path:
|
| 310 |
+
raise RuntimeError("db_sync not initialized — call initialize() first.")
|
| 311 |
+
|
| 312 |
+
|
| 313 |
+
def is_ready() -> bool:
|
| 314 |
+
"""Returns True if db_sync is initialized and ready."""
|
| 315 |
+
return _initialized and _db_path is not None
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
# =============================================================================
|
| 319 |
+
# SECTION 6 — Memory Orchestration: SQLite (Kurzzeit) → PostgreSQL (Dauer)
|
| 320 |
+
# =============================================================================
|
| 321 |
+
# ARCHITECTURE (Gedächtnis-Design):
|
| 322 |
+
# SQLite (hub_state / tool_cache / persist_queue) = Kurzzeit-Gedächtnis.
|
| 323 |
+
# Alle app/* Tools lesen/schreiben hier — immer verfügbar, kein Key nötig.
|
| 324 |
+
# PostgreSQL = Dauer-Gedächtnis — WICHTIGE Dinge wandern dorthin.
|
| 325 |
+
#
|
| 326 |
+
# persist_result Tool → enqueue_persist() (sofort, immer, kein Crash)
|
| 327 |
+
# ↓
|
| 328 |
+
# persist_queue (SQLite, überlebt Restarts)
|
| 329 |
+
# ↓
|
| 330 |
+
# flush_persist_queue() → PostgreSQL (Guardian-Writer)
|
| 331 |
+
# (Background-Retry bis die DB da ist — siehe start_persist_flusher)
|
| 332 |
+
#
|
| 333 |
+
# Delivery-Semantik: at-least-once (PG-Insert + Queue-Delete sind nicht
|
| 334 |
+
# atomar über zwei Systeme). Für ein Gedächtnis ist ein seltener Duplikat-
|
| 335 |
+
# Eintrag unkritisch — Datenverlust wäre es.
|
| 336 |
+
#
|
| 337 |
+
# HF-Spaces-Hinweis: SPACE_ID verschiebt SQLite nach /tmp — Queue überbrückt
|
| 338 |
+
# dort DB-Ausfälle innerhalb der Container-Lebenszeit; dauerhaft ist nur PG.
|
| 339 |
+
# =============================================================================
|
| 340 |
+
_psql_writer = None
|
| 341 |
+
_flush_task: Optional[asyncio.Task] = None
|
| 342 |
+
_pg_tables_ensured: set = set()
|
| 343 |
+
|
| 344 |
+
# Defense: Tabellennamen kommen aus .pyfun, aber wir validieren trotzdem.
|
| 345 |
+
_SAFE_TABLE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
| 346 |
+
|
| 347 |
+
# Ziel-Schema in PostgreSQL — source/model/score als eigene Spalten,
|
| 348 |
+
# damit dein Bullshit-Filter später direkt auf score filtern kann.
|
| 349 |
+
_PG_DDL = """
|
| 350 |
+
CREATE TABLE IF NOT EXISTS {table} (
|
| 351 |
+
id BIGSERIAL PRIMARY KEY,
|
| 352 |
+
created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
| 353 |
+
source TEXT,
|
| 354 |
+
model TEXT,
|
| 355 |
+
score REAL,
|
| 356 |
+
payload JSONB NOT NULL
|
| 357 |
+
)
|
| 358 |
+
"""
|
| 359 |
+
|
| 360 |
+
|
| 361 |
+
def set_psql_writer(writer_fn) -> None:
|
| 362 |
+
"""
|
| 363 |
+
Receives the asyncpg execute callable from Guardian via app.py.
|
| 364 |
+
Called once in start_application() if db_service is available.
|
| 365 |
+
app/* never imports postgresql.py directly — this is the only bridge.
|
| 366 |
+
"""
|
| 367 |
+
global _psql_writer
|
| 368 |
+
_psql_writer = writer_fn
|
| 369 |
+
_pg_tables_ensured.clear() # neue Connection → CREATE TABLE erneut sicherstellen
|
| 370 |
+
logger.info("PostgreSQL writer registered.")
|
| 371 |
+
|
| 372 |
+
|
| 373 |
+
async def enqueue_persist(
|
| 374 |
+
table: str,
|
| 375 |
+
payload: Any,
|
| 376 |
+
source: str = "",
|
| 377 |
+
model: str = "",
|
| 378 |
+
score: Optional[float] = None,
|
| 379 |
+
) -> int:
|
| 380 |
+
"""
|
| 381 |
+
Queue an entry for the PostgreSQL long-term memory.
|
| 382 |
+
ALWAYS works (no writer needed) — this is the 'no crash' guarantee:
|
| 383 |
+
persist_result ohne DATABASE_URL = Eintrag wartet restart-safe in der Queue.
|
| 384 |
+
|
| 385 |
+
Args:
|
| 386 |
+
table: Target table in PostgreSQL (from .pyfun target_table).
|
| 387 |
+
payload: JSON-serializable data to persist.
|
| 388 |
+
source: Which tool/component produced this entry.
|
| 389 |
+
model: Which model produced this entry.
|
| 390 |
+
score: Optional quality score — reserved slot for the bullshit filter.
|
| 391 |
+
|
| 392 |
+
Returns:
|
| 393 |
+
Queue depth after insert (int).
|
| 394 |
+
"""
|
| 395 |
+
_check_init()
|
| 396 |
+
max_q = int(config.get_db_sync().get("PERSIST_QUEUE_MAX", "500"))
|
| 397 |
+
now = datetime.now(timezone.utc).isoformat()
|
| 398 |
+
|
| 399 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 400 |
+
await db.execute(
|
| 401 |
+
"""INSERT INTO persist_queue
|
| 402 |
+
(target_table, payload, source, model, score, queued_at)
|
| 403 |
+
VALUES (?, ?, ?, ?, ?, ?)""",
|
| 404 |
+
(table, json.dumps(payload, ensure_ascii=False, default=str),
|
| 405 |
+
source, model, score, now),
|
| 406 |
+
)
|
| 407 |
+
# Queue begrenzen — älteste zuerst raus (gleiche Regel wie tool_cache)
|
| 408 |
+
await db.execute(
|
| 409 |
+
"DELETE FROM persist_queue WHERE id NOT IN "
|
| 410 |
+
"(SELECT id FROM persist_queue ORDER BY id DESC LIMIT ?)",
|
| 411 |
+
(max_q,),
|
| 412 |
+
)
|
| 413 |
+
await db.commit()
|
| 414 |
+
cursor = await db.execute("SELECT COUNT(*) FROM persist_queue")
|
| 415 |
+
return int((await cursor.fetchone())[0])
|
| 416 |
+
|
| 417 |
+
|
| 418 |
+
async def flush_persist_queue(limit: int = 50) -> Dict[str, Any]:
|
| 419 |
+
"""
|
| 420 |
+
Try to move queued entries to PostgreSQL. NEVER raises.
|
| 421 |
+
|
| 422 |
+
No writer → no-op with reason (entries keep waiting — no crash).
|
| 423 |
+
Writer set → CREATE TABLE IF NOT EXISTS + INSERT per entry + queue delete.
|
| 424 |
+
Failure → attempts+1 + sanitized last_error, entry stays queued.
|
| 425 |
+
|
| 426 |
+
Args:
|
| 427 |
+
limit: Max entries per flush run (bounded work per interval).
|
| 428 |
+
|
| 429 |
+
Returns:
|
| 430 |
+
Dict: {flushed, errors, remaining, reason}
|
| 431 |
+
"""
|
| 432 |
+
_check_init()
|
| 433 |
+
|
| 434 |
+
if not _psql_writer:
|
| 435 |
+
return {
|
| 436 |
+
"flushed": 0, "errors": 0,
|
| 437 |
+
"remaining": await _queue_count(),
|
| 438 |
+
"reason": "no_postgres_writer",
|
| 439 |
+
}
|
| 440 |
+
|
| 441 |
+
flushed = errors = 0
|
| 442 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 443 |
+
db.row_factory = aiosqlite.Row
|
| 444 |
+
cursor = await db.execute(
|
| 445 |
+
"SELECT id, target_table, payload, source, model, score "
|
| 446 |
+
"FROM persist_queue ORDER BY id ASC LIMIT ?",
|
| 447 |
+
(limit,),
|
| 448 |
+
)
|
| 449 |
+
rows = await cursor.fetchall()
|
| 450 |
+
|
| 451 |
+
for row in rows:
|
| 452 |
+
table = row["target_table"]
|
| 453 |
+
try:
|
| 454 |
+
if not _SAFE_TABLE_NAME.match(table or ""):
|
| 455 |
+
raise ValueError("invalid table name")
|
| 456 |
+
if table not in _pg_tables_ensured:
|
| 457 |
+
await _psql_writer(_PG_DDL.format(table=table))
|
| 458 |
+
_pg_tables_ensured.add(table)
|
| 459 |
+
await _psql_writer(
|
| 460 |
+
f"INSERT INTO {table} (source, model, score, payload) "
|
| 461 |
+
"VALUES ($1, $2, $3, $4::jsonb)",
|
| 462 |
+
row["source"], row["model"], row["score"], row["payload"],
|
| 463 |
+
)
|
| 464 |
+
await db.execute("DELETE FROM persist_queue WHERE id = ?", (row["id"],))
|
| 465 |
+
await db.commit()
|
| 466 |
+
flushed += 1
|
| 467 |
+
except Exception as e:
|
| 468 |
+
errors += 1
|
| 469 |
+
# Sanitized: nur Exception-TYP — niemals DSN/Payload in Logs oder DB
|
| 470 |
+
await db.execute(
|
| 471 |
+
"UPDATE persist_queue SET attempts = attempts + 1, last_error = ? "
|
| 472 |
+
"WHERE id = ?",
|
| 473 |
+
(type(e).__name__, row["id"]),
|
| 474 |
+
)
|
| 475 |
+
await db.commit()
|
| 476 |
+
|
| 477 |
+
remaining = await _queue_count()
|
| 478 |
+
if flushed:
|
| 479 |
+
logger.info(f"Persist flush: {flushed} → PostgreSQL, {remaining} queued.")
|
| 480 |
+
if errors:
|
| 481 |
+
logger.warning(f"Persist flush: {errors} failed — stay queued (retry next interval).")
|
| 482 |
+
return {"flushed": flushed, "errors": errors, "remaining": remaining}
|
| 483 |
+
|
| 484 |
+
|
| 485 |
+
async def _queue_count() -> int:
|
| 486 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 487 |
+
cursor = await db.execute("SELECT COUNT(*) FROM persist_queue")
|
| 488 |
+
return int((await cursor.fetchone())[0])
|
| 489 |
+
|
| 490 |
+
|
| 491 |
+
async def persist_queue_stats() -> Dict[str, Any]:
|
| 492 |
+
"""
|
| 493 |
+
Zustand des Gedächtnisses — Observability für die Orchestrierung.
|
| 494 |
+
Wird von persist_status (MCP + /api) gelesen.
|
| 495 |
+
"""
|
| 496 |
+
_check_init()
|
| 497 |
+
async with aiosqlite.connect(_db_path) as db:
|
| 498 |
+
cursor = await db.execute(
|
| 499 |
+
"SELECT COUNT(*), MIN(queued_at), "
|
| 500 |
+
"SUM(CASE WHEN attempts > 0 THEN 1 ELSE 0 END) FROM persist_queue"
|
| 501 |
+
)
|
| 502 |
+
row = await cursor.fetchone()
|
| 503 |
+
count, oldest, failed = row[0] or 0, row[1], row[2] or 0
|
| 504 |
+
return {
|
| 505 |
+
"queue_depth": count,
|
| 506 |
+
"oldest_queued_at": oldest,
|
| 507 |
+
"entries_with_failed_attempts": failed,
|
| 508 |
+
"postgres_bridge": "active" if _psql_writer else "inactive (DATABASE_URL not set)",
|
| 509 |
+
"note": "queue is restart-safe — flushes to PostgreSQL as soon as the bridge is active",
|
| 510 |
+
}
|
| 511 |
+
|
| 512 |
+
|
| 513 |
+
def start_persist_flusher() -> None:
|
| 514 |
+
"""
|
| 515 |
+
Background retry loop — started by app.py AFTER db_sync.initialize().
|
| 516 |
+
Works with AND without writer: ohne PG bleibt alles einfach liegen.
|
| 517 |
+
"""
|
| 518 |
+
global _flush_task
|
| 519 |
+
if _flush_task is not None:
|
| 520 |
+
return
|
| 521 |
+
interval = float(config.get_db_sync().get("PERSIST_FLUSH_INTERVAL_SEC", "60"))
|
| 522 |
+
_flush_task = asyncio.create_task(_persist_flush_loop(interval))
|
| 523 |
+
logger.info(f"Persist flusher started (interval: {interval}s).")
|
| 524 |
+
|
| 525 |
+
|
| 526 |
+
async def _persist_flush_loop(interval: float) -> None:
|
| 527 |
+
# Erster Versuch kurz nach dem Boot — fängt Einträge der letzten Session ab
|
| 528 |
+
# (genau dann, wenn PG wieder da ist, während er vorher down war).
|
| 529 |
+
await asyncio.sleep(5)
|
| 530 |
+
while True:
|
| 531 |
+
try:
|
| 532 |
+
result = await flush_persist_queue()
|
| 533 |
+
if result.get("flushed") or result.get("errors"):
|
| 534 |
+
logger.info(f"Persist flusher: {result}")
|
| 535 |
+
except Exception as e:
|
| 536 |
+
logger.warning(f"Persist flusher error (non-fatal): {type(e).__name__}")
|
| 537 |
+
await asyncio.sleep(interval)
|
| 538 |
+
|
| 539 |
+
|
| 540 |
+
async def persist(table: str, data: dict) -> None:
|
| 541 |
+
"""
|
| 542 |
+
Legacy direct-write — jetzt queue-safe: enqueue + sofortiger Flush-Versuch.
|
| 543 |
+
Beißt nicht mehr ohne DATABASE_URL (früher: RuntimeError). Backward-kompatibel
|
| 544 |
+
für externe Aufrufer; tools.py nutzt jetzt enqueue_persist() direkt.
|
| 545 |
+
"""
|
| 546 |
+
await enqueue_persist(table, payload=data)
|
| 547 |
+
await flush_persist_queue(limit=1)
|
| 548 |
+
|
| 549 |
+
# =============================================================================
|
| 550 |
+
# Direct execution guard
|
| 551 |
+
# =============================================================================
|
| 552 |
+
|
| 553 |
+
if __name__ == "__main__":
|
| 554 |
+
print("WARNING: Run via main.py → app.py, not directly.")
|
app/mcp.py
ADDED
|
@@ -0,0 +1,495 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# root/app/mcp.py
|
| 3 |
+
# 14.03.2026
|
| 4 |
+
# Universal MCP Hub (Sandboxed) - based on PyFundaments Architecture
|
| 5 |
+
# Copyright 2026 - Volkan Kücükbudak
|
| 6 |
+
# Apache License V. 2 + ESOL 1.1
|
| 7 |
+
# Repo: https://github.com/VolkanSah/Universal-MCP-Hub-sandboxed
|
| 8 |
+
# =============================================================================
|
| 9 |
+
# ARCHITECTURE NOTE:
|
| 10 |
+
# This file lives exclusively in app/ and is ONLY started by app/app.py.
|
| 11 |
+
# NO direct access to fundaments/*, .env, or Guardian (main.py).
|
| 12 |
+
# All config comes from app/.pyfun via app/config.py.
|
| 13 |
+
#
|
| 14 |
+
# TRANSPORT:
|
| 15 |
+
# Primary: Streamable HTTP (MCP spec 2025-11-25) → single /mcp endpoint
|
| 16 |
+
# Configured via HUB_TRANSPORT = "streamable-http" in .pyfun [HUB]
|
| 17 |
+
# ASGI-App via get_asgi_app() → mounted by app/app.py
|
| 18 |
+
#
|
| 19 |
+
# Fallback: SSE (legacy, deprecated per spec) → /mcp route via Quart
|
| 20 |
+
# Configured via HUB_TRANSPORT = "sse" in .pyfun [HUB]
|
| 21 |
+
# handle_request() called directly by app/app.py Quart route
|
| 22 |
+
#
|
| 23 |
+
# All MCP traffic (both transports) passes through app/app.py first —
|
| 24 |
+
# auth checks, rate limiting, logging can be added there before reaching MCP.
|
| 25 |
+
#
|
| 26 |
+
# TOOL REGISTRATION PRINCIPLE:
|
| 27 |
+
# Tools are registered via tools.py — NOT hardcoded here.
|
| 28 |
+
# No key = no provider = no tool = no crash.
|
| 29 |
+
# Server always starts, just with fewer tools.
|
| 30 |
+
# Adding a new tool = update .pyfun + providers.py only. Never touch mcp.py.
|
| 31 |
+
#
|
| 32 |
+
# DEPENDENCY CHAIN (app/* only, no fundaments!):
|
| 33 |
+
# config.py → parses app/.pyfun — single source of truth
|
| 34 |
+
# providers.py → LLM + Search provider registry + fallback chain
|
| 35 |
+
# models.py → model limits, costs, capabilities from .pyfun [MODELS]
|
| 36 |
+
# tools.py → tool registry + execution — reads .pyfun [TOOLS]
|
| 37 |
+
# db_sync.py → internal SQLite IPC (app/* state) — NOT postgresql.py!
|
| 38 |
+
# mcp.py → registers tools only, delegates all logic to tools.py
|
| 39 |
+
# =============================================================================
|
| 40 |
+
import contextlib
|
| 41 |
+
import logging
|
| 42 |
+
from typing import Dict, Any
|
| 43 |
+
from . import config as app_config
|
| 44 |
+
from . import providers
|
| 45 |
+
from . import models
|
| 46 |
+
from . import pricing
|
| 47 |
+
from . import tools
|
| 48 |
+
|
| 49 |
+
logger = logging.getLogger('mcp')
|
| 50 |
+
|
| 51 |
+
# =============================================================================
|
| 52 |
+
# Globals — set once during initialize(), never touched elsewhere
|
| 53 |
+
# =============================================================================
|
| 54 |
+
_mcp = None # FastMCP instance
|
| 55 |
+
_transport = None # "streamable-http" | "sse" — from .pyfun [HUB] HUB_TRANSPORT
|
| 56 |
+
_stateless = None # True = HF Spaces / horizontal scaling safe
|
| 57 |
+
|
| 58 |
+
# =============================================================================
|
| 59 |
+
# Initialization — called exclusively by app/app.py
|
| 60 |
+
# =============================================================================
|
| 61 |
+
async def initialize() -> None:
|
| 62 |
+
"""
|
| 63 |
+
Initializes the MCP instance and registers all tools.
|
| 64 |
+
Called once by app/app.py during startup sequence.
|
| 65 |
+
No fundaments passed in — fully sandboxed.
|
| 66 |
+
|
| 67 |
+
Reads HUB_TRANSPORT and HUB_STATELESS from .pyfun [HUB].
|
| 68 |
+
|
| 69 |
+
Transport modes:
|
| 70 |
+
streamable-http → get_asgi_app() returns ASGI app → app.py mounts it
|
| 71 |
+
sse → handle_request() used by Quart route in app.py
|
| 72 |
+
|
| 73 |
+
Registration order:
|
| 74 |
+
1. LLM tools → via tools.py + providers.py (key-gated)
|
| 75 |
+
2. Search tools → via tools.py + providers.py (key-gated)
|
| 76 |
+
3. System tools → always registered, no key required
|
| 77 |
+
4. DB tools → db_sync SELECT-only (app/* tables)
|
| 78 |
+
5. Persist tools → memory orchestration (works WITHOUT PostgreSQL — queue)
|
| 79 |
+
"""
|
| 80 |
+
global _mcp, _transport, _stateless
|
| 81 |
+
|
| 82 |
+
hub_cfg = app_config.get_hub()
|
| 83 |
+
_transport = hub_cfg.get("HUB_TRANSPORT", "streamable-http").lower()
|
| 84 |
+
_stateless = hub_cfg.get("HUB_STATELESS", "true").lower() == "true"
|
| 85 |
+
|
| 86 |
+
logger.info(f"MCP Hub initializing (transport: {_transport}, stateless: {_stateless})...")
|
| 87 |
+
|
| 88 |
+
try:
|
| 89 |
+
from mcp.server.fastmcp import FastMCP
|
| 90 |
+
except ImportError:
|
| 91 |
+
logger.critical("FastMCP not installed. Run: pip install mcp")
|
| 92 |
+
raise
|
| 93 |
+
|
| 94 |
+
_mcp = FastMCP(
|
| 95 |
+
name=hub_cfg.get("HUB_NAME", "Universal MCP Hub"),
|
| 96 |
+
instructions=(
|
| 97 |
+
f"{hub_cfg.get('HUB_DESCRIPTION', 'Universal MCP Hub on PyFundaments')} "
|
| 98 |
+
"Use list_active_tools to see what is currently available."
|
| 99 |
+
),
|
| 100 |
+
stateless_http=_stateless, # True = no session state, HF Spaces safe
|
| 101 |
+
)
|
| 102 |
+
|
| 103 |
+
# --- Initialize registries ---
|
| 104 |
+
providers.initialize()
|
| 105 |
+
models.initialize()
|
| 106 |
+
tools.initialize()
|
| 107 |
+
|
| 108 |
+
# --- Register MCP tools ---
|
| 109 |
+
_register_llm_tools(_mcp)
|
| 110 |
+
_register_search_tools(_mcp)
|
| 111 |
+
_register_system_tools(_mcp)
|
| 112 |
+
_register_pricing_tools(_mcp)
|
| 113 |
+
_register_db_tools(_mcp) # db_sync ready — SELECT-only, app/* tables
|
| 114 |
+
_register_persist_tools(_mcp) # memory orchestration — queue-safe, no PG required
|
| 115 |
+
|
| 116 |
+
logger.info(f"MCP Hub initialized. Transport: {_transport}")
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
# =============================================================================
|
| 120 |
+
# ASGI App — used by app/app.py for Streamable HTTP transport
|
| 121 |
+
# =============================================================================
|
| 122 |
+
def get_asgi_app():
|
| 123 |
+
"""
|
| 124 |
+
Returns the ASGI app for the configured transport.
|
| 125 |
+
Called by app/app.py AFTER initialize() — mounted as ASGI sub-app.
|
| 126 |
+
|
| 127 |
+
Streamable HTTP: mounts on /mcp — single endpoint for all MCP traffic.
|
| 128 |
+
SSE (fallback): returns sse_app() for legacy client compatibility.
|
| 129 |
+
|
| 130 |
+
NOTE: For SSE transport, app/app.py uses the Quart route + handle_request()
|
| 131 |
+
instead — get_asgi_app() is only called for streamable-http.
|
| 132 |
+
"""
|
| 133 |
+
if _mcp is None:
|
| 134 |
+
raise RuntimeError("MCP not initialized — call initialize() first.")
|
| 135 |
+
|
| 136 |
+
if _transport == "streamable-http":
|
| 137 |
+
logger.info("MCP ASGI app: Streamable HTTP → /mcp")
|
| 138 |
+
return _mcp.streamable_http_app()
|
| 139 |
+
else:
|
| 140 |
+
# SSE as ASGI app — only used if app.py mounts it directly
|
| 141 |
+
# (normally app.py uses the Quart route + handle_request() for SSE)
|
| 142 |
+
logger.info("MCP ASGI app: SSE (legacy) → /sse")
|
| 143 |
+
return _mcp.sse_app()
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
# =============================================================================
|
| 147 |
+
# Mount Prefixes — welche URL-Prefixe gehören der MCP-ASGI-App
|
| 148 |
+
# =============================================================================
|
| 149 |
+
def mount_prefixes():
|
| 150 |
+
"""URL prefixes the mounted ASGI app owns (used by app/app.py)."""
|
| 151 |
+
if _transport == "streamable-http":
|
| 152 |
+
return ("/mcp",)
|
| 153 |
+
return ("/sse", "/messages")
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
# =============================================================================
|
| 157 |
+
# Lifespan — Session-Manager für die Lebensdauer des Servers
|
| 158 |
+
# =============================================================================
|
| 159 |
+
@contextlib.asynccontextmanager
|
| 160 |
+
async def lifespan():
|
| 161 |
+
"""
|
| 162 |
+
Runs the FastMCP session manager for the lifetime of the server.
|
| 163 |
+
|
| 164 |
+
WHY: FastMCP's streamable_http_app() starts its session manager in the
|
| 165 |
+
Starlette lifespan. Because app/app.py mounts it behind a plain ASGI
|
| 166 |
+
middleware (lifespan scope goes to Quart, not to the inner app), we have
|
| 167 |
+
to run it ourselves — otherwise every request fails with
|
| 168 |
+
'Task group is not initialized'.
|
| 169 |
+
Call AFTER get_asgi_app() (it creates the session manager).
|
| 170 |
+
"""
|
| 171 |
+
if _mcp is not None and _transport == "streamable-http":
|
| 172 |
+
async with _mcp.session_manager.run():
|
| 173 |
+
yield
|
| 174 |
+
else:
|
| 175 |
+
yield
|
| 176 |
+
|
| 177 |
+
|
| 178 |
+
# =============================================================================
|
| 179 |
+
# Tool Registration — delegates all logic to tools.py
|
| 180 |
+
# =============================================================================
|
| 181 |
+
|
| 182 |
+
def _register_llm_tools(mcp) -> None:
|
| 183 |
+
"""
|
| 184 |
+
Register LLM completion tool.
|
| 185 |
+
All logic delegated to tools.py → providers.py.
|
| 186 |
+
Adding a new LLM provider = update .pyfun + providers.py. Never touch this.
|
| 187 |
+
"""
|
| 188 |
+
if not providers.list_active_llm():
|
| 189 |
+
logger.info("No active LLM providers — llm_complete tool skipped.")
|
| 190 |
+
return
|
| 191 |
+
|
| 192 |
+
@mcp.tool()
|
| 193 |
+
async def llm_complete(
|
| 194 |
+
prompt: str,
|
| 195 |
+
provider: str = None,
|
| 196 |
+
model: str = None,
|
| 197 |
+
max_tokens: int = 1024,
|
| 198 |
+
) -> str:
|
| 199 |
+
"""
|
| 200 |
+
Send a prompt to any configured LLM provider.
|
| 201 |
+
Automatically follows the fallback chain defined in .pyfun if a provider fails.
|
| 202 |
+
|
| 203 |
+
Args:
|
| 204 |
+
prompt: The input text to send to the model.
|
| 205 |
+
provider: Provider name (e.g. 'anthropic', 'gemini', 'openrouter', 'huggingface').
|
| 206 |
+
Defaults to default_provider from .pyfun [TOOL.llm_complete].
|
| 207 |
+
model: Model name override. Defaults to provider's default_model in .pyfun.
|
| 208 |
+
max_tokens: Maximum tokens in the response. Default: 1024.
|
| 209 |
+
|
| 210 |
+
Returns:
|
| 211 |
+
Model response as plain text string.
|
| 212 |
+
"""
|
| 213 |
+
return await tools.run(
|
| 214 |
+
tool_name="llm_complete",
|
| 215 |
+
prompt=prompt,
|
| 216 |
+
provider_name=provider,
|
| 217 |
+
model=model,
|
| 218 |
+
max_tokens=max_tokens,
|
| 219 |
+
)
|
| 220 |
+
|
| 221 |
+
logger.info(f"Tool registered: llm_complete (active providers: {providers.list_active_llm()})")
|
| 222 |
+
|
| 223 |
+
|
| 224 |
+
def _register_search_tools(mcp) -> None:
|
| 225 |
+
"""
|
| 226 |
+
Register web search tool.
|
| 227 |
+
All logic delegated to tools.py → providers.py.
|
| 228 |
+
Adding a new search provider = update .pyfun + providers.py. Never touch this.
|
| 229 |
+
"""
|
| 230 |
+
if not providers.list_active_search():
|
| 231 |
+
logger.info("No active search providers — web_search tool skipped.")
|
| 232 |
+
return
|
| 233 |
+
|
| 234 |
+
@mcp.tool()
|
| 235 |
+
async def web_search(
|
| 236 |
+
query: str,
|
| 237 |
+
provider: str = None,
|
| 238 |
+
max_results: int = 5,
|
| 239 |
+
) -> str:
|
| 240 |
+
"""
|
| 241 |
+
Search the web via any configured search provider.
|
| 242 |
+
Automatically follows the fallback chain defined in .pyfun if a provider fails.
|
| 243 |
+
|
| 244 |
+
Args:
|
| 245 |
+
query: Search query string.
|
| 246 |
+
provider: Provider name (e.g. 'brave', 'tavily').
|
| 247 |
+
Defaults to default_provider from .pyfun [TOOL.web_search].
|
| 248 |
+
max_results: Maximum number of results to return. Default: 5.
|
| 249 |
+
|
| 250 |
+
Returns:
|
| 251 |
+
Formatted search results as plain text string.
|
| 252 |
+
"""
|
| 253 |
+
return await tools.run(
|
| 254 |
+
tool_name="web_search",
|
| 255 |
+
prompt=query,
|
| 256 |
+
provider_name=provider,
|
| 257 |
+
max_results=max_results,
|
| 258 |
+
)
|
| 259 |
+
|
| 260 |
+
logger.info(f"Tool registered: web_search (active providers: {providers.list_active_search()})")
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
def _register_system_tools(mcp) -> None:
|
| 264 |
+
"""
|
| 265 |
+
System tools — always registered, no ENV key required.
|
| 266 |
+
Exposes hub status and model info without touching secrets.
|
| 267 |
+
"""
|
| 268 |
+
|
| 269 |
+
@mcp.tool()
|
| 270 |
+
def list_active_tools() -> Dict[str, Any]:
|
| 271 |
+
"""
|
| 272 |
+
List all active providers and registered tools.
|
| 273 |
+
Shows ENV key names only — never exposes values or secrets.
|
| 274 |
+
|
| 275 |
+
Returns:
|
| 276 |
+
Dict with hub info, active LLM providers, active search providers,
|
| 277 |
+
available tools and model names.
|
| 278 |
+
"""
|
| 279 |
+
hub = app_config.get_hub()
|
| 280 |
+
return {
|
| 281 |
+
"hub": hub.get("HUB_NAME", "Universal MCP Hub"),
|
| 282 |
+
"version": hub.get("HUB_VERSION", ""),
|
| 283 |
+
"transport": _transport,
|
| 284 |
+
"active_llm_providers": providers.list_active_llm(),
|
| 285 |
+
"active_search_providers": providers.list_active_search(),
|
| 286 |
+
"active_tools": tools.list_all(),
|
| 287 |
+
"available_models": models.list_all(),
|
| 288 |
+
"pricing_stats": pricing.stats(),
|
| 289 |
+
}
|
| 290 |
+
|
| 291 |
+
logger.info("Tool registered: list_active_tools")
|
| 292 |
+
|
| 293 |
+
@mcp.tool()
|
| 294 |
+
def health_check() -> Dict[str, str]:
|
| 295 |
+
"""
|
| 296 |
+
Health check endpoint for HuggingFace Spaces and monitoring systems.
|
| 297 |
+
|
| 298 |
+
Returns:
|
| 299 |
+
Dict with service status and active transport.
|
| 300 |
+
"""
|
| 301 |
+
return {
|
| 302 |
+
"status": "ok",
|
| 303 |
+
"service": "Universal MCP Hub",
|
| 304 |
+
"transport": _transport,
|
| 305 |
+
}
|
| 306 |
+
|
| 307 |
+
logger.info("Tool registered: health_check")
|
| 308 |
+
|
| 309 |
+
@mcp.tool()
|
| 310 |
+
async def get_model_info(model_name: str) -> Dict[str, Any]:
|
| 311 |
+
"""
|
| 312 |
+
Get limits, costs, and capabilities for a specific model.
|
| 313 |
+
Costs are MERGED: .pyfun manual values win, auto-fetched values
|
| 314 |
+
(OpenRouter / LiteLLM) fill the gaps.
|
| 315 |
+
|
| 316 |
+
Args:
|
| 317 |
+
model_name: Model name as defined in .pyfun [MODELS] (e.g. 'claude-sonnet-4-6').
|
| 318 |
+
|
| 319 |
+
Returns:
|
| 320 |
+
Dict with context size, max output tokens, rate limits, costs, and capabilities.
|
| 321 |
+
Returns pricing source info. Empty dict if model unknown.
|
| 322 |
+
"""
|
| 323 |
+
# .pyfun [MODELS] als Basis + Auto-Pricing füllt Lücken
|
| 324 |
+
merged = await pricing.get_pricing(model_name)
|
| 325 |
+
if not merged.get("provider") and not merged.get("cost_input_per_1k"):
|
| 326 |
+
return merged # komplett unbekanntes Model
|
| 327 |
+
return merged
|
| 328 |
+
|
| 329 |
+
logger.info("Tool registered: get_model_info")
|
| 330 |
+
|
| 331 |
+
|
| 332 |
+
# =============================================================================
|
| 333 |
+
# Pricing Tools — auto-fetched model costs (OpenRouter + LiteLLM)
|
| 334 |
+
# =============================================================================
|
| 335 |
+
def _register_pricing_tools(mcp) -> None:
|
| 336 |
+
"""
|
| 337 |
+
Register pricing tools.
|
| 338 |
+
Answers the question nobody wants to maintain by hand:
|
| 339 |
+
'Was kostet dieses Model?' — ohne [.MODELS] Kosten von Hand zu pflegen.
|
| 340 |
+
"""
|
| 341 |
+
|
| 342 |
+
@mcp.tool()
|
| 343 |
+
async def get_model_pricing(model_name: str) -> Dict[str, Any]:
|
| 344 |
+
"""
|
| 345 |
+
Get current pricing for a model — auto-fetched, no manual maintenance.
|
| 346 |
+
|
| 347 |
+
Merge order (highest priority first):
|
| 348 |
+
1. .pyfun [MODELS] manual values — your override always wins
|
| 349 |
+
2. OpenRouter public models API (all OR models, per-token USD)
|
| 350 |
+
3. LiteLLM community price list (Anthropic, Gemini, OpenAI, ...)
|
| 351 |
+
|
| 352 |
+
Args:
|
| 353 |
+
model_name: Model name (e.g. 'claude-opus-4-6', 'gemini-2.5-flash',
|
| 354 |
+
'meta-llama/llama-3.3-70b-instruct').
|
| 355 |
+
|
| 356 |
+
Returns:
|
| 357 |
+
Dict with cost_input_per_1k, cost_output_per_1k (USD),
|
| 358 |
+
context_tokens, max_output_tokens, source, pricing_note.
|
| 359 |
+
"""
|
| 360 |
+
return await pricing.get_pricing(model_name)
|
| 361 |
+
|
| 362 |
+
logger.info("Tool registered: get_model_pricing")
|
| 363 |
+
|
| 364 |
+
@mcp.tool()
|
| 365 |
+
async def refresh_pricing() -> Dict[str, Any]:
|
| 366 |
+
"""
|
| 367 |
+
Force a pricing refresh from OpenRouter + LiteLLM.
|
| 368 |
+
Updates the db_sync cache (TTL-gated, default 24h).
|
| 369 |
+
|
| 370 |
+
Returns:
|
| 371 |
+
Dict with fetched_at, per-source model counts, and total cached models.
|
| 372 |
+
"""
|
| 373 |
+
return await pricing.refresh()
|
| 374 |
+
|
| 375 |
+
logger.info("Tool registered: refresh_pricing")
|
| 376 |
+
|
| 377 |
+
|
| 378 |
+
# =============================================================================
|
| 379 |
+
# DB Tools — db_sync.py is ready, enabled
|
| 380 |
+
# =============================================================================
|
| 381 |
+
def _register_db_tools(mcp) -> None:
|
| 382 |
+
"""
|
| 383 |
+
Register internal SQLite query tool.
|
| 384 |
+
Uses db_sync.py (app/* internal SQLite) — NOT postgresql.py (Guardian-only)!
|
| 385 |
+
|
| 386 |
+
SECURITY: Only SELECT queries are permitted.
|
| 387 |
+
Enforced at application level in db_sync.query() — not just in docs.
|
| 388 |
+
Tables accessible: hub_state, tool_cache (app/* only)
|
| 389 |
+
Tables blocked: users, sessions (Guardian-only, different owner)
|
| 390 |
+
|
| 391 |
+
Requirements (all met):
|
| 392 |
+
1. This function enabled
|
| 393 |
+
2. _register_db_tools(_mcp) called in initialize()
|
| 394 |
+
3. db_sync.initialize() runs in app/app.py BEFORE mcp.initialize()
|
| 395 |
+
"""
|
| 396 |
+
from . import db_sync
|
| 397 |
+
|
| 398 |
+
@mcp.tool()
|
| 399 |
+
async def db_query(sql: str) -> list:
|
| 400 |
+
"""
|
| 401 |
+
Execute a read-only SELECT query on the internal hub state database.
|
| 402 |
+
|
| 403 |
+
Only SELECT statements are permitted — all write operations are blocked
|
| 404 |
+
at the db_sync layer (not just by convention).
|
| 405 |
+
|
| 406 |
+
Accessible tables:
|
| 407 |
+
hub_state — current hub runtime state (tool status, uptime, etc.)
|
| 408 |
+
tool_cache — cached tool results for repeated queries
|
| 409 |
+
|
| 410 |
+
NOT accessible (Guardian-only):
|
| 411 |
+
users — managed by fundaments/user_handler.py
|
| 412 |
+
sessions — managed by fundaments/user_handler.py
|
| 413 |
+
|
| 414 |
+
Args:
|
| 415 |
+
sql: SQL SELECT statement. Example: "SELECT * FROM hub_state LIMIT 10"
|
| 416 |
+
|
| 417 |
+
Returns:
|
| 418 |
+
List of result rows as dicts. Empty list if no results.
|
| 419 |
+
|
| 420 |
+
Raises:
|
| 421 |
+
ValueError: If statement is not a SELECT query.
|
| 422 |
+
RuntimeError: If db_sync is not initialized.
|
| 423 |
+
"""
|
| 424 |
+
return await db_sync.query(sql)
|
| 425 |
+
|
| 426 |
+
logger.info("Tool registered: db_query (SQLite SELECT-only, app/* tables)")
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
# =============================================================================
|
| 430 |
+
# Persist Tools — Gedächtnis-Orchestrierung (SQLite-Queue → PostgreSQL)
|
| 431 |
+
# =============================================================================
|
| 432 |
+
def _register_persist_tools(mcp) -> None:
|
| 433 |
+
"""
|
| 434 |
+
Register memory-orchestration tools.
|
| 435 |
+
|
| 436 |
+
persist_result works WITHOUT PostgreSQL: entries queue restart-safe in
|
| 437 |
+
the SQLite persist_queue (db_sync) and flush automatically once the
|
| 438 |
+
Guardian-injected PostgreSQL bridge is active. No key = queued, not crash.
|
| 439 |
+
"""
|
| 440 |
+
if "persist_result" not in tools.list_all():
|
| 441 |
+
logger.info("persist_result not active in .pyfun — skipped.")
|
| 442 |
+
return
|
| 443 |
+
|
| 444 |
+
@mcp.tool()
|
| 445 |
+
async def persist_result(
|
| 446 |
+
prompt: str = None,
|
| 447 |
+
source: str = None,
|
| 448 |
+
model: str = None,
|
| 449 |
+
score: float = None,
|
| 450 |
+
) -> Dict[str, Any]:
|
| 451 |
+
"""
|
| 452 |
+
Persist a result to long-term memory (PostgreSQL).
|
| 453 |
+
|
| 454 |
+
Restart-safe: without DATABASE_URL the entry queues in SQLite and
|
| 455 |
+
flushes automatically once PostgreSQL is available. Never crashes.
|
| 456 |
+
|
| 457 |
+
Args:
|
| 458 |
+
prompt: The text/result to persist (or hub_state key via .pyfun state_read_key).
|
| 459 |
+
source: Which tool/component produced this entry (optional).
|
| 460 |
+
model: Which model produced this entry (optional).
|
| 461 |
+
score: Optional quality score — reserved slot, set by your own filter.
|
| 462 |
+
|
| 463 |
+
Returns:
|
| 464 |
+
Dict with status ('persisted' | 'queued'), target_table, queue_depth.
|
| 465 |
+
"""
|
| 466 |
+
return await tools.run(
|
| 467 |
+
tool_name="persist_result",
|
| 468 |
+
prompt=prompt,
|
| 469 |
+
source=source,
|
| 470 |
+
model=model,
|
| 471 |
+
score=score,
|
| 472 |
+
)
|
| 473 |
+
|
| 474 |
+
logger.info("Tool registered: persist_result (queue-safe memory orchestration)")
|
| 475 |
+
|
| 476 |
+
@mcp.tool()
|
| 477 |
+
async def persist_status() -> Dict[str, Any]:
|
| 478 |
+
"""
|
| 479 |
+
Show memory orchestration status: queue depth, oldest entry,
|
| 480 |
+
failed attempts, and whether the PostgreSQL bridge is active.
|
| 481 |
+
|
| 482 |
+
Returns:
|
| 483 |
+
Dict with queue stats — observability for the memory pipeline.
|
| 484 |
+
"""
|
| 485 |
+
from . import db_sync
|
| 486 |
+
return await db_sync.persist_queue_stats()
|
| 487 |
+
|
| 488 |
+
logger.info("Tool registered: persist_status")
|
| 489 |
+
|
| 490 |
+
|
| 491 |
+
# =============================================================================
|
| 492 |
+
# Direct execution guard
|
| 493 |
+
# =============================================================================
|
| 494 |
+
if __name__ == '__main__':
|
| 495 |
+
print("WARNING: Run via main.py → app.py, not directly.")
|
app/models.py
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# # app/models.py
|
| 3 |
+
# Universal MCP Hub (Sandboxed) - based on PyFundaments Architecture
|
| 4 |
+
# Copyright 2026 - Volkan Kücükbudak
|
| 5 |
+
# Apache License V. 2 + ESOL 1.1
|
| 6 |
+
# Repo: https://github.com/VolkanSah/Universal-MCP-Hub-sandboxed
|
| 7 |
+
# =============================================================================
|
| 8 |
+
# ARCHITECTURE NOTE:
|
| 9 |
+
# This file lives exclusively in app/ and is ONLY started by app/app.py.
|
| 10 |
+
# NO direct access to fundaments/*, .env, or Guardian (main.py).
|
| 11 |
+
# All config comes from app/.pyfun via app/config.py.
|
| 12 |
+
#
|
| 13 |
+
#
|
| 14 |
+
# TOOL REGISTRATION PRINCIPLE:
|
| 15 |
+
# Tools are registered via models.py and models.py .
|
| 16 |
+
# No key = no provider = no tool = no crash.
|
| 17 |
+
# Adding a new provider = update .pyfun + providers.py only. Never touch mcp.py!
|
| 18 |
+
#
|
| 19 |
+
# DEPENDENCY CHAIN (app/* only, no fundaments!):
|
| 20 |
+
# config.py → parses app/.pyfun — single source of truth
|
| 21 |
+
# providers.py → LLM + Search provider registry + fallback chain
|
| 22 |
+
# models.py → model limits, costs, capabilities from .pyfun [MODELS]
|
| 23 |
+
# db_sync.py → internal SQLite IPC (app/* state) — NOT postgresql.py!
|
| 24 |
+
# mcp.py → registers tools only, delegates all logic to providers/*
|
| 25 |
+
# =============================================================================
|
| 26 |
+
# app/models.py
|
| 27 |
+
from . import config
|
| 28 |
+
import logging
|
| 29 |
+
|
| 30 |
+
logger = logging.getLogger("models")
|
| 31 |
+
|
| 32 |
+
_registry: dict = {}
|
| 33 |
+
|
| 34 |
+
def initialize() -> None:
|
| 35 |
+
"""Build model registry from .pyfun [MODELS]"""
|
| 36 |
+
global _registry
|
| 37 |
+
_registry = config.get_models()
|
| 38 |
+
logger.info(f"Models loaded: {list(_registry.keys())}")
|
| 39 |
+
|
| 40 |
+
|
| 41 |
+
def get(model_name: str) -> dict:
|
| 42 |
+
"""Get model config by name. Falls back to raw .pyfun if not initialized yet."""
|
| 43 |
+
if _registry:
|
| 44 |
+
return _registry.get(model_name, {})
|
| 45 |
+
return config.get_models().get(model_name, {})
|
| 46 |
+
|
| 47 |
+
|
| 48 |
+
def get_limit(model_name: str, key: str, default=None):
|
| 49 |
+
"""Get specific limit for a model."""
|
| 50 |
+
return _registry.get(model_name, {}).get(key, default)
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
def for_provider(provider_name: str) -> dict:
|
| 54 |
+
"""Get all models for a provider."""
|
| 55 |
+
return config.get_models_for_provider(provider_name)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
def max_tokens(model_name: str) -> int:
|
| 59 |
+
return int(get_limit(model_name, "max_output_tokens", "1024"))
|
| 60 |
+
|
| 61 |
+
|
| 62 |
+
def context_size(model_name: str) -> int:
|
| 63 |
+
return int(get_limit(model_name, "context_tokens", "4096"))
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
def cost_input(model_name: str) -> float:
|
| 67 |
+
return float(get_limit(model_name, "cost_input_per_1k", "0"))
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
def cost_output(model_name: str) -> float:
|
| 71 |
+
return float(get_limit(model_name, "cost_output_per_1k", "0"))
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def list_all() -> list:
|
| 75 |
+
return list(_registry.keys())
|
app/pricing.py
ADDED
|
@@ -0,0 +1,449 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# app/pricing.py
|
| 3 |
+
# Auto Model Pricing — weil niemand Lust hat, Kosten von Hand zu pflegen.
|
| 4 |
+
# Universal MCP Hub (Sandboxed) - based on PyFundaments Architecture
|
| 5 |
+
# Copyright 2026 - Volkan Kücükbudak
|
| 6 |
+
# Apache License V. 2 + ESOL 1.1
|
| 7 |
+
# =============================================================================
|
| 8 |
+
# ARCHITECTURE NOTE:
|
| 9 |
+
# This file lives exclusively in app/ and is ONLY started by app/app.py.
|
| 10 |
+
# NO direct access to fundaments/*, .env, or Guardian (main.py).
|
| 11 |
+
# All config comes from app/.pyfun [PRICING] via app/config.py.
|
| 12 |
+
#
|
| 13 |
+
# WHY THIS EXISTS:
|
| 14 |
+
# Not every provider has a pricing API (Anthropic & Google: none).
|
| 15 |
+
# Manually maintaining cost_input_per_1k in [MODELS] sucks. Solution:
|
| 16 |
+
# two free public sources cover almost everything, .pyfun stays the override.
|
| 17 |
+
#
|
| 18 |
+
# SOURCES (priority: .pyfun manual > OpenRouter > LiteLLM):
|
| 19 |
+
# 1. .pyfun [MODELS] → your values ALWAYS win (manual override)
|
| 20 |
+
# 2. OpenRouter /api/v1/models → public, no key. Pricing per token for all
|
| 21 |
+
# OR models + context_length + max_completion_tokens
|
| 22 |
+
# 3. LiteLLM community JSON → github.com/BerriAI/litellm main branch.
|
| 23 |
+
# input_cost_per_token / output_cost_per_token
|
| 24 |
+
# for Anthropic, Gemini, OpenAI, Mistral, ...
|
| 25 |
+
#
|
| 26 |
+
# PRINCIPLE (same as everything here):
|
| 27 |
+
# No network = no crash. No match = costs stay as configured.
|
| 28 |
+
# Server always starts — pricing refresh runs in the BACKGROUND.
|
| 29 |
+
#
|
| 30 |
+
# CACHING:
|
| 31 |
+
# Fetched data lands in db_sync hub_state ("pricing.cache") with TTL
|
| 32 |
+
# from .pyfun (default 24h). Restart within TTL = no external call.
|
| 33 |
+
#
|
| 34 |
+
# MCP TOOLS (registered in mcp.py):
|
| 35 |
+
# get_model_pricing(model_name) → merged cost info for one model
|
| 36 |
+
# refresh_pricing() → force refresh, returns source counts
|
| 37 |
+
# =============================================================================
|
| 38 |
+
import asyncio
|
| 39 |
+
import logging
|
| 40 |
+
from typing import Any, Dict, Optional
|
| 41 |
+
|
| 42 |
+
import httpx
|
| 43 |
+
|
| 44 |
+
from . import config
|
| 45 |
+
from . import db_sync
|
| 46 |
+
|
| 47 |
+
logger = logging.getLogger("pricing")
|
| 48 |
+
|
| 49 |
+
# =============================================================================
|
| 50 |
+
# Internal State
|
| 51 |
+
# =============================================================================
|
| 52 |
+
_cache: Dict[str, Dict[str, Any]] = {} # normalized_name → {cost_in_1k, cost_out_1k, context, max_out, source}
|
| 53 |
+
_meta: Dict[str, Any] = {} # {"fetched_at": iso, "sources": {...}, "model_count": int}
|
| 54 |
+
_task: Optional[asyncio.Task] = None # background refresh task
|
| 55 |
+
_initialized: bool = False
|
| 56 |
+
|
| 57 |
+
# Loose index: alnum-only keys for dot/dash-insensitive matching.
|
| 58 |
+
# Why: OpenRouter writes 'claude-opus-4.6', LiteLLM 'claude-opus-4-6',
|
| 59 |
+
# your .pyfun 'claude-opus-4-6'. All should find each other.
|
| 60 |
+
_loose_index: Dict[str, str] = {} # loose_key → normalized cache key
|
| 61 |
+
_loose_dirty: bool = True
|
| 62 |
+
|
| 63 |
+
_HUB_STATE_KEY = "pricing.cache"
|
| 64 |
+
|
| 65 |
+
# Known vendor prefixes used by OpenRouter / LiteLLM IDs
|
| 66 |
+
# ("anthropic/claude-...", "gemini/gemini-...", "meta-llama/llama-...", ...)
|
| 67 |
+
_VENDOR_PREFIXES = (
|
| 68 |
+
"anthropic/", "google/", "openai/", "meta-llama/", "mistralai/", "mistral/",
|
| 69 |
+
"qwen/", "nvidia/", "microsoft/", "deepseek/", "x-ai/", "xai/", "perplexity/",
|
| 70 |
+
"cohere/", "gryphe/", "thedrummer/", "undi95/", "cognitivecomputations/",
|
| 71 |
+
"gemini/", "groq/", "together_ai/", "together/", "openrouter/", "fireworks_ai/",
|
| 72 |
+
"azure/", "vertex_ai/", "bedrock/", "cloudflare/", "voyage/", "sambanova/",
|
| 73 |
+
"ai21/", "huggingface/", "ollama/", "ollama_chat/",
|
| 74 |
+
)
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
# =============================================================================
|
| 78 |
+
# Normalization — model IDs differ wildly between sources
|
| 79 |
+
# =============================================================================
|
| 80 |
+
def _normalize(name: str) -> str:
|
| 81 |
+
"""
|
| 82 |
+
Normalize a model name for matching across sources.
|
| 83 |
+
'anthropic/claude-haiku-4.5' → 'claude-haiku-4.5'
|
| 84 |
+
'Claude-Haiku-4.5' → 'claude-haiku-4.5'
|
| 85 |
+
Dots vs dashes are kept — providers are inconsistent there by design.
|
| 86 |
+
"""
|
| 87 |
+
n = (name or "").strip().lower()
|
| 88 |
+
for prefix in _VENDOR_PREFIXES:
|
| 89 |
+
if n.startswith(prefix):
|
| 90 |
+
n = n[len(prefix):]
|
| 91 |
+
break
|
| 92 |
+
return n
|
| 93 |
+
|
| 94 |
+
|
| 95 |
+
def _safe_int(value: Any) -> Optional[int]:
|
| 96 |
+
"""int() that returns None instead of raising (LiteLLM ships doc samples
|
| 97 |
+
like sample_spec.max_input_tokens = 'max input tokens, if the provider specifies it')."""
|
| 98 |
+
try:
|
| 99 |
+
return int(value)
|
| 100 |
+
except (TypeError, ValueError):
|
| 101 |
+
return None
|
| 102 |
+
|
| 103 |
+
|
| 104 |
+
def _per_token_to_per_1k(cost: Any) -> Optional[float]:
|
| 105 |
+
"""'0.0000015' per token → 0.0015 per 1k tokens. Returns None on garbage."""
|
| 106 |
+
try:
|
| 107 |
+
val = float(cost)
|
| 108 |
+
except (TypeError, ValueError):
|
| 109 |
+
return None
|
| 110 |
+
if val < 0: # negative = OpenRouter's "we pay you" marker → treat as 0
|
| 111 |
+
val = 0.0
|
| 112 |
+
return round(val * 1000, 6)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def _loose(name: str) -> str:
|
| 116 |
+
"""Strip everything except a-z0-9 — kills dot/dash/underscore differences."""
|
| 117 |
+
return "".join(ch for ch in name if ch.isalnum())
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _rebuild_loose_index() -> None:
|
| 121 |
+
"""Rebuild the alnum-only lookup index after a cache change."""
|
| 122 |
+
global _loose_index, _loose_dirty
|
| 123 |
+
_loose_index = {_loose(k): k for k in _cache}
|
| 124 |
+
_loose_dirty = False
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _lookup(model_name: str) -> Optional[Dict[str, Any]]:
|
| 128 |
+
"""
|
| 129 |
+
Cache lookup with graceful fallback:
|
| 130 |
+
1. exact normalized match ('claude-opus-4-6')
|
| 131 |
+
2. loose alnum match ('claude-opus-4.6' ≈ 'claude-opus-4-6')
|
| 132 |
+
"""
|
| 133 |
+
norm = _normalize(model_name)
|
| 134 |
+
hit = _cache.get(norm)
|
| 135 |
+
if hit:
|
| 136 |
+
return hit
|
| 137 |
+
if _loose_dirty:
|
| 138 |
+
_rebuild_loose_index()
|
| 139 |
+
loose_key = _loose_index.get(_loose(norm))
|
| 140 |
+
if loose_key:
|
| 141 |
+
return _cache.get(loose_key)
|
| 142 |
+
return None
|
| 143 |
+
|
| 144 |
+
|
| 145 |
+
# =============================================================================
|
| 146 |
+
# Source Fetchers — each returns {normalized_name: entry}
|
| 147 |
+
# =============================================================================
|
| 148 |
+
async def _fetch_openrouter(url: str) -> Dict[str, Dict[str, Any]]:
|
| 149 |
+
"""
|
| 150 |
+
OpenRouter public models endpoint — no key required.
|
| 151 |
+
https://openrouter.ai/api/v1/models → data[].{id, pricing{prompt,completion}, context_length}
|
| 152 |
+
"""
|
| 153 |
+
out: Dict[str, Dict[str, Any]] = {}
|
| 154 |
+
async with httpx.AsyncClient() as client:
|
| 155 |
+
r = await client.get(url, timeout=30)
|
| 156 |
+
r.raise_for_status()
|
| 157 |
+
payload = r.json()
|
| 158 |
+
|
| 159 |
+
for m in payload.get("data", []):
|
| 160 |
+
model_id = m.get("id", "")
|
| 161 |
+
if not model_id:
|
| 162 |
+
continue
|
| 163 |
+
pricing = m.get("pricing", {}) or {}
|
| 164 |
+
entry: Dict[str, Any] = {"source": "openrouter"}
|
| 165 |
+
|
| 166 |
+
cost_in = _per_token_to_per_1k(pricing.get("prompt"))
|
| 167 |
+
cost_out = _per_token_to_per_1k(pricing.get("completion"))
|
| 168 |
+
if cost_in is not None:
|
| 169 |
+
entry["cost_input_per_1k"] = cost_in
|
| 170 |
+
if cost_out is not None:
|
| 171 |
+
entry["cost_output_per_1k"] = cost_out
|
| 172 |
+
if m.get("context_length"):
|
| 173 |
+
ctx = _safe_int(m["context_length"])
|
| 174 |
+
if ctx:
|
| 175 |
+
entry["context_tokens"] = ctx
|
| 176 |
+
|
| 177 |
+
# max_completion_tokens is nested under top_provider in OR payloads
|
| 178 |
+
max_out = _safe_int((m.get("top_provider") or {}).get("max_completion_tokens"))
|
| 179 |
+
if max_out:
|
| 180 |
+
entry["max_output_tokens"] = max_out
|
| 181 |
+
|
| 182 |
+
out[_normalize(model_id)] = entry
|
| 183 |
+
return out
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
async def _fetch_litellm(url: str) -> Dict[str, Dict[str, Any]]:
|
| 187 |
+
"""
|
| 188 |
+
LiteLLM community price list — no key required.
|
| 189 |
+
github.com/BerriAI/litellm → model_prices_and_context_window.json
|
| 190 |
+
Keys like 'claude-sonnet-4-5', 'gemini/gemini-2.5-flash', 'gpt-4o'.
|
| 191 |
+
"""
|
| 192 |
+
out: Dict[str, Dict[str, Any]] = {}
|
| 193 |
+
async with httpx.AsyncClient() as client:
|
| 194 |
+
r = await client.get(url, timeout=30)
|
| 195 |
+
r.raise_for_status()
|
| 196 |
+
payload = r.json()
|
| 197 |
+
|
| 198 |
+
for model_id, info in payload.items():
|
| 199 |
+
if not isinstance(info, dict):
|
| 200 |
+
continue # skip metadata keys like "additional_vendor_names"
|
| 201 |
+
entry: Dict[str, Any] = {"source": "litellm"}
|
| 202 |
+
|
| 203 |
+
cost_in = _per_token_to_per_1k(info.get("input_cost_per_token"))
|
| 204 |
+
cost_out = _per_token_to_per_1k(info.get("output_cost_per_token"))
|
| 205 |
+
if cost_in is not None:
|
| 206 |
+
entry["cost_input_per_1k"] = cost_in
|
| 207 |
+
if cost_out is not None:
|
| 208 |
+
entry["cost_output_per_1k"] = cost_out
|
| 209 |
+
if info.get("max_input_tokens"):
|
| 210 |
+
ctx = _safe_int(info["max_input_tokens"])
|
| 211 |
+
if ctx:
|
| 212 |
+
entry["context_tokens"] = ctx
|
| 213 |
+
if info.get("max_output_tokens"):
|
| 214 |
+
max_out = _safe_int(info["max_output_tokens"])
|
| 215 |
+
if max_out:
|
| 216 |
+
entry["max_output_tokens"] = max_out
|
| 217 |
+
|
| 218 |
+
out[_normalize(model_id)] = entry
|
| 219 |
+
return out
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
# =============================================================================
|
| 223 |
+
# Cache — db_sync hub_state, TTL-gated
|
| 224 |
+
# =============================================================================
|
| 225 |
+
def _cache_is_fresh() -> bool:
|
| 226 |
+
if not _cache or not _meta.get("fetched_at"):
|
| 227 |
+
return False
|
| 228 |
+
from datetime import datetime, timezone
|
| 229 |
+
try:
|
| 230 |
+
fetched = datetime.fromisoformat(_meta["fetched_at"])
|
| 231 |
+
except (ValueError, TypeError):
|
| 232 |
+
return False
|
| 233 |
+
ttl_hours = float(config.get_pricing().get("PRICING_TTL_HOURS", "24"))
|
| 234 |
+
age = datetime.now(timezone.utc) - fetched
|
| 235 |
+
return age.total_seconds() < ttl_hours * 3600
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
async def _load_cache_from_db() -> None:
|
| 239 |
+
"""Read pricing cache from db_sync — never raises."""
|
| 240 |
+
global _cache, _meta, _loose_dirty
|
| 241 |
+
try:
|
| 242 |
+
stored = await db_sync.read(_HUB_STATE_KEY, default=None)
|
| 243 |
+
if isinstance(stored, dict) and stored.get("models"):
|
| 244 |
+
_cache = stored["models"]
|
| 245 |
+
_meta = stored.get("meta", {})
|
| 246 |
+
_loose_dirty = True
|
| 247 |
+
logger.info(
|
| 248 |
+
f"Pricing cache loaded from db_sync: {len(_cache)} models "
|
| 249 |
+
f"(fetched_at: {_meta.get('fetched_at', 'unknown')})"
|
| 250 |
+
)
|
| 251 |
+
except Exception as e:
|
| 252 |
+
logger.warning(f"Pricing cache load failed (continuing without): {type(e).__name__}")
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
async def _save_cache_to_db() -> None:
|
| 256 |
+
"""Write pricing cache to db_sync — never raises."""
|
| 257 |
+
try:
|
| 258 |
+
await db_sync.write(_HUB_STATE_KEY, {"models": _cache, "meta": _meta})
|
| 259 |
+
except Exception as e:
|
| 260 |
+
logger.warning(f"Pricing cache save failed: {type(e).__name__}")
|
| 261 |
+
|
| 262 |
+
|
| 263 |
+
# =============================================================================
|
| 264 |
+
# Refresh — fetch sources, merge, cache
|
| 265 |
+
# =============================================================================
|
| 266 |
+
async def refresh() -> Dict[str, Any]:
|
| 267 |
+
"""
|
| 268 |
+
Fetch all pricing sources, rebuild the auto-pricing cache.
|
| 269 |
+
Never raises — a failing source is just skipped (no fetch = no crash).
|
| 270 |
+
"""
|
| 271 |
+
global _cache, _meta, _loose_dirty
|
| 272 |
+
from datetime import datetime, timezone
|
| 273 |
+
|
| 274 |
+
cfg = config.get_pricing()
|
| 275 |
+
or_url = cfg.get("OPENROUTER_MODELS_URL", "https://openrouter.ai/api/v1/models")
|
| 276 |
+
llm_url = cfg.get(
|
| 277 |
+
"LITELLM_PRICES_URL",
|
| 278 |
+
"https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json",
|
| 279 |
+
)
|
| 280 |
+
|
| 281 |
+
merged: Dict[str, Dict[str, Any]] = {}
|
| 282 |
+
sources: Dict[str, int] = {}
|
| 283 |
+
|
| 284 |
+
# --- Source 1: OpenRouter ---
|
| 285 |
+
try:
|
| 286 |
+
or_models = await _fetch_openrouter(or_url)
|
| 287 |
+
merged.update(or_models)
|
| 288 |
+
sources["openrouter"] = len(or_models)
|
| 289 |
+
logger.info(f"Pricing source OpenRouter: {len(or_models)} models.")
|
| 290 |
+
except Exception as e:
|
| 291 |
+
sources["openrouter"] = 0
|
| 292 |
+
logger.warning(f"Pricing source OpenRouter failed: {type(e).__name__} — skipped.")
|
| 293 |
+
|
| 294 |
+
# --- Source 2: LiteLLM (fills gaps — never overwrites OpenRouter) ---
|
| 295 |
+
try:
|
| 296 |
+
llm_models = await _fetch_litellm(llm_url)
|
| 297 |
+
for name, entry in llm_models.items():
|
| 298 |
+
if name not in merged:
|
| 299 |
+
merged[name] = entry
|
| 300 |
+
sources["litellm"] = len(llm_models)
|
| 301 |
+
logger.info(f"Pricing source LiteLLM: {len(llm_models)} models.")
|
| 302 |
+
except Exception as e:
|
| 303 |
+
sources["litellm"] = 0
|
| 304 |
+
logger.warning(f"Pricing source LiteLLM failed: {type(e).__name__} — skipped.")
|
| 305 |
+
|
| 306 |
+
if merged:
|
| 307 |
+
_cache = merged
|
| 308 |
+
_meta = {
|
| 309 |
+
"fetched_at": datetime.now(timezone.utc).isoformat(),
|
| 310 |
+
"sources": sources,
|
| 311 |
+
"model_count": len(merged),
|
| 312 |
+
}
|
| 313 |
+
_loose_dirty = True
|
| 314 |
+
await _save_cache_to_db()
|
| 315 |
+
|
| 316 |
+
return {
|
| 317 |
+
"fetched_at": _meta.get("fetched_at"),
|
| 318 |
+
"sources": sources,
|
| 319 |
+
"models_cached": len(_cache),
|
| 320 |
+
}
|
| 321 |
+
|
| 322 |
+
|
| 323 |
+
# =============================================================================
|
| 324 |
+
# Initialization — called by app/app.py, NON-BLOCKING
|
| 325 |
+
# =============================================================================
|
| 326 |
+
async def initialize() -> None:
|
| 327 |
+
"""
|
| 328 |
+
Prepare auto-pricing. Loads the db_sync cache if fresh; otherwise kicks
|
| 329 |
+
off a BACKGROUND refresh (asyncio.create_task) — the server never waits
|
| 330 |
+
for external pricing APIs. No network = no crash.
|
| 331 |
+
"""
|
| 332 |
+
global _initialized, _task
|
| 333 |
+
if _initialized:
|
| 334 |
+
return
|
| 335 |
+
|
| 336 |
+
cfg = config.get_pricing()
|
| 337 |
+
enabled = cfg.get("PRICING_ENABLED", "true").lower() == "true"
|
| 338 |
+
|
| 339 |
+
if not enabled:
|
| 340 |
+
logger.info("Auto-pricing disabled in .pyfun [PRICING] — .pyfun [MODELS] values only.")
|
| 341 |
+
_initialized = True
|
| 342 |
+
return
|
| 343 |
+
|
| 344 |
+
await _load_cache_from_db()
|
| 345 |
+
|
| 346 |
+
if _cache_is_fresh():
|
| 347 |
+
logger.info("Auto-pricing cache fresh — no external fetch needed.")
|
| 348 |
+
else:
|
| 349 |
+
logger.info("Auto-pricing cache stale/empty — background refresh started.")
|
| 350 |
+
_task = asyncio.create_task(_background_refresh())
|
| 351 |
+
|
| 352 |
+
_initialized = True
|
| 353 |
+
|
| 354 |
+
|
| 355 |
+
async def _background_refresh() -> None:
|
| 356 |
+
"""Runs refresh() and swallows everything — background task must never crash the app."""
|
| 357 |
+
try:
|
| 358 |
+
await refresh()
|
| 359 |
+
except Exception as e:
|
| 360 |
+
logger.warning(f"Pricing background refresh failed (non-fatal): {type(e).__name__}")
|
| 361 |
+
|
| 362 |
+
|
| 363 |
+
# =============================================================================
|
| 364 |
+
# Public API — merged view: .pyfun manual values WIN over auto-fetched
|
| 365 |
+
# =============================================================================
|
| 366 |
+
async def get_pricing(model_name: str) -> Dict[str, Any]:
|
| 367 |
+
"""
|
| 368 |
+
Returns merged pricing info for one model.
|
| 369 |
+
|
| 370 |
+
Merge order (highest priority first):
|
| 371 |
+
1. .pyfun [MODELS.<model>] → your manual values always win
|
| 372 |
+
2. auto-fetched cache → OpenRouter / LiteLLM
|
| 373 |
+
3. nothing → costs omitted, source: "unknown"
|
| 374 |
+
|
| 375 |
+
Returns dict with: model, provider (if known from .pyfun),
|
| 376 |
+
cost_input_per_1k, cost_output_per_1k (USD), context_tokens,
|
| 377 |
+
max_output_tokens, source ("pyfun" | "openrouter" | "litellm" | "unknown"),
|
| 378 |
+
pricing_note.
|
| 379 |
+
"""
|
| 380 |
+
from . import models as models_mod
|
| 381 |
+
|
| 382 |
+
result: Dict[str, Any] = {"model": model_name}
|
| 383 |
+
notes = []
|
| 384 |
+
|
| 385 |
+
# --- Layer 1: .pyfun manual config (models.py reads [MODELS]) ---
|
| 386 |
+
manual = models_mod.get(model_name)
|
| 387 |
+
if manual:
|
| 388 |
+
result["provider"] = manual.get("provider", "")
|
| 389 |
+
for key in ("cost_input_per_1k", "cost_output_per_1k",
|
| 390 |
+
"context_tokens", "max_output_tokens",
|
| 391 |
+
"requests_per_min", "requests_per_day", "capabilities"):
|
| 392 |
+
if manual.get(key):
|
| 393 |
+
value = manual[key]
|
| 394 |
+
if key.startswith("cost_"):
|
| 395 |
+
# .pyfun-Werte sind Strings ("0.015") — Auto-Werte sind float.
|
| 396 |
+
# Normalisieren, damit die API immer Zahlen liefert.
|
| 397 |
+
try:
|
| 398 |
+
value = round(float(value), 6)
|
| 399 |
+
except (TypeError, ValueError):
|
| 400 |
+
pass
|
| 401 |
+
result[key] = value
|
| 402 |
+
if manual.get("cost_input_per_1k") or manual.get("cost_output_per_1k"):
|
| 403 |
+
result["source"] = "pyfun"
|
| 404 |
+
notes.append(".pyfun manual values (override)")
|
| 405 |
+
else:
|
| 406 |
+
notes.append(".pyfun entry has no costs — auto values used")
|
| 407 |
+
|
| 408 |
+
# --- Layer 2: auto-fetched cache fills the gaps ---
|
| 409 |
+
auto = _lookup(model_name)
|
| 410 |
+
if auto:
|
| 411 |
+
for key in ("cost_input_per_1k", "cost_output_per_1k",
|
| 412 |
+
"context_tokens", "max_output_tokens"):
|
| 413 |
+
if key not in result and auto.get(key) is not None:
|
| 414 |
+
result[key] = auto[key]
|
| 415 |
+
if "source" not in result:
|
| 416 |
+
result["source"] = auto.get("source", "auto")
|
| 417 |
+
notes.append(f"auto-fetched from {auto.get('source', 'unknown')}")
|
| 418 |
+
elif "source" not in result:
|
| 419 |
+
result["source"] = "unknown"
|
| 420 |
+
notes.append("no auto source matched this model name")
|
| 421 |
+
|
| 422 |
+
# --- Stale hint ---
|
| 423 |
+
if _cache and not _cache_is_fresh():
|
| 424 |
+
notes.append("cache stale — refresh_pricing() to update")
|
| 425 |
+
|
| 426 |
+
result["pricing_note"] = "; ".join(notes) if notes else "ok"
|
| 427 |
+
return result
|
| 428 |
+
|
| 429 |
+
|
| 430 |
+
def is_ready() -> bool:
|
| 431 |
+
"""True if pricing cache holds at least one model."""
|
| 432 |
+
return bool(_cache)
|
| 433 |
+
|
| 434 |
+
|
| 435 |
+
def stats() -> Dict[str, Any]:
|
| 436 |
+
"""Cache stats for list_active_tools / debugging."""
|
| 437 |
+
return {
|
| 438 |
+
"models_cached": len(_cache),
|
| 439 |
+
"fetched_at": _meta.get("fetched_at"),
|
| 440 |
+
"sources": _meta.get("sources", {}),
|
| 441 |
+
"fresh": _cache_is_fresh(),
|
| 442 |
+
}
|
| 443 |
+
|
| 444 |
+
|
| 445 |
+
# =============================================================================
|
| 446 |
+
# Direct execution guard
|
| 447 |
+
# =============================================================================
|
| 448 |
+
if __name__ == "__main__":
|
| 449 |
+
print("WARNING: Run via main.py → app.py, not directly.")
|
app/providers.py
ADDED
|
@@ -0,0 +1,825 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# app/providers.py
|
| 3 |
+
# 09.03.2026 | updated 23.03.2026
|
| 4 |
+
# LLM + Search Provider Registry + Fallback Chain
|
| 5 |
+
# Universal MCP Hub (Sandboxed) - based on PyFundaments Architecture
|
| 6 |
+
# Copyright 2026 - Volkan Kücükbudak
|
| 7 |
+
# Apache License V. 2 + ESOL 1.1
|
| 8 |
+
# Repo: https://github.com/VolkanSah/Universal-MCP-Hub-sandboxed
|
| 9 |
+
# =============================================================================
|
| 10 |
+
# ARCHITECTURE NOTE:
|
| 11 |
+
# This file lives exclusively in app/ and is ONLY started by app/app.py.
|
| 12 |
+
# NO direct access to fundaments/*, .env, or Guardian (main.py).
|
| 13 |
+
# All config comes from app/.pyfun via app/config.py.
|
| 14 |
+
#
|
| 15 |
+
# PROVIDER PRINCIPLE:
|
| 16 |
+
# No key = no provider = no tool = no crash.
|
| 17 |
+
# Server always starts, just with fewer providers.
|
| 18 |
+
# Adding a new provider = update .pyfun + add class here. Never touch mcp.py!
|
| 19 |
+
#
|
| 20 |
+
# FALLBACK CHAIN:
|
| 21 |
+
# Defined in .pyfun per provider via fallback_to field.
|
| 22 |
+
# anthropic → fails → gemini → fails → openrouter → fails → RuntimeError
|
| 23 |
+
# Visited set prevents infinite loops.
|
| 24 |
+
#
|
| 25 |
+
# SECURITY NOTE:
|
| 26 |
+
# API keys are NEVER logged or included in exception messages.
|
| 27 |
+
# All errors are sanitized before propagation — only HTTP status codes
|
| 28 |
+
# and safe_url (query params stripped) are ever exposed in logs.
|
| 29 |
+
#
|
| 30 |
+
# CACHING NOTE:
|
| 31 |
+
# Anthropic → prompt_caching (cache_control: ephemeral)
|
| 32 |
+
# Requires anthropic-beta: prompt-caching-2024-07-31 header.
|
| 33 |
+
# Caches system prompt + long user prompts (>1024 tokens estimated).
|
| 34 |
+
# Saves up to 90% input token costs on repeated context.
|
| 35 |
+
# Enable per provider in .pyfun: supports_cache = "true"
|
| 36 |
+
#
|
| 37 |
+
# Gemini → Implicit caching (automatic, no extra API call needed)
|
| 38 |
+
# Google automatically caches repeated prompt prefixes server-side.
|
| 39 |
+
# No code change needed — Gemini handles it transparently.
|
| 40 |
+
# Explicit Context Caching API exists but requires separate cache management
|
| 41 |
+
# and is only worth it for very large static contexts (32k+ tokens).
|
| 42 |
+
# Enable per provider in .pyfun: supports_cache = "true"
|
| 43 |
+
# (currently used as log hint only for Gemini — implicit cache is always on)
|
| 44 |
+
#
|
| 45 |
+
# HOW TO ADD A NEW LLM PROVIDER — 3 steps, nothing else to touch:
|
| 46 |
+
# 1. Add class below (copy a dummy, implement complete())
|
| 47 |
+
# 2. Register name → class in _PROVIDER_CLASSES dict
|
| 48 |
+
# 3. Add [LLM_PROVIDER.yourprovider] block in app/.pyfun
|
| 49 |
+
# → env_key, base_url, default_model, fallback_to
|
| 50 |
+
#
|
| 51 |
+
# DEPENDENCY CHAIN (app/* only, no fundaments!):
|
| 52 |
+
# config.py → parses app/.pyfun — single source of truth
|
| 53 |
+
# providers.py → LLM + Search registry + fallback chain
|
| 54 |
+
# tools.py → calls providers.llm_complete() / providers.search()
|
| 55 |
+
# mcp.py → calls providers.list_active_llm() / list_active_search()
|
| 56 |
+
# =============================================================================
|
| 57 |
+
|
| 58 |
+
import os
|
| 59 |
+
import logging
|
| 60 |
+
import httpx
|
| 61 |
+
|
| 62 |
+
from . import config
|
| 63 |
+
logging.getLogger("httpx").setLevel(logging.WARNING)
|
| 64 |
+
logger = logging.getLogger("providers")
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
# =============================================================================
|
| 68 |
+
# SECTION 1 — Base Provider
|
| 69 |
+
# Shared HTTP logic — implemented ONCE, reused by all providers.
|
| 70 |
+
# =============================================================================
|
| 71 |
+
|
| 72 |
+
class BaseProvider:
|
| 73 |
+
"""
|
| 74 |
+
Base class for all LLM providers.
|
| 75 |
+
Subclasses only implement complete() — HTTP logic lives here.
|
| 76 |
+
"""
|
| 77 |
+
def __init__(self, name: str, cfg: dict):
|
| 78 |
+
self.name = name
|
| 79 |
+
self.key = os.getenv(cfg.get("env_key", ""))
|
| 80 |
+
self.base_url = cfg.get("base_url", "")
|
| 81 |
+
self.fallback = cfg.get("fallback_to", "")
|
| 82 |
+
self.timeout = int(config.get_limits().get("REQUEST_TIMEOUT_SEC", "60"))
|
| 83 |
+
self.model = cfg.get("default_model", "")
|
| 84 |
+
self.supports_cache = cfg.get("supports_cache", "false").lower() == "true"
|
| 85 |
+
# Safe key hint for debug logs — never log the full key
|
| 86 |
+
self._key_hint = (
|
| 87 |
+
f"{self.key[:4]}...{self.key[-4:]}"
|
| 88 |
+
if self.key and len(self.key) > 8
|
| 89 |
+
else "***"
|
| 90 |
+
)
|
| 91 |
+
|
| 92 |
+
async def complete(self, prompt: str, model: str, max_tokens: int) -> str:
|
| 93 |
+
"""Override in each provider subclass."""
|
| 94 |
+
raise NotImplementedError
|
| 95 |
+
|
| 96 |
+
async def _post(self, url: str, headers: dict, payload: dict) -> dict:
|
| 97 |
+
"""
|
| 98 |
+
Shared HTTP POST — used by all providers.
|
| 99 |
+
Raises RuntimeError with sanitized message on non-2xx responses.
|
| 100 |
+
API keys are never included in raised exceptions or log output.
|
| 101 |
+
"""
|
| 102 |
+
safe_url = url.split("?")[0] # strip query params (may contain API keys)
|
| 103 |
+
logger.debug(f"POST → {safe_url}")
|
| 104 |
+
async with httpx.AsyncClient() as client:
|
| 105 |
+
r = await client.post(
|
| 106 |
+
url,
|
| 107 |
+
headers=headers,
|
| 108 |
+
json=payload,
|
| 109 |
+
timeout=self.timeout,
|
| 110 |
+
)
|
| 111 |
+
try:
|
| 112 |
+
r.raise_for_status()
|
| 113 |
+
except httpx.HTTPStatusError as e:
|
| 114 |
+
# Sanitize: only status code + safe_url, never headers or body
|
| 115 |
+
raise RuntimeError(
|
| 116 |
+
f"HTTP {e.response.status_code} from {safe_url}"
|
| 117 |
+
) from None
|
| 118 |
+
return r.json()
|
| 119 |
+
|
| 120 |
+
|
| 121 |
+
# =============================================================================
|
| 122 |
+
# SECTION 2 — LLM Provider Implementations
|
| 123 |
+
# Only the API-specific parsing logic differs per provider.
|
| 124 |
+
# =============================================================================
|
| 125 |
+
|
| 126 |
+
# --- SmolLM2 (Custom Assistant Space) ----------------------------------------
|
| 127 |
+
class SmolLMProvider(BaseProvider):
|
| 128 |
+
"""
|
| 129 |
+
SmolLM2 Custom Assistant Space — OpenAI-compatible, ADI routing included.
|
| 130 |
+
Free tier on HF Spaces (CPU). Falls back to next provider on 503.
|
| 131 |
+
Response includes extra 'adi' field with score + decision (ignored by hub).
|
| 132 |
+
Deploy: https://github.com/VolkanSah/Multi-LLM-API-Gateway (smollm-space/)
|
| 133 |
+
|
| 134 |
+
.pyfun block:
|
| 135 |
+
[LLM_PROVIDER.smollm]
|
| 136 |
+
active = "true"
|
| 137 |
+
base_url = "https://codey-lab-SmolLM2-customs.hf.space/v1"
|
| 138 |
+
env_key = "SMOLLM_API_KEY"
|
| 139 |
+
default_model = "smollm2-360m"
|
| 140 |
+
models = "smollm2-360m, codey-lab/model.universal-mcp-hub"
|
| 141 |
+
fallback_to = "anthropic"
|
| 142 |
+
[LLM_PROVIDER.smollm_END]
|
| 143 |
+
"""
|
| 144 |
+
|
| 145 |
+
async def complete(self, prompt: str, model: str = None, max_tokens: int = 150) -> str:
|
| 146 |
+
data = await self._post(
|
| 147 |
+
f"{self.base_url}/chat/completions",
|
| 148 |
+
headers={
|
| 149 |
+
"Authorization": f"Bearer {self.key}",
|
| 150 |
+
"X-IP-Token": self.key,
|
| 151 |
+
"content-type": "application/json",
|
| 152 |
+
},
|
| 153 |
+
payload={
|
| 154 |
+
"model": model or self.model,
|
| 155 |
+
"max_tokens": max_tokens,
|
| 156 |
+
"messages": [{"role": "user", "content": prompt}],
|
| 157 |
+
},
|
| 158 |
+
)
|
| 159 |
+
return data["choices"][0]["message"]["content"]
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
# --- Anthropic ----------------------------------------------------------------
|
| 163 |
+
class AnthropicProvider(BaseProvider):
|
| 164 |
+
"""
|
| 165 |
+
Anthropic Claude API — Messages endpoint.
|
| 166 |
+
|
| 167 |
+
Prompt Caching (supports_cache = "true" in .pyfun):
|
| 168 |
+
Uses cache_control: ephemeral on system prompt and long user prompts.
|
| 169 |
+
Requires anthropic-beta: prompt-caching-2024-07-31 header.
|
| 170 |
+
Cache TTL: 5 minutes, extended on each cache hit.
|
| 171 |
+
Min tokens to cache: ~1024 (Anthropic requirement).
|
| 172 |
+
Cost: cache write ~25% more, cache read ~90% less than normal input.
|
| 173 |
+
|
| 174 |
+
.pyfun block:
|
| 175 |
+
[LLM_PROVIDER.anthropic]
|
| 176 |
+
active = "true"
|
| 177 |
+
base_url = "https://api.anthropic.com/v1"
|
| 178 |
+
env_key = "ANTHROPIC_API_KEY"
|
| 179 |
+
api_version_header = "2023-06-01"
|
| 180 |
+
default_model = "claude-haiku-4-5"
|
| 181 |
+
supports_cache = "true"
|
| 182 |
+
fallback_to = "gemini"
|
| 183 |
+
[LLM_PROVIDER.anthropic_END]
|
| 184 |
+
"""
|
| 185 |
+
|
| 186 |
+
# Rough chars-per-token estimate — avoids importing tiktoken in sandbox
|
| 187 |
+
_CHARS_PER_TOKEN = 4
|
| 188 |
+
_CACHE_MIN_TOKENS = 1024
|
| 189 |
+
|
| 190 |
+
def _is_cacheable(self, text: str) -> bool:
|
| 191 |
+
"""Estimate if text is long enough to benefit from caching."""
|
| 192 |
+
return len(text) >= self._CACHE_MIN_TOKENS * self._CHARS_PER_TOKEN
|
| 193 |
+
|
| 194 |
+
async def complete(
|
| 195 |
+
self,
|
| 196 |
+
prompt: str,
|
| 197 |
+
model: str = None,
|
| 198 |
+
max_tokens: int = 1024,
|
| 199 |
+
system: str = None,
|
| 200 |
+
) -> str:
|
| 201 |
+
cfg = config.get_active_llm_providers().get("anthropic", {})
|
| 202 |
+
|
| 203 |
+
headers = {
|
| 204 |
+
"x-api-key": self.key,
|
| 205 |
+
"anthropic-version": cfg.get("api_version_header", "2023-06-01"),
|
| 206 |
+
"content-type": "application/json",
|
| 207 |
+
}
|
| 208 |
+
|
| 209 |
+
# --- Build user content ---
|
| 210 |
+
# Add cache_control if caching enabled + prompt long enough
|
| 211 |
+
if self.supports_cache and self._is_cacheable(prompt):
|
| 212 |
+
user_content = [
|
| 213 |
+
{
|
| 214 |
+
"type": "text",
|
| 215 |
+
"text": prompt,
|
| 216 |
+
"cache_control": {"type": "ephemeral"},
|
| 217 |
+
}
|
| 218 |
+
]
|
| 219 |
+
headers["anthropic-beta"] = "prompt-caching-2024-07-31"
|
| 220 |
+
logger.debug("Anthropic: prompt cache_control applied to user message.")
|
| 221 |
+
else:
|
| 222 |
+
user_content = prompt # short prompt — plain string, no overhead
|
| 223 |
+
|
| 224 |
+
payload = {
|
| 225 |
+
"model": model or self.model,
|
| 226 |
+
"max_tokens": max_tokens,
|
| 227 |
+
"messages": [{"role": "user", "content": user_content}],
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
# --- Optional system prompt with cache_control ---
|
| 231 |
+
if system:
|
| 232 |
+
if self.supports_cache and self._is_cacheable(system):
|
| 233 |
+
payload["system"] = [
|
| 234 |
+
{
|
| 235 |
+
"type": "text",
|
| 236 |
+
"text": system,
|
| 237 |
+
"cache_control": {"type": "ephemeral"},
|
| 238 |
+
}
|
| 239 |
+
]
|
| 240 |
+
headers["anthropic-beta"] = "prompt-caching-2024-07-31"
|
| 241 |
+
logger.debug("Anthropic: prompt cache_control applied to system prompt.")
|
| 242 |
+
else:
|
| 243 |
+
payload["system"] = system
|
| 244 |
+
|
| 245 |
+
data = await self._post(f"{self.base_url}/messages", headers, payload)
|
| 246 |
+
return data["content"][0]["text"]
|
| 247 |
+
|
| 248 |
+
|
| 249 |
+
# --- Gemini ------------------------------------------------------------------
|
| 250 |
+
class GeminiProvider(BaseProvider):
|
| 251 |
+
"""
|
| 252 |
+
Google Gemini API — generateContent endpoint.
|
| 253 |
+
|
| 254 |
+
Implicit Caching (always active on Gemini side, no code needed):
|
| 255 |
+
Google automatically caches repeated prompt prefixes server-side.
|
| 256 |
+
No extra API call, no cache key, no TTL management needed.
|
| 257 |
+
Just send the same prompt structure and Gemini handles the rest.
|
| 258 |
+
supports_cache = "true" in .pyfun logs cache hint only.
|
| 259 |
+
|
| 260 |
+
Explicit Context Caching (NOT implemented here — when to use it):
|
| 261 |
+
Only worth the extra API complexity for very large static contexts
|
| 262 |
+
(32k+ tokens, e.g. large documents sent on every request).
|
| 263 |
+
Requires separate POST to /cachedContents, returns a cache_name,
|
| 264 |
+
which is then referenced in generateContent as cachedContent.name.
|
| 265 |
+
Implement as a separate tool (cache_create / cache_use) when needed.
|
| 266 |
+
|
| 267 |
+
.pyfun block:
|
| 268 |
+
[LLM_PROVIDER.gemini]
|
| 269 |
+
active = "true"
|
| 270 |
+
base_url = "https://generativelanguage.googleapis.com/v1beta"
|
| 271 |
+
env_key = "GEMINI_API_KEY"
|
| 272 |
+
default_model = "gemini-2.0-flash"
|
| 273 |
+
supports_cache = "true"
|
| 274 |
+
fallback_to = "openrouter"
|
| 275 |
+
[LLM_PROVIDER.gemini_END]
|
| 276 |
+
"""
|
| 277 |
+
|
| 278 |
+
async def complete(self, prompt: str, model: str = None, max_tokens: int = 1024) -> str:
|
| 279 |
+
m = model or self.model
|
| 280 |
+
safe_url = f"{self.base_url}/models/{m}:generateContent"
|
| 281 |
+
|
| 282 |
+
if self.supports_cache:
|
| 283 |
+
logger.debug(f"Gemini: implicit caching active for model {m} (server-side, automatic).")
|
| 284 |
+
|
| 285 |
+
async with httpx.AsyncClient() as client:
|
| 286 |
+
r = await client.post(
|
| 287 |
+
safe_url,
|
| 288 |
+
params={"key": self.key}, # key in query param, never in logs
|
| 289 |
+
json={
|
| 290 |
+
"contents": [{"parts": [{"text": prompt}]}],
|
| 291 |
+
"generationConfig": {"maxOutputTokens": max_tokens},
|
| 292 |
+
},
|
| 293 |
+
timeout=self.timeout,
|
| 294 |
+
)
|
| 295 |
+
try:
|
| 296 |
+
r.raise_for_status()
|
| 297 |
+
except httpx.HTTPStatusError as e:
|
| 298 |
+
raise RuntimeError(
|
| 299 |
+
f"HTTP {e.response.status_code} from {safe_url}"
|
| 300 |
+
) from None
|
| 301 |
+
return r.json()["candidates"][0]["content"]["parts"][0]["text"]
|
| 302 |
+
|
| 303 |
+
|
| 304 |
+
# --- OpenRouter ---------------------------------------------------------------
|
| 305 |
+
class OpenRouterProvider(BaseProvider):
|
| 306 |
+
"""OpenRouter API — OpenAI-compatible chat completions endpoint.
|
| 307 |
+
|
| 308 |
+
Required headers: HTTP-Referer + X-Title (required by OpenRouter for
|
| 309 |
+
free models and rate limit attribution).
|
| 310 |
+
"""
|
| 311 |
+
|
| 312 |
+
async def complete(self, prompt: str, model: str = None, max_tokens: int = 1024) -> str:
|
| 313 |
+
data = await self._post(
|
| 314 |
+
f"{self.base_url}/chat/completions",
|
| 315 |
+
headers={
|
| 316 |
+
"Authorization": f"Bearer {self.key}",
|
| 317 |
+
"HTTP-Referer": os.getenv("APP_URL", "https://huggingface.co"),
|
| 318 |
+
"X-Title": os.getenv("HUB_NAME", "Universal AI Hub"), # required!
|
| 319 |
+
"content-type": "application/json",
|
| 320 |
+
},
|
| 321 |
+
payload={
|
| 322 |
+
"model": model or self.model,
|
| 323 |
+
"max_tokens": max_tokens,
|
| 324 |
+
"messages": [{"role": "user", "content": prompt}],
|
| 325 |
+
},
|
| 326 |
+
)
|
| 327 |
+
return data["choices"][0]["message"]["content"]
|
| 328 |
+
|
| 329 |
+
|
| 330 |
+
# --- HuggingFace --------------------------------------------------------------
|
| 331 |
+
class HuggingFaceProvider(BaseProvider):
|
| 332 |
+
"""HuggingFace Inference API — OpenAI-compatible serverless endpoint.
|
| 333 |
+
|
| 334 |
+
base_url in .pyfun: https://api-inference.huggingface.co/v1
|
| 335 |
+
Model goes in payload, not in URL.
|
| 336 |
+
Free tier: max ~8B models. PRO required for 70B+.
|
| 337 |
+
"""
|
| 338 |
+
|
| 339 |
+
async def complete(self, prompt: str, model: str = None, max_tokens: int = 512) -> str:
|
| 340 |
+
m = model or self.model
|
| 341 |
+
data = await self._post(
|
| 342 |
+
f"{self.base_url}/chat/completions",
|
| 343 |
+
headers={
|
| 344 |
+
"Authorization": f"Bearer {self.key}",
|
| 345 |
+
"content-type": "application/json",
|
| 346 |
+
},
|
| 347 |
+
payload={
|
| 348 |
+
"model": m,
|
| 349 |
+
"max_tokens": max_tokens,
|
| 350 |
+
"messages": [{"role": "user", "content": prompt}],
|
| 351 |
+
},
|
| 352 |
+
)
|
| 353 |
+
return data["choices"][0]["message"]["content"]
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
# =============================================================================
|
| 357 |
+
# DUMMY PROVIDERS — copy, uncomment, adapt
|
| 358 |
+
# Steps: (1) uncomment class (2) add to _PROVIDER_CLASSES (3) add to .pyfun
|
| 359 |
+
# =============================================================================
|
| 360 |
+
|
| 361 |
+
# --- OpenAI -------------------------------------------------------------------
|
| 362 |
+
# .pyfun block to add:
|
| 363 |
+
#
|
| 364 |
+
# [LLM_PROVIDER.openai]
|
| 365 |
+
# active = "true"
|
| 366 |
+
# base_url = "https://api.openai.com/v1"
|
| 367 |
+
# env_key = "OPENAI_API_KEY"
|
| 368 |
+
# default_model = "gpt-4o-mini"
|
| 369 |
+
# models = "gpt-4o, gpt-4o-mini, gpt-3.5-turbo"
|
| 370 |
+
# fallback_to = ""
|
| 371 |
+
# [LLM_PROVIDER.openai_END]
|
| 372 |
+
#
|
| 373 |
+
# class OpenAIProvider(BaseProvider):
|
| 374 |
+
# """OpenAI API — OpenAI-compatible chat completions endpoint."""
|
| 375 |
+
#
|
| 376 |
+
# async def complete(self, prompt: str, model: str = None, max_tokens: int = 1024) -> str:
|
| 377 |
+
# data = await self._post(
|
| 378 |
+
# f"{self.base_url}/chat/completions",
|
| 379 |
+
# headers={
|
| 380 |
+
# "Authorization": f"Bearer {self.key}",
|
| 381 |
+
# "content-type": "application/json",
|
| 382 |
+
# },
|
| 383 |
+
# payload={
|
| 384 |
+
# "model": model or self.model,
|
| 385 |
+
# "max_tokens": max_tokens,
|
| 386 |
+
# "messages": [{"role": "user", "content": prompt}],
|
| 387 |
+
# },
|
| 388 |
+
# )
|
| 389 |
+
# return data["choices"][0]["message"]["content"]
|
| 390 |
+
|
| 391 |
+
|
| 392 |
+
# --- Mistral ------------------------------------------------------------------
|
| 393 |
+
# .pyfun block to add:
|
| 394 |
+
#
|
| 395 |
+
# [LLM_PROVIDER.mistral]
|
| 396 |
+
# active = "true"
|
| 397 |
+
# base_url = "https://api.mistral.ai/v1"
|
| 398 |
+
# env_key = "MISTRAL_API_KEY"
|
| 399 |
+
# default_model = "mistral-large-latest"
|
| 400 |
+
# models = "mistral-large-latest, mistral-small-latest, codestral-latest"
|
| 401 |
+
# fallback_to = ""
|
| 402 |
+
# [LLM_PROVIDER.mistral_END]
|
| 403 |
+
#
|
| 404 |
+
# class MistralProvider(BaseProvider):
|
| 405 |
+
# """Mistral AI API — OpenAI-compatible chat completions endpoint."""
|
| 406 |
+
#
|
| 407 |
+
# async def complete(self, prompt: str, model: str = None, max_tokens: int = 1024) -> str:
|
| 408 |
+
# data = await self._post(
|
| 409 |
+
# f"{self.base_url}/chat/completions",
|
| 410 |
+
# headers={
|
| 411 |
+
# "Authorization": f"Bearer {self.key}",
|
| 412 |
+
# "content-type": "application/json",
|
| 413 |
+
# },
|
| 414 |
+
# payload={
|
| 415 |
+
# "model": model or self.model,
|
| 416 |
+
# "max_tokens": max_tokens,
|
| 417 |
+
# "messages": [{"role": "user", "content": prompt}],
|
| 418 |
+
# },
|
| 419 |
+
# )
|
| 420 |
+
# return data["choices"][0]["message"]["content"]
|
| 421 |
+
|
| 422 |
+
|
| 423 |
+
# --- xAI (Grok) ---------------------------------------------------------------
|
| 424 |
+
# .pyfun block to add:
|
| 425 |
+
#
|
| 426 |
+
# [LLM_PROVIDER.xai]
|
| 427 |
+
# active = "true"
|
| 428 |
+
# base_url = "https://api.x.ai/v1"
|
| 429 |
+
# env_key = "XAI_API_KEY"
|
| 430 |
+
# default_model = "grok-3-mini"
|
| 431 |
+
# models = "grok-3, grok-3-mini, grok-3-fast"
|
| 432 |
+
# fallback_to = ""
|
| 433 |
+
# [LLM_PROVIDER.xai_END]
|
| 434 |
+
#
|
| 435 |
+
# class XAIProvider(BaseProvider):
|
| 436 |
+
# """xAI Grok API — OpenAI-compatible chat completions endpoint."""
|
| 437 |
+
#
|
| 438 |
+
# async def complete(self, prompt: str, model: str = None, max_tokens: int = 1024) -> str:
|
| 439 |
+
# data = await self._post(
|
| 440 |
+
# f"{self.base_url}/chat/completions",
|
| 441 |
+
# headers={
|
| 442 |
+
# "Authorization": f"Bearer {self.key}",
|
| 443 |
+
# "content-type": "application/json",
|
| 444 |
+
# },
|
| 445 |
+
# payload={
|
| 446 |
+
# "model": model or self.model,
|
| 447 |
+
# "max_tokens": max_tokens,
|
| 448 |
+
# "messages": [{"role": "user", "content": prompt}],
|
| 449 |
+
# },
|
| 450 |
+
# )
|
| 451 |
+
# return data["choices"][0]["message"]["content"]
|
| 452 |
+
|
| 453 |
+
|
| 454 |
+
# =============================================================================
|
| 455 |
+
# SECTION 3 — Provider Registry
|
| 456 |
+
# Built from .pyfun [LLM_PROVIDERS] at initialize().
|
| 457 |
+
# Maps provider names → classes.
|
| 458 |
+
# To activate a dummy: uncomment class above + add entry here.
|
| 459 |
+
# =============================================================================
|
| 460 |
+
|
| 461 |
+
_PROVIDER_CLASSES = {
|
| 462 |
+
"smollm": SmolLMProvider,
|
| 463 |
+
"anthropic": AnthropicProvider,
|
| 464 |
+
"gemini": GeminiProvider,
|
| 465 |
+
"openrouter": OpenRouterProvider,
|
| 466 |
+
"huggingface": HuggingFaceProvider,
|
| 467 |
+
# "openai": OpenAIProvider, # ← uncomment to activate
|
| 468 |
+
# "mistral": MistralProvider, # ← uncomment to activate
|
| 469 |
+
# "xai": XAIProvider, # ← uncomment to activate
|
| 470 |
+
}
|
| 471 |
+
|
| 472 |
+
_registry: dict = {}
|
| 473 |
+
|
| 474 |
+
|
| 475 |
+
def initialize() -> None:
|
| 476 |
+
"""
|
| 477 |
+
Build provider registries from .pyfun [LLM_PROVIDERS] + [SEARCH_PROVIDERS].
|
| 478 |
+
Called once by mcp.py during startup sequence.
|
| 479 |
+
Skips providers with missing ENV keys — no crash, just fewer tools.
|
| 480 |
+
"""
|
| 481 |
+
global _registry
|
| 482 |
+
active = config.get_active_llm_providers()
|
| 483 |
+
|
| 484 |
+
for name, cfg in active.items():
|
| 485 |
+
env_key = cfg.get("env_key", "")
|
| 486 |
+
if not env_key or not os.getenv(env_key):
|
| 487 |
+
logger.info(f"Provider '{name}' skipped — ENV key not set.")
|
| 488 |
+
continue
|
| 489 |
+
cls = _PROVIDER_CLASSES.get(name)
|
| 490 |
+
if not cls:
|
| 491 |
+
logger.info(f"Provider '{name}' has no handler yet — skipped.")
|
| 492 |
+
continue
|
| 493 |
+
_registry[name] = cls(name, cfg)
|
| 494 |
+
cache_hint = " [cache: ON]" if cfg.get("supports_cache", "false") == "true" else ""
|
| 495 |
+
logger.info(f"Provider registered: {name}{cache_hint}")
|
| 496 |
+
|
| 497 |
+
# Search providers — same key-gating principle
|
| 498 |
+
_initialize_search()
|
| 499 |
+
|
| 500 |
+
|
| 501 |
+
# =============================================================================
|
| 502 |
+
# SECTION 4 — LLM Execution + Fallback Chain
|
| 503 |
+
# =============================================================================
|
| 504 |
+
|
| 505 |
+
async def llm_complete(
|
| 506 |
+
prompt: str,
|
| 507 |
+
provider_name: str = None,
|
| 508 |
+
model: str = None,
|
| 509 |
+
max_tokens: int = 1024,
|
| 510 |
+
system: str = None,
|
| 511 |
+
) -> str:
|
| 512 |
+
"""
|
| 513 |
+
Send prompt to LLM provider with automatic fallback chain.
|
| 514 |
+
Fallback order is defined in .pyfun via fallback_to field.
|
| 515 |
+
Raises RuntimeError if all providers in the chain fail.
|
| 516 |
+
|
| 517 |
+
Args:
|
| 518 |
+
prompt: Input text to send to the model.
|
| 519 |
+
provider_name: Provider name override. Defaults to default_provider
|
| 520 |
+
from .pyfun [TOOL.llm_complete].
|
| 521 |
+
model: Model name override. Defaults to provider's default_model.
|
| 522 |
+
max_tokens: Max tokens in response. Default: 1024.
|
| 523 |
+
system: Optional system prompt. Passed to providers that support it.
|
| 524 |
+
AnthropicProvider caches it automatically if supports_cache = true
|
| 525 |
+
and the system prompt is long enough (>= ~1024 tokens).
|
| 526 |
+
|
| 527 |
+
Returns:
|
| 528 |
+
Model response as plain text string.
|
| 529 |
+
"""
|
| 530 |
+
if not provider_name:
|
| 531 |
+
tools_cfg = config.get_active_tools()
|
| 532 |
+
provider_name = tools_cfg.get("llm_complete", {}).get("default_provider", "anthropic")
|
| 533 |
+
|
| 534 |
+
visited = set()
|
| 535 |
+
current = provider_name
|
| 536 |
+
|
| 537 |
+
while current and current not in visited:
|
| 538 |
+
visited.add(current)
|
| 539 |
+
provider = _registry.get(current)
|
| 540 |
+
|
| 541 |
+
if not provider:
|
| 542 |
+
logger.warning(f"Provider '{current}' not in registry — trying fallback.")
|
| 543 |
+
else:
|
| 544 |
+
try:
|
| 545 |
+
# Pass system prompt if provider supports it (Anthropic)
|
| 546 |
+
# Other providers accept **kwargs and ignore unknown params safely
|
| 547 |
+
if system is not None and hasattr(provider, 'complete'):
|
| 548 |
+
import inspect
|
| 549 |
+
sig = inspect.signature(provider.complete)
|
| 550 |
+
if 'system' in sig.parameters:
|
| 551 |
+
result = await provider.complete(prompt, model, max_tokens, system=system)
|
| 552 |
+
else:
|
| 553 |
+
result = await provider.complete(prompt, model, max_tokens)
|
| 554 |
+
else:
|
| 555 |
+
result = await provider.complete(prompt, model, max_tokens)
|
| 556 |
+
|
| 557 |
+
logger.info(f"Response from provider: '{current}'")
|
| 558 |
+
return f"[{current}] {result}"
|
| 559 |
+
except Exception as e:
|
| 560 |
+
# Log only exception type + sanitized message — never raw {e}
|
| 561 |
+
# which may contain headers, keys, or response bodies
|
| 562 |
+
logger.warning(
|
| 563 |
+
f"Provider '{current}' failed: {type(e).__name__}: {e} — trying fallback."
|
| 564 |
+
)
|
| 565 |
+
|
| 566 |
+
cfg = config.get_active_llm_providers().get(current, {})
|
| 567 |
+
current = cfg.get("fallback_to", "")
|
| 568 |
+
|
| 569 |
+
raise RuntimeError("All providers failed — no fallback available.")
|
| 570 |
+
|
| 571 |
+
|
| 572 |
+
# Alias — used internally by tools.py
|
| 573 |
+
complete = llm_complete
|
| 574 |
+
|
| 575 |
+
|
| 576 |
+
# =============================================================================
|
| 577 |
+
# SECTION 5 — Search Execution
|
| 578 |
+
# Same pattern as LLM providers: class + _SEARCH_PROVIDER_CLASSES + fallback.
|
| 579 |
+
# Adding a search provider = add class + register + .pyfun block. Nothing else.
|
| 580 |
+
# =============================================================================
|
| 581 |
+
|
| 582 |
+
class BaseSearchProvider:
|
| 583 |
+
"""
|
| 584 |
+
Base class for all search providers.
|
| 585 |
+
Subclasses only implement search() — HTTP logic lives here.
|
| 586 |
+
"""
|
| 587 |
+
def __init__(self, name: str, cfg: dict):
|
| 588 |
+
self.name = name
|
| 589 |
+
self.key = os.getenv(cfg.get("env_key", ""))
|
| 590 |
+
self.base_url = cfg.get("base_url", "")
|
| 591 |
+
self.fallback = cfg.get("fallback_to", "")
|
| 592 |
+
self.timeout = int(config.get_limits().get("SEARCH_TIMEOUT_SEC", "30"))
|
| 593 |
+
self.default_results = int(cfg.get("default_results", "5"))
|
| 594 |
+
self.max_results = int(cfg.get("max_results", "20"))
|
| 595 |
+
self._key_hint = (
|
| 596 |
+
f"{self.key[:4]}...{self.key[-4:]}"
|
| 597 |
+
if self.key and len(self.key) > 8
|
| 598 |
+
else "***"
|
| 599 |
+
)
|
| 600 |
+
|
| 601 |
+
async def search(self, query: str, max_results: int) -> str:
|
| 602 |
+
"""Override in each search provider subclass."""
|
| 603 |
+
raise NotImplementedError
|
| 604 |
+
|
| 605 |
+
async def _get(self, url: str, headers: dict, params: dict) -> dict:
|
| 606 |
+
"""Shared HTTP GET with sanitized errors (never leak keys/bodies)."""
|
| 607 |
+
safe_url = url.split("?")[0]
|
| 608 |
+
async with httpx.AsyncClient() as client:
|
| 609 |
+
r = await client.get(url, headers=headers, params=params, timeout=self.timeout)
|
| 610 |
+
try:
|
| 611 |
+
r.raise_for_status()
|
| 612 |
+
except httpx.HTTPStatusError as e:
|
| 613 |
+
raise RuntimeError(f"HTTP {e.response.status_code} from {safe_url}") from None
|
| 614 |
+
return r.json()
|
| 615 |
+
|
| 616 |
+
async def _post(self, url: str, headers: dict, payload: dict) -> dict:
|
| 617 |
+
"""Shared HTTP POST with sanitized errors."""
|
| 618 |
+
safe_url = url.split("?")[0]
|
| 619 |
+
async with httpx.AsyncClient() as client:
|
| 620 |
+
r = await client.post(url, headers=headers, json=payload, timeout=self.timeout)
|
| 621 |
+
try:
|
| 622 |
+
r.raise_for_status()
|
| 623 |
+
except httpx.HTTPStatusError as e:
|
| 624 |
+
raise RuntimeError(f"HTTP {e.response.status_code} from {safe_url}") from None
|
| 625 |
+
return r.json()
|
| 626 |
+
|
| 627 |
+
def _clip(self, max_results: int) -> int:
|
| 628 |
+
"""Clamp requested results into [1, max_results]."""
|
| 629 |
+
try:
|
| 630 |
+
n = int(max_results)
|
| 631 |
+
except (TypeError, ValueError):
|
| 632 |
+
n = self.default_results
|
| 633 |
+
return max(1, min(n, self.max_results))
|
| 634 |
+
|
| 635 |
+
|
| 636 |
+
# --- Brave Search ---------------------------------------------------------------
|
| 637 |
+
class BraveSearchProvider(BaseSearchProvider):
|
| 638 |
+
"""
|
| 639 |
+
Brave Search API — web search endpoint.
|
| 640 |
+
Auth via X-Subscription-Token header (NOT Bearer!).
|
| 641 |
+
|
| 642 |
+
.pyfun block:
|
| 643 |
+
[SEARCH_PROVIDER.brave]
|
| 644 |
+
active = "true"
|
| 645 |
+
base_url = "https://api.search.brave.com/res/v1/web/search"
|
| 646 |
+
env_key = "BRAVE_API_KEY"
|
| 647 |
+
default_results = "5"
|
| 648 |
+
max_results = "20"
|
| 649 |
+
fallback_to = "tavily"
|
| 650 |
+
[SEARCH_PROVIDER.brave_END]
|
| 651 |
+
"""
|
| 652 |
+
|
| 653 |
+
async def search(self, query: str, max_results: int = 5) -> str:
|
| 654 |
+
data = await self._get(
|
| 655 |
+
self.base_url,
|
| 656 |
+
headers={
|
| 657 |
+
"X-Subscription-Token": self.key,
|
| 658 |
+
"Accept": "application/json",
|
| 659 |
+
"Accept-Encoding": "gzip",
|
| 660 |
+
},
|
| 661 |
+
params={
|
| 662 |
+
"q": query,
|
| 663 |
+
"count": self._clip(max_results),
|
| 664 |
+
},
|
| 665 |
+
)
|
| 666 |
+
results = data.get("web", {}).get("results", [])
|
| 667 |
+
if not results:
|
| 668 |
+
raise RuntimeError("Brave returned no results")
|
| 669 |
+
lines = [f"[{self.name}] Search results for: '{query}'"]
|
| 670 |
+
for i, r in enumerate(results, 1):
|
| 671 |
+
lines.append(f"{i}. {r.get('title', '')}\n {r.get('url', '')}\n {r.get('description', '')}")
|
| 672 |
+
return "\n\n".join(lines)
|
| 673 |
+
|
| 674 |
+
|
| 675 |
+
# --- Tavily ---------------------------------------------------------------------
|
| 676 |
+
class TavilySearchProvider(BaseSearchProvider):
|
| 677 |
+
"""
|
| 678 |
+
Tavily Search API — AI-optimized web search.
|
| 679 |
+
include_answer = "true" in .pyfun → adds AI-synthesized answer on top.
|
| 680 |
+
|
| 681 |
+
.pyfun block:
|
| 682 |
+
[SEARCH_PROVIDER.tavily]
|
| 683 |
+
active = "true"
|
| 684 |
+
base_url = "https://api.tavily.com/search"
|
| 685 |
+
env_key = "TAVILY_API_KEY"
|
| 686 |
+
default_results = "5"
|
| 687 |
+
max_results = "10"
|
| 688 |
+
include_answer = "true"
|
| 689 |
+
fallback_to = ""
|
| 690 |
+
[SEARCH_PROVIDER.tavily_END]
|
| 691 |
+
"""
|
| 692 |
+
|
| 693 |
+
def __init__(self, name: str, cfg: dict):
|
| 694 |
+
super().__init__(name, cfg)
|
| 695 |
+
self.include_answer = cfg.get("include_answer", "false").lower() == "true"
|
| 696 |
+
|
| 697 |
+
async def search(self, query: str, max_results: int = 5) -> str:
|
| 698 |
+
data = await self._post(
|
| 699 |
+
self.base_url,
|
| 700 |
+
headers={"Content-Type": "application/json"},
|
| 701 |
+
payload={
|
| 702 |
+
"api_key": self.key, # Tavily takes the key in the payload
|
| 703 |
+
"query": query,
|
| 704 |
+
"max_results": self._clip(max_results),
|
| 705 |
+
"include_answer": self.include_answer,
|
| 706 |
+
},
|
| 707 |
+
)
|
| 708 |
+
results = data.get("results", [])
|
| 709 |
+
if not results and not data.get("answer"):
|
| 710 |
+
raise RuntimeError("Tavily returned no results")
|
| 711 |
+
lines = [f"[{self.name}] Search results for: '{query}'"]
|
| 712 |
+
if self.include_answer and data.get("answer"):
|
| 713 |
+
lines.append(f"AI answer: {data['answer']}")
|
| 714 |
+
for i, r in enumerate(results, 1):
|
| 715 |
+
lines.append(f"{i}. {r.get('title', '')}\n {r.get('url', '')}\n {r.get('content', '')}")
|
| 716 |
+
return "\n\n".join(lines)
|
| 717 |
+
|
| 718 |
+
|
| 719 |
+
# =============================================================================
|
| 720 |
+
# Search Registry — built from .pyfun [SEARCH_PROVIDERS] at initialize()
|
| 721 |
+
# =============================================================================
|
| 722 |
+
|
| 723 |
+
_SEARCH_PROVIDER_CLASSES = {
|
| 724 |
+
"brave": BraveSearchProvider,
|
| 725 |
+
"tavily": TavilySearchProvider,
|
| 726 |
+
# "serper": SerperSearchProvider, # ← uncomment to activate
|
| 727 |
+
}
|
| 728 |
+
|
| 729 |
+
_search_registry: dict = {}
|
| 730 |
+
|
| 731 |
+
|
| 732 |
+
def _initialize_search() -> None:
|
| 733 |
+
"""
|
| 734 |
+
Build search provider registry from .pyfun [SEARCH_PROVIDERS].
|
| 735 |
+
Called inside initialize() — same key-gating as LLM providers:
|
| 736 |
+
no ENV key = provider skipped, no crash.
|
| 737 |
+
"""
|
| 738 |
+
global _search_registry
|
| 739 |
+
for name, cfg in config.get_active_search_providers().items():
|
| 740 |
+
env_key = cfg.get("env_key", "")
|
| 741 |
+
if not env_key or not os.getenv(env_key):
|
| 742 |
+
logger.info(f"Search provider '{name}' skipped — ENV key not set.")
|
| 743 |
+
continue
|
| 744 |
+
cls = _SEARCH_PROVIDER_CLASSES.get(name)
|
| 745 |
+
if not cls:
|
| 746 |
+
logger.info(f"Search provider '{name}' has no handler yet — skipped.")
|
| 747 |
+
continue
|
| 748 |
+
_search_registry[name] = cls(name, cfg)
|
| 749 |
+
logger.info(f"Search provider registered: {name}")
|
| 750 |
+
|
| 751 |
+
|
| 752 |
+
async def search(
|
| 753 |
+
query: str,
|
| 754 |
+
provider_name: str = None,
|
| 755 |
+
max_results: int = 5,
|
| 756 |
+
) -> str:
|
| 757 |
+
"""
|
| 758 |
+
Search the web via configured search provider with automatic fallback chain.
|
| 759 |
+
Fallback order is defined in .pyfun via fallback_to field.
|
| 760 |
+
Raises RuntimeError if all providers in the chain fail.
|
| 761 |
+
|
| 762 |
+
Args:
|
| 763 |
+
query: Search query string.
|
| 764 |
+
provider_name: Provider name override (e.g. 'brave', 'tavily').
|
| 765 |
+
Defaults to default_provider from .pyfun [TOOL.web_search].
|
| 766 |
+
max_results: Maximum number of results. Default: 5.
|
| 767 |
+
|
| 768 |
+
Returns:
|
| 769 |
+
Formatted search results as plain text string.
|
| 770 |
+
"""
|
| 771 |
+
if not provider_name:
|
| 772 |
+
tools_cfg = config.get_active_tools()
|
| 773 |
+
provider_name = tools_cfg.get("web_search", {}).get("default_provider", "brave")
|
| 774 |
+
|
| 775 |
+
visited = set()
|
| 776 |
+
current = provider_name
|
| 777 |
+
|
| 778 |
+
while current and current not in visited:
|
| 779 |
+
visited.add(current)
|
| 780 |
+
provider = _search_registry.get(current)
|
| 781 |
+
|
| 782 |
+
if not provider:
|
| 783 |
+
logger.warning(f"Search provider '{current}' not in registry — trying fallback.")
|
| 784 |
+
else:
|
| 785 |
+
try:
|
| 786 |
+
result = await provider.search(query, max_results)
|
| 787 |
+
logger.info(f"Search response from provider: '{current}'")
|
| 788 |
+
return result
|
| 789 |
+
except Exception as e:
|
| 790 |
+
logger.warning(
|
| 791 |
+
f"Search provider '{current}' failed: {type(e).__name__}: {e} — trying fallback."
|
| 792 |
+
)
|
| 793 |
+
|
| 794 |
+
cfg = config.get_active_search_providers().get(current, {})
|
| 795 |
+
current = cfg.get("fallback_to", "")
|
| 796 |
+
|
| 797 |
+
raise RuntimeError("All search providers failed — no fallback available.")
|
| 798 |
+
|
| 799 |
+
|
| 800 |
+
# =============================================================================
|
| 801 |
+
# SECTION 6 — Registry Helpers
|
| 802 |
+
# Used by mcp.py for tool registration decisions.
|
| 803 |
+
# =============================================================================
|
| 804 |
+
|
| 805 |
+
def list_active_llm() -> list:
|
| 806 |
+
"""Returns list of active LLM provider names."""
|
| 807 |
+
return list(_registry.keys())
|
| 808 |
+
|
| 809 |
+
|
| 810 |
+
def list_active_search() -> list:
|
| 811 |
+
"""Returns list of active search provider names."""
|
| 812 |
+
return list(_search_registry.keys())
|
| 813 |
+
|
| 814 |
+
|
| 815 |
+
def get(name: str) -> BaseProvider:
|
| 816 |
+
"""Get a specific provider instance by name."""
|
| 817 |
+
return _registry.get(name)
|
| 818 |
+
|
| 819 |
+
|
| 820 |
+
# =============================================================================
|
| 821 |
+
# Direct execution guard
|
| 822 |
+
# =============================================================================
|
| 823 |
+
|
| 824 |
+
if __name__ == "__main__":
|
| 825 |
+
print("WARNING: Run via main.py → app.py, not directly.")
|
app/shellmaster.py
ADDED
|
@@ -0,0 +1,323 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# app/shellmaster.py
|
| 3 |
+
# ShellMaster 2.0 — Modular Shell Tool
|
| 4 |
+
# Universal MCP Hub (Sandboxed) - based on PyFundaments Architecture
|
| 5 |
+
# Copyright 2026 - Volkan Kücükbudak
|
| 6 |
+
# Apache License V. 2 + ESOL 1.1
|
| 7 |
+
# Repo: https://github.com/VolkanSah/Multi-LLM-API-Gateway
|
| 8 |
+
# =============================================================================
|
| 9 |
+
# ARCHITECTURE NOTE:
|
| 10 |
+
# Modular tool living in app/* only.
|
| 11 |
+
# NO low-level access, NO os.environ, NO fundaments access.
|
| 12 |
+
# Config comes from app/.pyfun [TOOL.shellmaster] via app/config.py.
|
| 13 |
+
# LLM (SmolLM2) generates commands — shellmaster validates + confirms + executes.
|
| 14 |
+
#
|
| 15 |
+
# FLOW:
|
| 16 |
+
# tools.py calls shellmaster.run(user_input)
|
| 17 |
+
# ↓
|
| 18 |
+
# LLM generates JSON {command, backup, recovery, risk}
|
| 19 |
+
# ↓
|
| 20 |
+
# validate against JSONL registry (local file or HF Dataset)
|
| 21 |
+
# ↓
|
| 22 |
+
# return confirmation request to user (NEVER auto-execute!)
|
| 23 |
+
# ↓
|
| 24 |
+
# user confirms → send to agent → return output
|
| 25 |
+
# ↓
|
| 26 |
+
# store recovery plan in db_sync
|
| 27 |
+
#
|
| 28 |
+
# CONFIRMATION FLOW:
|
| 29 |
+
# Every command requires explicit user confirmation.
|
| 30 |
+
# High-risk commands require backup + recovery plan from LLM.
|
| 31 |
+
# Recovery plan is stored in db_sync hub_state for later use.
|
| 32 |
+
# =============================================================================
|
| 33 |
+
|
| 34 |
+
import json
|
| 35 |
+
import logging
|
| 36 |
+
import os
|
| 37 |
+
import httpx
|
| 38 |
+
from pathlib import Path
|
| 39 |
+
from typing import Optional
|
| 40 |
+
|
| 41 |
+
from . import config
|
| 42 |
+
from . import db_sync
|
| 43 |
+
|
| 44 |
+
logger = logging.getLogger("shellmaster")
|
| 45 |
+
|
| 46 |
+
# =============================================================================
|
| 47 |
+
# HARDCODED BLOCKS — cannot be overridden by registry or LLM
|
| 48 |
+
# Both Unix and Windows covered
|
| 49 |
+
# =============================================================================
|
| 50 |
+
_HARDCODED_BLOCKED = {
|
| 51 |
+
# ── Unix destructive ──────────────────────────────────────────────────────
|
| 52 |
+
"rm -rf", "rm -r /", "rm -f /",
|
| 53 |
+
"mkfs", "dd if=", "> /dev/sd", "> /dev/hd",
|
| 54 |
+
"shred /dev/", "wipefs",
|
| 55 |
+
# ── Windows destructive ───────────────────────────────────────────────────
|
| 56 |
+
"format c:", "format d:", "format /fs",
|
| 57 |
+
"rmdir /s", "rd /s", "del /f /s /q c:\\",
|
| 58 |
+
# ── Unix system destruction ───────────────────────────────────────────────
|
| 59 |
+
":(){ :|:& };:",
|
| 60 |
+
"chmod -r 777 /", "chmod -r 000 /", "chmod 000 /",
|
| 61 |
+
"chown -r root /", "chown -r 0:0 /",
|
| 62 |
+
# ── Windows system destruction ────────────────────────────────────────────
|
| 63 |
+
"icacls c:\\ /grant", "icacls c:\\ /deny",
|
| 64 |
+
"takeown /f c:\\",
|
| 65 |
+
"reg delete hklm", "reg delete hkcu",
|
| 66 |
+
# ── Unix privilege escalation ─────────────────────────────────────────────
|
| 67 |
+
"sudo su", "sudo -i", "sudo -s",
|
| 68 |
+
"su root", "su -", "passwd root", "visudo",
|
| 69 |
+
# ── Windows privilege escalation ──────────────────────────────────────────
|
| 70 |
+
"runas /user:administrator",
|
| 71 |
+
"net user administrator",
|
| 72 |
+
"net localgroup administrators",
|
| 73 |
+
"psexec -s",
|
| 74 |
+
# ── Critical system files — Unix ─────────────────────────────────────────
|
| 75 |
+
"/etc/passwd", "/etc/shadow", "/etc/sudoers",
|
| 76 |
+
"/etc/crontab", "/boot/", "/proc/sysrq",
|
| 77 |
+
# ── Critical system files — Windows ──────────────────────────────────────
|
| 78 |
+
"c:\\windows\\system32\\", "c:\\windows\\system\\",
|
| 79 |
+
"hklm\\system", "hklm\\security",
|
| 80 |
+
# ── Shutdown/reboot — Unix ────────────────────────────────────────────────
|
| 81 |
+
"reboot", "halt", "poweroff",
|
| 82 |
+
"init 0", "init 6",
|
| 83 |
+
"systemctl poweroff", "systemctl reboot", "systemctl halt",
|
| 84 |
+
# ── Shutdown/reboot — Windows ─────────────────────────────────────────────
|
| 85 |
+
"shutdown /s", "shutdown /r", "shutdown /h",
|
| 86 |
+
# ── Network destruction ───────────────────────────────────────────────────
|
| 87 |
+
"iptables -f", "iptables --flush", "ufw disable",
|
| 88 |
+
"netsh firewall set opmode disable",
|
| 89 |
+
"netsh advfirewall set allprofiles state off",
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
|
| 93 |
+
def _is_hardblocked(cmd_str: str) -> bool:
|
| 94 |
+
"""Check if command contains any hardcoded blocked pattern."""
|
| 95 |
+
cmd_lower = cmd_str.lower().strip()
|
| 96 |
+
return any(blocked in cmd_lower for blocked in _HARDCODED_BLOCKED)
|
| 97 |
+
|
| 98 |
+
|
| 99 |
+
# =============================================================================
|
| 100 |
+
# Module State — populated by initialize()
|
| 101 |
+
# =============================================================================
|
| 102 |
+
_cfg = {}
|
| 103 |
+
_agent_url = ""
|
| 104 |
+
_token = ""
|
| 105 |
+
_cmd_file = Path("shellmaster_commands.jsonl")
|
| 106 |
+
_timeout = 30
|
| 107 |
+
_initialized = False
|
| 108 |
+
|
| 109 |
+
|
| 110 |
+
# =============================================================================
|
| 111 |
+
# Initialization — called by app/app.py
|
| 112 |
+
# =============================================================================
|
| 113 |
+
|
| 114 |
+
async def initialize() -> None:
|
| 115 |
+
"""
|
| 116 |
+
Initialize ShellMaster from app/.pyfun [TOOL.shellmaster].
|
| 117 |
+
Called by app/app.py — never reads os.environ directly.
|
| 118 |
+
"""
|
| 119 |
+
global _cfg, _agent_url, _token, _cmd_file, _timeout, _initialized
|
| 120 |
+
|
| 121 |
+
tools_cfg = config.get_active_tools()
|
| 122 |
+
_cfg = tools_cfg.get("shellmaster", {})
|
| 123 |
+
|
| 124 |
+
if not _cfg or _cfg.get("active", "false").lower() != "true":
|
| 125 |
+
logger.info("ShellMaster not active in .pyfun — skipped")
|
| 126 |
+
return
|
| 127 |
+
|
| 128 |
+
_agent_url = _cfg.get("shellmaster_agent_url", "http://localhost:5004")
|
| 129 |
+
# Token: ENV zuerst (shellmaster_env_key → .env, Guardian lädt ENV) —
|
| 130 |
+
# das .pyfun-Feld ist nur noch Fallback. Secrets gehören NICHT in .pyfun
|
| 131 |
+
# (gleiche Regel wie bei allen Providern: env_key = NAME, Wert in .env).
|
| 132 |
+
env_key = _cfg.get("shellmaster_env_key", "")
|
| 133 |
+
_token = os.getenv(env_key, "") if env_key else ""
|
| 134 |
+
if not _token:
|
| 135 |
+
_token = _cfg.get("shellmaster_token", "")
|
| 136 |
+
_timeout = int(_cfg.get("timeout_sec", "30"))
|
| 137 |
+
|
| 138 |
+
# Command registry — local file first
|
| 139 |
+
cmd_file_path = _cfg.get("shellmaster_commands_file", "shellmaster_commands.jsonl")
|
| 140 |
+
_cmd_file = Path(cmd_file_path)
|
| 141 |
+
|
| 142 |
+
if not _token:
|
| 143 |
+
logger.warning("ShellMaster: shellmaster_token not set in .pyfun — disabled")
|
| 144 |
+
return
|
| 145 |
+
|
| 146 |
+
if not _cmd_file.exists():
|
| 147 |
+
logger.warning(f"ShellMaster: commands file not found: {_cmd_file} — disabled")
|
| 148 |
+
return
|
| 149 |
+
|
| 150 |
+
allowed = sum(
|
| 151 |
+
1 for line in _cmd_file.open()
|
| 152 |
+
if line.strip() and json.loads(line).get("allowed", False)
|
| 153 |
+
)
|
| 154 |
+
logger.info(f"ShellMaster initialized | agent: {_agent_url} | allowed commands: {allowed}")
|
| 155 |
+
_initialized = True
|
| 156 |
+
|
| 157 |
+
|
| 158 |
+
# =============================================================================
|
| 159 |
+
# Command Registry
|
| 160 |
+
# =============================================================================
|
| 161 |
+
|
| 162 |
+
def _load_registry() -> dict:
|
| 163 |
+
"""
|
| 164 |
+
Load command registry from local JSONL.
|
| 165 |
+
Hot-reloaded on each call — no restart needed for registry changes.
|
| 166 |
+
TODO: merge with HF Dataset registry if shellmaster_commands_dataset is set.
|
| 167 |
+
"""
|
| 168 |
+
registry = {}
|
| 169 |
+
try:
|
| 170 |
+
with open(_cmd_file) as f:
|
| 171 |
+
for line in f:
|
| 172 |
+
line = line.strip()
|
| 173 |
+
if not line:
|
| 174 |
+
continue
|
| 175 |
+
cmd = json.loads(line)
|
| 176 |
+
registry[cmd["id"]] = cmd
|
| 177 |
+
except Exception as e:
|
| 178 |
+
logger.error(f"Registry load failed: {type(e).__name__}: {e}")
|
| 179 |
+
return registry
|
| 180 |
+
|
| 181 |
+
|
| 182 |
+
def _validate_command(cmd_str: str) -> tuple[bool, str]:
|
| 183 |
+
"""
|
| 184 |
+
Validate a command string against registry + hardblock list.
|
| 185 |
+
|
| 186 |
+
Returns:
|
| 187 |
+
(True, command_id) if valid and allowed
|
| 188 |
+
(False, reason) if blocked
|
| 189 |
+
"""
|
| 190 |
+
# Hardblock first — no exceptions
|
| 191 |
+
if _is_hardblocked(cmd_str):
|
| 192 |
+
return False, "blocked by system policy"
|
| 193 |
+
|
| 194 |
+
# Match against registry
|
| 195 |
+
registry = _load_registry()
|
| 196 |
+
cmd_lower = cmd_str.lower().strip()
|
| 197 |
+
|
| 198 |
+
for cmd_id, cmd in registry.items():
|
| 199 |
+
# Check both unix and win templates
|
| 200 |
+
for os_key in ("unix", "win"):
|
| 201 |
+
template_base = cmd.get(os_key, "").split()[0].lower()
|
| 202 |
+
if template_base and cmd_lower.startswith(template_base):
|
| 203 |
+
if not cmd.get("allowed", False):
|
| 204 |
+
return False, f"command '{cmd_id}' is blocked in registry"
|
| 205 |
+
return True, cmd_id
|
| 206 |
+
|
| 207 |
+
return False, f"command not found in registry: {cmd_str[:50]}"
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def list_commands(category: str = None) -> list:
|
| 211 |
+
"""List allowed commands, optionally filtered by category."""
|
| 212 |
+
cmds = [c for c in _load_registry().values() if c.get("allowed", False)]
|
| 213 |
+
if category:
|
| 214 |
+
cmds = [c for c in cmds if c.get("category") == category]
|
| 215 |
+
return cmds
|
| 216 |
+
|
| 217 |
+
|
| 218 |
+
# =============================================================================
|
| 219 |
+
# Confirmation Flow
|
| 220 |
+
# =============================================================================
|
| 221 |
+
|
| 222 |
+
def build_confirmation(llm_result: dict) -> str:
|
| 223 |
+
"""
|
| 224 |
+
Build confirmation message from LLM-generated command plan.
|
| 225 |
+
Returns formatted string for user to confirm/deny.
|
| 226 |
+
"""
|
| 227 |
+
command = llm_result.get("command", "")
|
| 228 |
+
backup = llm_result.get("backup", "")
|
| 229 |
+
recovery = llm_result.get("recovery", "")
|
| 230 |
+
risk = llm_result.get("risk", "unknown").upper()
|
| 231 |
+
|
| 232 |
+
lines = [
|
| 233 |
+
f"⚠️ ShellMaster — Confirmation Required",
|
| 234 |
+
f"",
|
| 235 |
+
f"Command: {command}",
|
| 236 |
+
f"Risk: {risk}",
|
| 237 |
+
]
|
| 238 |
+
|
| 239 |
+
if backup:
|
| 240 |
+
lines.append(f"Backup: {backup}")
|
| 241 |
+
if recovery:
|
| 242 |
+
lines.append(f"Recovery: {recovery}")
|
| 243 |
+
|
| 244 |
+
lines += [
|
| 245 |
+
f"",
|
| 246 |
+
f"Type 'confirm shellmaster' to execute or 'cancel' to abort.",
|
| 247 |
+
]
|
| 248 |
+
return "\n".join(lines)
|
| 249 |
+
|
| 250 |
+
|
| 251 |
+
async def store_recovery_plan(command: str, backup: str, recovery: str) -> None:
|
| 252 |
+
"""Store recovery plan in db_sync hub_state for later use."""
|
| 253 |
+
try:
|
| 254 |
+
await db_sync.write("shellmaster.last_command", {
|
| 255 |
+
"command": command,
|
| 256 |
+
"backup": backup,
|
| 257 |
+
"recovery": recovery,
|
| 258 |
+
})
|
| 259 |
+
logger.info("ShellMaster: recovery plan stored in db_sync")
|
| 260 |
+
except Exception as e:
|
| 261 |
+
logger.warning(f"ShellMaster: recovery plan store failed: {type(e).__name__}")
|
| 262 |
+
|
| 263 |
+
|
| 264 |
+
async def get_recovery_plan() -> dict:
|
| 265 |
+
"""Retrieve last recovery plan from db_sync."""
|
| 266 |
+
return await db_sync.read("shellmaster.last_command", default={})
|
| 267 |
+
|
| 268 |
+
|
| 269 |
+
# =============================================================================
|
| 270 |
+
# Agent Execution — only called after explicit user confirmation
|
| 271 |
+
# =============================================================================
|
| 272 |
+
|
| 273 |
+
async def execute_confirmed(command: str) -> str:
|
| 274 |
+
"""
|
| 275 |
+
Execute a command on the local agent.
|
| 276 |
+
ONLY called after explicit user confirmation — never auto-execute!
|
| 277 |
+
|
| 278 |
+
Args:
|
| 279 |
+
command: Raw shell command string (already validated).
|
| 280 |
+
|
| 281 |
+
Returns:
|
| 282 |
+
Command output or error message.
|
| 283 |
+
"""
|
| 284 |
+
if not _initialized:
|
| 285 |
+
return "ShellMaster not initialized."
|
| 286 |
+
|
| 287 |
+
# Final hardblock check before sending to agent
|
| 288 |
+
if _is_hardblocked(command):
|
| 289 |
+
logger.warning(f"HARDBLOCK on confirmed command: {command[:60]}")
|
| 290 |
+
return "Blocked by system policy — cannot execute."
|
| 291 |
+
|
| 292 |
+
try:
|
| 293 |
+
async with httpx.AsyncClient() as client:
|
| 294 |
+
r = await client.post(
|
| 295 |
+
f"{_agent_url}/command",
|
| 296 |
+
json={"command": command},
|
| 297 |
+
headers={
|
| 298 |
+
"Authorization": f"Bearer {_token}",
|
| 299 |
+
"Content-Type": "application/json",
|
| 300 |
+
},
|
| 301 |
+
timeout=_timeout,
|
| 302 |
+
)
|
| 303 |
+
r.raise_for_status()
|
| 304 |
+
data = r.json()
|
| 305 |
+
output = data.get("result", data.get("output", ""))
|
| 306 |
+
logger.info(f"ShellMaster executed: {command[:60]}")
|
| 307 |
+
return output
|
| 308 |
+
|
| 309 |
+
except httpx.ConnectError:
|
| 310 |
+
return f"ShellMaster Agent not reachable at {_agent_url} — is it running?"
|
| 311 |
+
except httpx.HTTPStatusError as e:
|
| 312 |
+
return f"Agent error: HTTP {e.response.status_code}"
|
| 313 |
+
except Exception as e:
|
| 314 |
+
logger.warning(f"ShellMaster execute failed: {type(e).__name__}: {e}")
|
| 315 |
+
return f"Error: {type(e).__name__}"
|
| 316 |
+
|
| 317 |
+
|
| 318 |
+
def is_ready() -> bool:
|
| 319 |
+
return _initialized
|
| 320 |
+
|
| 321 |
+
|
| 322 |
+
def get_config() -> dict:
|
| 323 |
+
return _cfg
|
app/tools.py
ADDED
|
@@ -0,0 +1,276 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# =============================================================================
|
| 2 |
+
# app/tools.py
|
| 3 |
+
# Tool Registry — Modular Wrapper
|
| 4 |
+
# Universal MCP Hub (Sandboxed) - based on PyFundaments Architecture
|
| 5 |
+
# Copyright 2026 - Volkan Kücükbudak
|
| 6 |
+
# Apache License V. 2 + ESOL 1.1
|
| 7 |
+
# Repo: https://github.com/VolkanSah/Universal-MCP-Hub-sandboxed
|
| 8 |
+
# =============================================================================
|
| 9 |
+
# ARCHITECTURE NOTE:
|
| 10 |
+
# This file lives exclusively in app/ and is ONLY started by app/app.py.
|
| 11 |
+
# NO direct access to fundaments/*, .env, or Guardian (main.py).
|
| 12 |
+
# All config comes from app/.pyfun via app/config.py.
|
| 13 |
+
#
|
| 14 |
+
# TOOL REGISTRY PRINCIPLE:
|
| 15 |
+
# Tools are defined in .pyfun [TOOLS] — never hardcoded here.
|
| 16 |
+
# Adding a new tool = update .pyfun only. Never touch this file.
|
| 17 |
+
# config.py parses [TOOLS] and delivers ready-to-use dicts.
|
| 18 |
+
#
|
| 19 |
+
# DEPENDENCY CHAIN:
|
| 20 |
+
# .pyfun → config.py → tools.py → mcp.py
|
| 21 |
+
# tools.py delegates execution to providers.py — never calls APIs directly.
|
| 22 |
+
# =============================================================================
|
| 23 |
+
|
| 24 |
+
import logging
|
| 25 |
+
import os
|
| 26 |
+
from typing import Any, Dict, Optional
|
| 27 |
+
|
| 28 |
+
from . import config # reads app/.pyfun — single source of truth
|
| 29 |
+
from . import providers # LLM + Search execution + fallback chain
|
| 30 |
+
from . import db_sync
|
| 31 |
+
|
| 32 |
+
logger = logging.getLogger("tools")
|
| 33 |
+
|
| 34 |
+
# =============================================================================
|
| 35 |
+
# Internal Registry — built from .pyfun [TOOLS] at initialize()
|
| 36 |
+
# =============================================================================
|
| 37 |
+
_registry: Dict[str, Dict] = {}
|
| 38 |
+
|
| 39 |
+
|
| 40 |
+
# =============================================================================
|
| 41 |
+
# Initialization — called by app/app.py (parameterless, sandboxed)
|
| 42 |
+
# =============================================================================
|
| 43 |
+
|
| 44 |
+
def initialize() -> None:
|
| 45 |
+
"""
|
| 46 |
+
Builds the tool registry from .pyfun [TOOLS].
|
| 47 |
+
Called once by app/app.py during startup sequence.
|
| 48 |
+
No fundaments passed in — fully sandboxed.
|
| 49 |
+
|
| 50 |
+
Loads all active tools and their config (description, provider_type,
|
| 51 |
+
default_provider, timeout_sec, system_prompt, etc.) into _registry.
|
| 52 |
+
Inactive tools (active = "false") are skipped silently.
|
| 53 |
+
"""
|
| 54 |
+
global _registry
|
| 55 |
+
_registry = config.get_active_tools()
|
| 56 |
+
logger.info(f"Tools loaded: {list(_registry.keys())}")
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
# =============================================================================
|
| 60 |
+
# Public API — used by mcp.py tool handlers
|
| 61 |
+
# =============================================================================
|
| 62 |
+
|
| 63 |
+
async def run(
|
| 64 |
+
tool_name: str,
|
| 65 |
+
prompt: Optional[str] = None,
|
| 66 |
+
provider_name: Optional[str] = None,
|
| 67 |
+
model: Optional[str] = None,
|
| 68 |
+
max_tokens: int = 1024,
|
| 69 |
+
max_results: int = 5,
|
| 70 |
+
**params: Any,
|
| 71 |
+
) -> str:
|
| 72 |
+
"""
|
| 73 |
+
Execute a tool by name.
|
| 74 |
+
Reads tool config from registry, delegates to providers.py / db_sync.py.
|
| 75 |
+
Applies system_prompt from .pyfun if defined.
|
| 76 |
+
|
| 77 |
+
Args:
|
| 78 |
+
tool_name: Tool name as defined in .pyfun [TOOLS] (e.g. 'llm_complete').
|
| 79 |
+
prompt: User input / query string. For db tools this is the SQL query.
|
| 80 |
+
provider_name: Override provider. Defaults to tool's default_provider in .pyfun.
|
| 81 |
+
model: Override model. Defaults to provider's default_model in .pyfun.
|
| 82 |
+
max_tokens: Max tokens for LLM response. Default: 1024.
|
| 83 |
+
max_results: Max results for search tools. Default: 5.
|
| 84 |
+
**params: Ignored extras — keeps /api calls with unknown keys from crashing.
|
| 85 |
+
|
| 86 |
+
Returns:
|
| 87 |
+
Tool response as plain text string.
|
| 88 |
+
|
| 89 |
+
Raises:
|
| 90 |
+
ValueError: If tool_name is not found in registry or prompt is missing.
|
| 91 |
+
RuntimeError: If all providers fail (propagated from providers.py).
|
| 92 |
+
"""
|
| 93 |
+
tool_cfg = _registry.get(tool_name)
|
| 94 |
+
if not tool_cfg:
|
| 95 |
+
raise ValueError(f"Tool '{tool_name}' not found in registry or not active.")
|
| 96 |
+
|
| 97 |
+
provider_type = tool_cfg.get("provider_type", "llm")
|
| 98 |
+
default_provider = provider_name or tool_cfg.get("default_provider", "")
|
| 99 |
+
system_prompt = tool_cfg.get("system_prompt", "")
|
| 100 |
+
|
| 101 |
+
# prompt ist für llm/search/db Pflicht — sauberer 400er statt TypeError/500er
|
| 102 |
+
# persist ist die Ausnahme: ohne state_read_key braucht es einen prompt,
|
| 103 |
+
# mit state_read_key liest es stattdessen aus hub_state (Gedächtnis-Pattern).
|
| 104 |
+
if provider_type in ("llm", "search", "db") and (prompt is None or str(prompt).strip() == ""):
|
| 105 |
+
raise ValueError(
|
| 106 |
+
f"Tool '{tool_name}' requires a 'prompt' parameter (for db tools: the SQL query)."
|
| 107 |
+
)
|
| 108 |
+
if provider_type == "persist" and not tool_cfg.get("state_read_key", "") \
|
| 109 |
+
and (prompt is None or str(prompt).strip() == ""):
|
| 110 |
+
raise ValueError(
|
| 111 |
+
"persist_result requires 'prompt' (or set state_read_key in .pyfun [TOOL.persist_result])."
|
| 112 |
+
)
|
| 113 |
+
|
| 114 |
+
# Build full prompt — prepend system_prompt if defined in .pyfun
|
| 115 |
+
full_prompt = f"{system_prompt}\n\n{prompt}".strip() if system_prompt else prompt
|
| 116 |
+
|
| 117 |
+
# --- LLM tools ---
|
| 118 |
+
if provider_type == "llm":
|
| 119 |
+
return await providers.llm_complete(
|
| 120 |
+
prompt=full_prompt,
|
| 121 |
+
provider_name=default_provider,
|
| 122 |
+
model=model,
|
| 123 |
+
max_tokens=max_tokens,
|
| 124 |
+
)
|
| 125 |
+
|
| 126 |
+
# --- Search tools ---
|
| 127 |
+
if provider_type == "search":
|
| 128 |
+
return await providers.search(
|
| 129 |
+
query=prompt,
|
| 130 |
+
provider_name=default_provider,
|
| 131 |
+
max_results=max_results or int(tool_cfg.get("default_results", "5")),
|
| 132 |
+
)
|
| 133 |
+
|
| 134 |
+
# --- DB tools (read-only, delegated to db_sync when ready) ---
|
| 135 |
+
if provider_type == "db":
|
| 136 |
+
sql = prompt # prompt ist hier die SQL-Query
|
| 137 |
+
return await db_sync.query(sql)
|
| 138 |
+
|
| 139 |
+
# --- Persist tools (Gedächtnis-Orchestrierung: SQLite-Queue → PostgreSQL) ---
|
| 140 |
+
# NIEMALS direkt an PG schreiben — enqueue in die restart-safe Queue,
|
| 141 |
+
# dann EINEN best-effort Flush-Versuch. Ohne DATABASE_URL = status "queued",
|
| 142 |
+
# kein Crash. Der Background-Flusher (db_sync) retries später automatisch.
|
| 143 |
+
if provider_type == "persist":
|
| 144 |
+
key = tool_cfg.get("state_read_key", "")
|
| 145 |
+
table = tool_cfg.get("target_table", "hub_results")
|
| 146 |
+
source = str(params.get("source") or tool_name)
|
| 147 |
+
# score: optionaler Slot für den Bullshit-Filter (Nutzer-eigenes Scoring).
|
| 148 |
+
# Wird 1:1 durchgereicht — KEIN Tool hier interpretiert oder berechnet ihn.
|
| 149 |
+
try:
|
| 150 |
+
score = float(params["score"]) if params.get("score") is not None else None
|
| 151 |
+
except (TypeError, ValueError):
|
| 152 |
+
score = None
|
| 153 |
+
data = await db_sync.read(key) if key else {"prompt": prompt}
|
| 154 |
+
depth = await db_sync.enqueue_persist(
|
| 155 |
+
table, payload=data, source=source, model=model or "", score=score,
|
| 156 |
+
)
|
| 157 |
+
flush = await db_sync.flush_persist_queue() # best effort — sofortiger Versuch
|
| 158 |
+
if flush.get("flushed", 0) > 0:
|
| 159 |
+
status, note = "persisted", "written to PostgreSQL"
|
| 160 |
+
else:
|
| 161 |
+
status, note = "queued", "restart-safe in SQLite — flushes to PostgreSQL when the bridge becomes active"
|
| 162 |
+
return {
|
| 163 |
+
"status": status,
|
| 164 |
+
"target_table": table,
|
| 165 |
+
"queue_depth": depth,
|
| 166 |
+
"flushed": flush.get("flushed", 0),
|
| 167 |
+
"note": note,
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
# --- Unknown provider type ---
|
| 171 |
+
logger.warning(f"Tool '{tool_name}' has unknown provider_type '{provider_type}' — skipped.")
|
| 172 |
+
return f"Tool '{tool_name}' provider type '{provider_type}' is not yet implemented."
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
# =============================================================================
|
| 176 |
+
# Registry helpers — used by mcp.py and system tools
|
| 177 |
+
# =============================================================================
|
| 178 |
+
|
| 179 |
+
def get(tool_name: str) -> Dict[str, Any]:
|
| 180 |
+
"""
|
| 181 |
+
Get full config dict for a tool.
|
| 182 |
+
|
| 183 |
+
Args:
|
| 184 |
+
tool_name: Tool name as defined in .pyfun [TOOLS].
|
| 185 |
+
|
| 186 |
+
Returns:
|
| 187 |
+
Tool config dict, or empty dict if not found.
|
| 188 |
+
"""
|
| 189 |
+
return _registry.get(tool_name, {})
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
def get_description(tool_name: str) -> str:
|
| 193 |
+
"""
|
| 194 |
+
Get the description of a tool (from .pyfun).
|
| 195 |
+
|
| 196 |
+
Args:
|
| 197 |
+
tool_name: Tool name as defined in .pyfun [TOOLS].
|
| 198 |
+
|
| 199 |
+
Returns:
|
| 200 |
+
Description string, or empty string if not found.
|
| 201 |
+
"""
|
| 202 |
+
return _registry.get(tool_name, {}).get("description", "")
|
| 203 |
+
|
| 204 |
+
|
| 205 |
+
def get_system_prompt(tool_name: str) -> str:
|
| 206 |
+
"""
|
| 207 |
+
Get the system_prompt of a tool (from .pyfun).
|
| 208 |
+
Returns empty string if no system_prompt is defined.
|
| 209 |
+
|
| 210 |
+
Args:
|
| 211 |
+
tool_name: Tool name as defined in .pyfun [TOOLS].
|
| 212 |
+
|
| 213 |
+
Returns:
|
| 214 |
+
System prompt string, or empty string if not configured.
|
| 215 |
+
"""
|
| 216 |
+
return _registry.get(tool_name, {}).get("system_prompt", "")
|
| 217 |
+
|
| 218 |
+
|
| 219 |
+
def get_timeout(tool_name: str) -> int:
|
| 220 |
+
"""
|
| 221 |
+
Get the timeout in seconds for a tool (from .pyfun).
|
| 222 |
+
|
| 223 |
+
Args:
|
| 224 |
+
tool_name: Tool name as defined in .pyfun [TOOLS].
|
| 225 |
+
|
| 226 |
+
Returns:
|
| 227 |
+
Timeout in seconds (int). Defaults to 60 if not configured.
|
| 228 |
+
"""
|
| 229 |
+
return int(_registry.get(tool_name, {}).get("timeout_sec", "60"))
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def get_provider_type(tool_name: str) -> str:
|
| 233 |
+
"""
|
| 234 |
+
Get the provider_type of a tool (llm | search | db | image | sandbox).
|
| 235 |
+
|
| 236 |
+
Args:
|
| 237 |
+
tool_name: Tool name as defined in .pyfun [TOOLS].
|
| 238 |
+
|
| 239 |
+
Returns:
|
| 240 |
+
Provider type string, or empty string if not found.
|
| 241 |
+
"""
|
| 242 |
+
return _registry.get(tool_name, {}).get("provider_type", "")
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def list_all() -> list:
|
| 246 |
+
"""
|
| 247 |
+
List all active tool names from registry.
|
| 248 |
+
|
| 249 |
+
Returns:
|
| 250 |
+
List of active tool name strings.
|
| 251 |
+
"""
|
| 252 |
+
return list(_registry.keys())
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def list_by_type(provider_type: str) -> list:
|
| 256 |
+
"""
|
| 257 |
+
List all active tools of a specific provider_type.
|
| 258 |
+
|
| 259 |
+
Args:
|
| 260 |
+
provider_type: e.g. 'llm', 'search', 'db', 'image', 'sandbox'.
|
| 261 |
+
|
| 262 |
+
Returns:
|
| 263 |
+
List of tool name strings matching the provider_type.
|
| 264 |
+
"""
|
| 265 |
+
return [
|
| 266 |
+
name for name, cfg in _registry.items()
|
| 267 |
+
if cfg.get("provider_type", "") == provider_type
|
| 268 |
+
]
|
| 269 |
+
|
| 270 |
+
|
| 271 |
+
# =============================================================================
|
| 272 |
+
# Direct execution guard
|
| 273 |
+
# =============================================================================
|
| 274 |
+
|
| 275 |
+
if __name__ == "__main__":
|
| 276 |
+
print("WARNING: Run via main.py → app.py, not directly.")
|
example-mcp___.env
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# ===========================================
|
| 2 |
+
# .env.example — Multi-LLM-API-Gateway
|
| 3 |
+
# Copy this file to .env and fill in the values.
|
| 4 |
+
# IMPORTANT: Never commit the .env file to version control.
|
| 5 |
+
# ===========================================
|
| 6 |
+
# ARCHITECTURE NOTE:
|
| 7 |
+
# The Guardian (main.py) is the ONLY process that reads this file.
|
| 8 |
+
# app/* modules NEVER touch .env — they get ENV-VAR NAMES via app/.pyfun
|
| 9 |
+
# and the values are read with os.getenv() inside the sandbox boundary.
|
| 10 |
+
#
|
| 11 |
+
# No key set = provider skipped = tool not registered = NO CRASH.
|
| 12 |
+
# ===========================================
|
| 13 |
+
|
| 14 |
+
# --- LLM Provider Keys ---
|
| 15 |
+
# Which providers activate is defined in app/.pyfun [LLM_PROVIDERS].
|
| 16 |
+
# A provider only registers when its ENV key here is set.
|
| 17 |
+
|
| 18 |
+
# Anthropic (api.anthropic.com)
|
| 19 |
+
ANTHROPIC_API_KEY=""
|
| 20 |
+
|
| 21 |
+
# Google Gemini (generativelanguage.googleapis.com)
|
| 22 |
+
GEMINI_API_KEY=""
|
| 23 |
+
|
| 24 |
+
# OpenRouter (openrouter.ai) — free models available
|
| 25 |
+
OPENROUTER_API_KEY=""
|
| 26 |
+
|
| 27 |
+
# HuggingFace Inference API (api-inference.huggingface.co)
|
| 28 |
+
HF_TOKEN=""
|
| 29 |
+
|
| 30 |
+
# SmolLM2 custom Space (your own deployment)
|
| 31 |
+
SMOLLM_API_KEY=""
|
| 32 |
+
|
| 33 |
+
# --- Search Provider Keys ---
|
| 34 |
+
# Activate search by setting at least one of these.
|
| 35 |
+
|
| 36 |
+
# Brave Search (api.search.brave.com)
|
| 37 |
+
BRAVE_API_KEY=""
|
| 38 |
+
|
| 39 |
+
# Tavily (api.tavily.com) — AI-optimized search
|
| 40 |
+
TAVILY_API_KEY=""
|
| 41 |
+
|
| 42 |
+
# --- App Identity (optional) ---
|
| 43 |
+
# Used by OpenRouter HTTP-Referer header (rate limit attribution)
|
| 44 |
+
APP_URL="https://huggingface.co/spaces/codey-lab/Universal-MCP-Hub"
|
| 45 |
+
|
| 46 |
+
# ===========================================
|
| 47 |
+
# PyFundaments CORE CONFIGURATION
|
| 48 |
+
# ===========================================
|
| 49 |
+
|
| 50 |
+
# --- Database Connection (PostgreSQL) ---
|
| 51 |
+
# The complete Data Source Name (DSN) for your PostgreSQL database.
|
| 52 |
+
# This works for both local instances and cloud providers like Neon.tech.
|
| 53 |
+
# Example for Neon: postgresql://user:password@host.eu-central-1.aws.neon.tech/neondb?sslmode=require
|
| 54 |
+
# Example for local: postgresql://user:password@localhost:5432/mydb
|
| 55 |
+
# Leave empty or comment out if your app doesn't need database
|
| 56 |
+
DATABASE_URL="your_database_dsn_here"
|
| 57 |
+
|
| 58 |
+
# --- Application-Level Encryption Keys ---
|
| 59 |
+
# These keys are used by the application itself to encrypt sensitive data
|
| 60 |
+
# before it is written to the database.
|
| 61 |
+
# The master key must be a 256-bit key (32 bytes).
|
| 62 |
+
MASTER_ENCRYPTION_KEY="your_256_bit_key_here"
|
| 63 |
+
|
| 64 |
+
# The salt is a unique, persistent value used with the master key.
|
| 65 |
+
# It should also be a secure, random string.
|
| 66 |
+
PERSISTENT_ENCRYPTION_SALT="your_unique_salt_here"
|
| 67 |
+
|
| 68 |
+
# ===========================================
|
| 69 |
+
# LOGGING CONFIGURATION (Optional)
|
| 70 |
+
# ===========================================
|
| 71 |
+
|
| 72 |
+
# PyFundaments Debug Mode (true = startup diagnostics: Python version,
|
| 73 |
+
# sys.path, file checks). Only needed while developing — false for normal use.
|
| 74 |
+
PYFUNDAMENTS_DEBUG="false"
|
| 75 |
+
|
| 76 |
+
# Log level: DEBUG, INFO, WARNING, ERROR, CRITICAL
|
| 77 |
+
# INFO is enough for daily operation; DEBUG only while hunting a bug.
|
| 78 |
+
LOG_LEVEL="INFO"
|
| 79 |
+
|
| 80 |
+
# Store logs in /tmp directory (cleared on restart, host-local only)
|
| 81 |
+
LOG_TO_TMP="false"
|
| 82 |
+
|
| 83 |
+
# Public logging — CAREFUL: stdout logs are PUBLICLY VISIBLE on HF Spaces
|
| 84 |
+
# (Space -> Logs tab). Set to "false" for production. Only keep "true"
|
| 85 |
+
# while actively debugging a deployment.
|
| 86 |
+
ENABLE_PUBLIC_LOGS="false"
|
fundaments/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
|
fundaments/access_control.py
ADDED
|
@@ -0,0 +1,247 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments: A Secure Python Architecture
|
| 2 |
+
# Copyright 2008-2025 - Volkan Kücükbudak
|
| 3 |
+
# Apache License V. 2
|
| 4 |
+
# Repo: https://github.com/VolkanSah/PyFundaments
|
| 5 |
+
# access_control.py
|
| 6 |
+
# This is the standalone module for access control.
|
| 7 |
+
# It is intended to be imported by main.py or app.py.
|
| 8 |
+
|
| 9 |
+
import sys
|
| 10 |
+
from typing import Optional, List, Dict, Any
|
| 11 |
+
|
| 12 |
+
# Your foundational PostgreSQL module is imported here.
|
| 13 |
+
# IMPORTANT: Ensure 'postgresql.py' is in the same directory or accessible
|
| 14 |
+
# via the Python path.
|
| 15 |
+
from fundaments import postgresql as db
|
| 16 |
+
# local db use: db.execute_secured_query(), db.init_db_pool(), etc. but this fundaments are optimized for clouds
|
| 17 |
+
|
| 18 |
+
|
| 19 |
+
# If asyncpg is not installed, exit the script gracefully.
|
| 20 |
+
try:
|
| 21 |
+
import asyncpg
|
| 22 |
+
except ImportError:
|
| 23 |
+
print("Error: The 'asyncpg' library is required. Please install it with 'pip install asyncpg'.")
|
| 24 |
+
sys.exit(1)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class AccessControl:
|
| 28 |
+
"""
|
| 29 |
+
Asynchronous class for managing access control.
|
| 30 |
+
It builds directly on the functions of your secure database module.
|
| 31 |
+
This is the layer that uses the foundation without modifying it.
|
| 32 |
+
|
| 33 |
+
How to extend this class:
|
| 34 |
+
To add new functionality, simply create a new asynchronous method
|
| 35 |
+
within this class. This method can call the `db.execute_secured_query`
|
| 36 |
+
function to interact with the database. For example:
|
| 37 |
+
|
| 38 |
+
async def get_user_last_login(self):
|
| 39 |
+
sql = "SELECT last_login FROM users WHERE id = $1"
|
| 40 |
+
result = await db.execute_secured_query(sql, self.user_id, fetch_method='fetchrow')
|
| 41 |
+
return result['last_login'] if result else None
|
| 42 |
+
"""
|
| 43 |
+
def __init__(self, user_id: Optional[int] = None):
|
| 44 |
+
self.user_id = user_id
|
| 45 |
+
|
| 46 |
+
async def has_permission(self, permission_name: str, user_id: Optional[int] = None) -> bool:
|
| 47 |
+
"""
|
| 48 |
+
Checks if the user has a specific permission.
|
| 49 |
+
Uses the secure execute_secured_query function from the foundation.
|
| 50 |
+
|
| 51 |
+
Args:
|
| 52 |
+
permission_name: Name of the permission to check.
|
| 53 |
+
user_id: Optional override — lets Security.check_permission()
|
| 54 |
+
check ANY user without constructing a new instance.
|
| 55 |
+
Falls back to the constructor user_id.
|
| 56 |
+
"""
|
| 57 |
+
effective_user_id = user_id if user_id is not None else self.user_id
|
| 58 |
+
if effective_user_id is None:
|
| 59 |
+
return False
|
| 60 |
+
|
| 61 |
+
# SQL Query Explanation:
|
| 62 |
+
# This query counts the number of rows where the user (user_id)
|
| 63 |
+
# has a role (user_role_assignments) that is linked to a permission
|
| 64 |
+
# (role_permissions) which has the specified name (user_permissions.name).
|
| 65 |
+
# If the count is > 0, the user has the permission.
|
| 66 |
+
sql = """
|
| 67 |
+
SELECT COUNT(*) AS count
|
| 68 |
+
FROM user_role_assignments ura
|
| 69 |
+
JOIN role_permissions rp ON ura.role_id = rp.role_id
|
| 70 |
+
JOIN user_permissions up ON rp.permission_id = up.id
|
| 71 |
+
WHERE ura.user_id = $1 AND up.name = $2
|
| 72 |
+
"""
|
| 73 |
+
try:
|
| 74 |
+
result = await db.execute_secured_query(
|
| 75 |
+
sql,
|
| 76 |
+
effective_user_id,
|
| 77 |
+
permission_name,
|
| 78 |
+
fetch_method='fetchrow'
|
| 79 |
+
)
|
| 80 |
+
return result['count'] > 0
|
| 81 |
+
except Exception as e:
|
| 82 |
+
# Error handling is managed by your postgresql module.
|
| 83 |
+
# We just re-raise the exception here to propagate it.
|
| 84 |
+
raise Exception(f'Failed to check permission: {e}')
|
| 85 |
+
|
| 86 |
+
async def get_user_permissions(self) -> List[Dict[str, Any]]:
|
| 87 |
+
"""Returns all permissions for a user."""
|
| 88 |
+
if self.user_id is None:
|
| 89 |
+
return []
|
| 90 |
+
|
| 91 |
+
# SQL Query Explanation:
|
| 92 |
+
# This query retrieves all distinct permission names and descriptions
|
| 93 |
+
# associated with the user's roles. `DISTINCT` ensures each permission
|
| 94 |
+
# is listed only once.
|
| 95 |
+
sql = """
|
| 96 |
+
SELECT DISTINCT up.name, up.description
|
| 97 |
+
FROM user_role_assignments ura
|
| 98 |
+
JOIN role_permissions rp ON ura.role_id = rp.role_id
|
| 99 |
+
JOIN user_permissions up ON rp.permission_id = up.id
|
| 100 |
+
WHERE ura.user_id = $1
|
| 101 |
+
ORDER BY up.name
|
| 102 |
+
"""
|
| 103 |
+
try:
|
| 104 |
+
return await db.execute_secured_query(sql, self.user_id)
|
| 105 |
+
except Exception as e:
|
| 106 |
+
raise Exception(f'Failed to get user permissions: {e}')
|
| 107 |
+
|
| 108 |
+
async def get_user_roles(self) -> List[Dict[str, Any]]:
|
| 109 |
+
"""Returns all roles for a user."""
|
| 110 |
+
if self.user_id is None:
|
| 111 |
+
return []
|
| 112 |
+
|
| 113 |
+
# SQL Query Explanation:
|
| 114 |
+
# This query selects the details (id, name, description) of the roles
|
| 115 |
+
# that are assigned to the specified user (user_id) in the
|
| 116 |
+
# `user_role_assignments` table.
|
| 117 |
+
sql = """
|
| 118 |
+
SELECT r.id, r.name, r.description
|
| 119 |
+
FROM user_role_assignments ura
|
| 120 |
+
JOIN user_roles r ON ura.role_id = r.id
|
| 121 |
+
WHERE ura.user_id = $1
|
| 122 |
+
ORDER BY r.name
|
| 123 |
+
"""
|
| 124 |
+
try:
|
| 125 |
+
return await db.execute_secured_query(sql, self.user_id)
|
| 126 |
+
except Exception as e:
|
| 127 |
+
raise Exception(f'Failed to get user roles: {e}')
|
| 128 |
+
|
| 129 |
+
async def assign_role(self, role_id: int) -> None:
|
| 130 |
+
"""Assigns a role to a user."""
|
| 131 |
+
if self.user_id is None:
|
| 132 |
+
raise Exception('No user specified')
|
| 133 |
+
|
| 134 |
+
# SQL Query Explanation:
|
| 135 |
+
# Inserts a new row into the `user_role_assignments` table to create
|
| 136 |
+
# the relationship between a user and a role.
|
| 137 |
+
sql = "INSERT INTO user_role_assignments (user_id, role_id) VALUES ($1, $2)"
|
| 138 |
+
try:
|
| 139 |
+
await db.execute_secured_query(
|
| 140 |
+
sql,
|
| 141 |
+
self.user_id,
|
| 142 |
+
role_id,
|
| 143 |
+
fetch_method='execute'
|
| 144 |
+
)
|
| 145 |
+
except Exception as e:
|
| 146 |
+
raise Exception(f'Failed to assign role: {e}')
|
| 147 |
+
|
| 148 |
+
async def remove_role(self, role_id: int) -> None:
|
| 149 |
+
"""Removes a role from a user."""
|
| 150 |
+
if self.user_id is None:
|
| 151 |
+
raise Exception('No user specified')
|
| 152 |
+
|
| 153 |
+
# SQL Query Explanation:
|
| 154 |
+
# Deletes the row from the `user_role_assignments` table that matches
|
| 155 |
+
# the specified user ($1) and role ($2).
|
| 156 |
+
sql = "DELETE FROM user_role_assignments WHERE user_id = $1 AND role_id = $2"
|
| 157 |
+
try:
|
| 158 |
+
await db.execute_secured_query(
|
| 159 |
+
sql,
|
| 160 |
+
self.user_id,
|
| 161 |
+
role_id,
|
| 162 |
+
fetch_method='execute'
|
| 163 |
+
)
|
| 164 |
+
except Exception as e:
|
| 165 |
+
raise Exception(f'Failed to remove role: {e}')
|
| 166 |
+
|
| 167 |
+
async def get_all_roles(self) -> List[Dict[str, Any]]:
|
| 168 |
+
"""Returns all available roles."""
|
| 169 |
+
# SQL Query Explanation:
|
| 170 |
+
# Selects all roles from the `user_roles` table.
|
| 171 |
+
sql = "SELECT id, name, description FROM user_roles ORDER BY name"
|
| 172 |
+
try:
|
| 173 |
+
return await db.execute_secured_query(sql)
|
| 174 |
+
except Exception as e:
|
| 175 |
+
raise Exception(f'Failed to get roles: {e}')
|
| 176 |
+
|
| 177 |
+
async def get_all_permissions(self) -> List[Dict[str, Any]]:
|
| 178 |
+
"""Returns all available permissions."""
|
| 179 |
+
# SQL Query Explanation:
|
| 180 |
+
# Selects all permissions from the `user_permissions` table.
|
| 181 |
+
sql = "SELECT id, name, description FROM user_permissions ORDER BY name"
|
| 182 |
+
try:
|
| 183 |
+
return await db.execute_secured_query(sql)
|
| 184 |
+
except Exception as e:
|
| 185 |
+
raise Exception(f'Failed to get permissions: {e}')
|
| 186 |
+
|
| 187 |
+
async def create_role(self, name: str, description: str) -> int:
|
| 188 |
+
"""Creates a new role."""
|
| 189 |
+
# SQL Query Explanation:
|
| 190 |
+
# Inserts a new role into the `user_roles` table and returns the
|
| 191 |
+
# automatically generated ID of the new role (`RETURNING id`).
|
| 192 |
+
sql = "INSERT INTO user_roles (name, description) VALUES ($1, $2) RETURNING id"
|
| 193 |
+
try:
|
| 194 |
+
result = await db.execute_secured_query(
|
| 195 |
+
sql,
|
| 196 |
+
name,
|
| 197 |
+
description,
|
| 198 |
+
fetch_method='fetchrow'
|
| 199 |
+
)
|
| 200 |
+
return result['id']
|
| 201 |
+
except Exception as e:
|
| 202 |
+
raise Exception(f'Failed to create role: {e}')
|
| 203 |
+
|
| 204 |
+
async def update_role_permissions(self, role_id: int, permission_ids: List[int]) -> None:
|
| 205 |
+
"""Updates the permissions for a role."""
|
| 206 |
+
# IMPORTANT: Since your module does not handle transactions across multiple
|
| 207 |
+
# queries, we perform these actions sequentially. Query-level security
|
| 208 |
+
# is guaranteed by your module.
|
| 209 |
+
try:
|
| 210 |
+
# SQL Query Explanation:
|
| 211 |
+
# Deletes all existing permissions for the given role.
|
| 212 |
+
sql_delete = "DELETE FROM role_permissions WHERE role_id = $1"
|
| 213 |
+
await db.execute_secured_query(sql_delete, role_id, fetch_method='execute')
|
| 214 |
+
|
| 215 |
+
# SQL Query Explanation:
|
| 216 |
+
# Inserts a new row for each permission_id passed into the
|
| 217 |
+
# `role_permissions` table.
|
| 218 |
+
if permission_ids:
|
| 219 |
+
sql_insert = "INSERT INTO role_permissions (role_id, permission_id) VALUES ($1, $2)"
|
| 220 |
+
for permission_id in permission_ids:
|
| 221 |
+
await db.execute_secured_query(
|
| 222 |
+
sql_insert,
|
| 223 |
+
role_id,
|
| 224 |
+
permission_id,
|
| 225 |
+
fetch_method='execute'
|
| 226 |
+
)
|
| 227 |
+
except Exception as e:
|
| 228 |
+
# If an error occurs, the underlying foundation will log the issue.
|
| 229 |
+
# We re-raise the error here.
|
| 230 |
+
raise Exception(f'Failed to update role permissions: {e}')
|
| 231 |
+
|
| 232 |
+
async def get_role_permissions(self, role_id: int) -> List[Dict[str, Any]]:
|
| 233 |
+
"""Returns all permissions for a role."""
|
| 234 |
+
# SQL Query Explanation:
|
| 235 |
+
# Selects the details (id, name, description) of all permissions
|
| 236 |
+
# linked to the specified role.
|
| 237 |
+
sql = """
|
| 238 |
+
SELECT p.id, p.name, p.description
|
| 239 |
+
FROM role_permissions rp
|
| 240 |
+
JOIN user_permissions p ON rp.permission_id = p.id
|
| 241 |
+
WHERE rp.role_id = $1
|
| 242 |
+
ORDER BY p.name
|
| 243 |
+
"""
|
| 244 |
+
try:
|
| 245 |
+
return await db.execute_secured_query(sql, role_id)
|
| 246 |
+
except Exception as e:
|
| 247 |
+
raise Exception(f'Failed to get role permissions: {e}')
|
fundaments/config_handler.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments: A Secure Python Architecture
|
| 2 |
+
# Copyright 2008-2025 - Volkan Kücükbudak
|
| 3 |
+
# Apache License V. 2
|
| 4 |
+
# Repo: https://github.com/VolkanSah/PyFundaments
|
| 5 |
+
# fundaments/config_handler.py
|
| 6 |
+
# This module loads environment variables without validation.
|
| 7 |
+
# Validation is handled by main.py based on required services.
|
| 8 |
+
import os
|
| 9 |
+
import sys
|
| 10 |
+
from dotenv import load_dotenv
|
| 11 |
+
from typing import Optional, Dict, Any
|
| 12 |
+
|
| 13 |
+
class ConfigHandler:
|
| 14 |
+
"""
|
| 15 |
+
A universal configuration loader that reads all environment variables
|
| 16 |
+
without imposing requirements. This ensures the config handler never
|
| 17 |
+
needs updates regardless of what services are used.
|
| 18 |
+
|
| 19 |
+
Validation is delegated to main.py which knows what services are needed.
|
| 20 |
+
"""
|
| 21 |
+
|
| 22 |
+
def __init__(self):
|
| 23 |
+
"""
|
| 24 |
+
Loads all environment variables from .env file and system environment.
|
| 25 |
+
No validation is performed - that's main.py's responsibility.
|
| 26 |
+
"""
|
| 27 |
+
load_dotenv()
|
| 28 |
+
self.config = {}
|
| 29 |
+
self.load_all_config()
|
| 30 |
+
|
| 31 |
+
def load_all_config(self):
|
| 32 |
+
"""
|
| 33 |
+
Loads all available environment variables into the config dictionary.
|
| 34 |
+
No validation - main.py decides what's required based on enabled services.
|
| 35 |
+
"""
|
| 36 |
+
# Load all environment variables
|
| 37 |
+
for key, value in os.environ.items():
|
| 38 |
+
if value: # Only store non-empty values
|
| 39 |
+
self.config[key] = value
|
| 40 |
+
|
| 41 |
+
def get(self, key: str) -> Optional[str]:
|
| 42 |
+
"""
|
| 43 |
+
Retrieves a configuration value by key.
|
| 44 |
+
Returns None if the key doesn't exist.
|
| 45 |
+
|
| 46 |
+
Args:
|
| 47 |
+
key: The environment variable name
|
| 48 |
+
|
| 49 |
+
Returns:
|
| 50 |
+
The value as string or None if not found
|
| 51 |
+
"""
|
| 52 |
+
return self.config.get(key)
|
| 53 |
+
|
| 54 |
+
def get_bool(self, key: str, default: bool = False) -> bool:
|
| 55 |
+
"""
|
| 56 |
+
Retrieves a boolean configuration value.
|
| 57 |
+
Recognizes: true, false, 1, 0, yes, no (case insensitive)
|
| 58 |
+
|
| 59 |
+
Args:
|
| 60 |
+
key: The environment variable name
|
| 61 |
+
default: Default value if key not found
|
| 62 |
+
|
| 63 |
+
Returns:
|
| 64 |
+
Boolean value
|
| 65 |
+
"""
|
| 66 |
+
value = self.get(key)
|
| 67 |
+
if value is None:
|
| 68 |
+
return default
|
| 69 |
+
|
| 70 |
+
return value.lower() in ('true', '1', 'yes', 'on')
|
| 71 |
+
|
| 72 |
+
def get_int(self, key: str, default: int = 0) -> int:
|
| 73 |
+
"""
|
| 74 |
+
Retrieves an integer configuration value.
|
| 75 |
+
|
| 76 |
+
Args:
|
| 77 |
+
key: The environment variable name
|
| 78 |
+
default: Default value if key not found or invalid
|
| 79 |
+
|
| 80 |
+
Returns:
|
| 81 |
+
Integer value
|
| 82 |
+
"""
|
| 83 |
+
value = self.get(key)
|
| 84 |
+
if value is None:
|
| 85 |
+
return default
|
| 86 |
+
|
| 87 |
+
try:
|
| 88 |
+
return int(value)
|
| 89 |
+
except ValueError:
|
| 90 |
+
return default
|
| 91 |
+
|
| 92 |
+
def has(self, key: str) -> bool:
|
| 93 |
+
"""
|
| 94 |
+
Checks if a configuration key exists and has a non-empty value.
|
| 95 |
+
|
| 96 |
+
Args:
|
| 97 |
+
key: The environment variable name
|
| 98 |
+
|
| 99 |
+
Returns:
|
| 100 |
+
True if key exists and has value, False otherwise
|
| 101 |
+
"""
|
| 102 |
+
return key in self.config and bool(self.config[key])
|
| 103 |
+
|
| 104 |
+
def get_all(self) -> Dict[str, str]:
|
| 105 |
+
"""
|
| 106 |
+
Returns all loaded configuration as a dictionary.
|
| 107 |
+
Useful for debugging or passing to other services.
|
| 108 |
+
|
| 109 |
+
Returns:
|
| 110 |
+
Dictionary of all loaded environment variables
|
| 111 |
+
"""
|
| 112 |
+
return self.config.copy()
|
| 113 |
+
|
| 114 |
+
# Global singleton instance
|
| 115 |
+
config_service = ConfigHandler()
|
fundaments/debug.py
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
import os
|
| 2 |
+
import sys
|
| 3 |
+
import importlib.util
|
| 4 |
+
import datetime
|
| 5 |
+
import logging
|
| 6 |
+
from logging.handlers import RotatingFileHandler
|
| 7 |
+
|
| 8 |
+
class PyFundamentsDebug:
|
| 9 |
+
"""
|
| 10 |
+
Debug-Klasse für PyFundaments.
|
| 11 |
+
Liest ENV-Variablen und konfiguriert Logging.
|
| 12 |
+
Gibt Startup-Infos aus, wenn PYFUNDAMENTS_DEBUG=true.
|
| 13 |
+
"""
|
| 14 |
+
|
| 15 |
+
def __init__(self):
|
| 16 |
+
# Debug aktivieren mit ENV VAR (default: False)
|
| 17 |
+
self.enabled = os.getenv("PYFUNDAMENTS_DEBUG", "false").lower() == "true"
|
| 18 |
+
|
| 19 |
+
# Log-Level aus ENV lesen, default INFO
|
| 20 |
+
log_level_str = os.getenv("LOG_LEVEL", "INFO").upper()
|
| 21 |
+
self.log_level = getattr(logging, log_level_str, logging.INFO)
|
| 22 |
+
|
| 23 |
+
# Logs in /tmp oder stdout? (default False)
|
| 24 |
+
self.log_to_tmp = os.getenv("LOG_TO_TMP", "false").lower() == "true"
|
| 25 |
+
|
| 26 |
+
# Öffentliches Logging an/aus (default True)
|
| 27 |
+
self.enable_public_logs = os.getenv("ENABLE_PUBLIC_LOGS", "true").lower() == "true"
|
| 28 |
+
|
| 29 |
+
# Logger Setup
|
| 30 |
+
self.logger = logging.getLogger('pyfundaments_debug')
|
| 31 |
+
self._setup_logger()
|
| 32 |
+
|
| 33 |
+
def _setup_logger(self):
|
| 34 |
+
if not self.enable_public_logs:
|
| 35 |
+
# Nur kritische Fehler im Log, wenn deaktiviert
|
| 36 |
+
logging.basicConfig(level=logging.CRITICAL)
|
| 37 |
+
return
|
| 38 |
+
|
| 39 |
+
handlers = []
|
| 40 |
+
if self.log_to_tmp:
|
| 41 |
+
log_file = '/tmp/pyfundaments_debug.log'
|
| 42 |
+
file_handler = RotatingFileHandler(log_file, maxBytes=1024*1024, backupCount=3)
|
| 43 |
+
handlers.append(file_handler)
|
| 44 |
+
handlers.append(logging.StreamHandler(sys.stdout))
|
| 45 |
+
|
| 46 |
+
logging.basicConfig(
|
| 47 |
+
level=self.log_level,
|
| 48 |
+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
| 49 |
+
handlers=handlers
|
| 50 |
+
)
|
| 51 |
+
|
| 52 |
+
def run(self):
|
| 53 |
+
if not self.enabled:
|
| 54 |
+
return
|
| 55 |
+
|
| 56 |
+
self.logger.info(f"==== PYFUNDAMENTS DEBUG STARTUP ({datetime.datetime.now()}) ====")
|
| 57 |
+
self.logger.info(f"Python version: {sys.version}")
|
| 58 |
+
self.logger.info(f"CWD: {os.getcwd()}")
|
| 59 |
+
self.logger.info(f"sys.path: {sys.path}")
|
| 60 |
+
|
| 61 |
+
fund_dir = "fundaments"
|
| 62 |
+
self.logger.info(f"fundaments exists: {os.path.isdir(fund_dir)}")
|
| 63 |
+
|
| 64 |
+
required_files = [
|
| 65 |
+
"__init__.py",
|
| 66 |
+
"access_control.py",
|
| 67 |
+
"postgresql.py",
|
| 68 |
+
"config_handler.py",
|
| 69 |
+
"security.py",
|
| 70 |
+
"user_handler.py",
|
| 71 |
+
"encryption.py"
|
| 72 |
+
]
|
| 73 |
+
|
| 74 |
+
for file in required_files:
|
| 75 |
+
path = os.path.join(fund_dir, file)
|
| 76 |
+
exists = os.path.isfile(path)
|
| 77 |
+
readable = os.access(path, os.R_OK)
|
| 78 |
+
self.logger.info(f"{file} exists: {exists} | readable: {readable}")
|
| 79 |
+
|
| 80 |
+
if os.path.isdir(fund_dir):
|
| 81 |
+
self.logger.info(f"Listing fundaments contents: {os.listdir(fund_dir)}")
|
| 82 |
+
else:
|
| 83 |
+
self.logger.warning("fundaments folder not found.")
|
| 84 |
+
|
| 85 |
+
spec = importlib.util.find_spec("fundaments")
|
| 86 |
+
self.logger.info(f"fundaments importable: {spec is not None}")
|
| 87 |
+
|
| 88 |
+
conflicts = [mod for mod in sys.modules if mod == "fundaments"]
|
| 89 |
+
self.logger.info(f"Name conflict in sys.modules: {conflicts}")
|
| 90 |
+
|
| 91 |
+
self.logger.info("==== PYFUNDAMENTS DEBUG END ====")
|
fundaments/encryption.py
ADDED
|
@@ -0,0 +1,233 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments: A Secure Python Architecture
|
| 2 |
+
# Copyright 2008-2025 - Volkan Kücükbudak
|
| 3 |
+
# Apache License V. 2
|
| 4 |
+
# Repo: https://github.com/VolkanSah/PyFundaments
|
| 5 |
+
# encryption.py
|
| 6 |
+
# A secure and robust encryption module using the cryptography library.
|
| 7 |
+
# This module is designed as a core component for a CMS architecture.
|
| 8 |
+
|
| 9 |
+
import os
|
| 10 |
+
import sys
|
| 11 |
+
import base64
|
| 12 |
+
import binascii
|
| 13 |
+
from typing import Dict, Union, Optional
|
| 14 |
+
|
| 15 |
+
# IMPORTANT: The cryptography library is required for this module.
|
| 16 |
+
# Please install it with 'pip install cryptography'.
|
| 17 |
+
try:
|
| 18 |
+
from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes
|
| 19 |
+
from cryptography.hazmat.primitives import hashes
|
| 20 |
+
from cryptography.hazmat.backends import default_backend
|
| 21 |
+
from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC
|
| 22 |
+
from cryptography.exceptions import InvalidTag
|
| 23 |
+
except ImportError:
|
| 24 |
+
print("Error: The 'cryptography' library is required. Please install it with 'pip install cryptography'.")
|
| 25 |
+
sys.exit(1)
|
| 26 |
+
|
| 27 |
+
class Encryption:
|
| 28 |
+
"""
|
| 29 |
+
A class for symmetric encryption and decryption using AES-256-GCM.
|
| 30 |
+
It securely handles both string data and file streaming.
|
| 31 |
+
|
| 32 |
+
This version is designed as a reusable core component for a larger application.
|
| 33 |
+
"""
|
| 34 |
+
CIPHER_NAME = 'AES-256-GCM'
|
| 35 |
+
KEY_LENGTH = 32 # 256 bits
|
| 36 |
+
NONCE_LENGTH = 12 # 96 bits, standard for GCM
|
| 37 |
+
TAG_LENGTH = 16 # 128 bits
|
| 38 |
+
SALT_LENGTH = 16
|
| 39 |
+
|
| 40 |
+
@staticmethod
|
| 41 |
+
def generate_salt() -> str:
|
| 42 |
+
"""
|
| 43 |
+
Generates a new, cryptographically secure random salt for key derivation.
|
| 44 |
+
This should be done once during application setup and stored securely.
|
| 45 |
+
|
| 46 |
+
Returns:
|
| 47 |
+
A hex-encoded string of the salt.
|
| 48 |
+
"""
|
| 49 |
+
return binascii.hexlify(os.urandom(Encryption.SALT_LENGTH)).decode('utf-8')
|
| 50 |
+
|
| 51 |
+
def __init__(self, master_key: str, salt: str):
|
| 52 |
+
"""
|
| 53 |
+
Initializes the encryption class by deriving a secure key from a master key.
|
| 54 |
+
The provided master_key and a persistent salt are used for key derivation.
|
| 55 |
+
|
| 56 |
+
Args:
|
| 57 |
+
master_key: The string to be used as the master key.
|
| 58 |
+
salt: The hex-encoded string of the persistent salt.
|
| 59 |
+
|
| 60 |
+
Raises:
|
| 61 |
+
ValueError: If the provided salt is not a valid hex string or has an incorrect length.
|
| 62 |
+
"""
|
| 63 |
+
try:
|
| 64 |
+
salt_bytes = binascii.unhexlify(salt)
|
| 65 |
+
except binascii.Error:
|
| 66 |
+
raise ValueError("Invalid salt format. Must be a hex-encoded string.")
|
| 67 |
+
|
| 68 |
+
if len(salt_bytes) != self.SALT_LENGTH:
|
| 69 |
+
raise ValueError(f"Invalid salt length. Expected {self.SALT_LENGTH} bytes, got {len(salt_bytes)}.")
|
| 70 |
+
|
| 71 |
+
kdf = PBKDF2HMAC(
|
| 72 |
+
algorithm=hashes.SHA256(),
|
| 73 |
+
length=self.KEY_LENGTH,
|
| 74 |
+
salt=salt_bytes,
|
| 75 |
+
iterations=480000, # Recommended value for 2023+
|
| 76 |
+
backend=default_backend()
|
| 77 |
+
)
|
| 78 |
+
self.key = kdf.derive(master_key.encode('utf-8'))
|
| 79 |
+
|
| 80 |
+
def encrypt(self, data: str) -> Dict[str, str]:
|
| 81 |
+
"""
|
| 82 |
+
Encrypts a string using AES-256-GCM.
|
| 83 |
+
|
| 84 |
+
Args:
|
| 85 |
+
data: The string to be encrypted.
|
| 86 |
+
|
| 87 |
+
Returns:
|
| 88 |
+
A dictionary containing the base64-encoded ciphertext, hex-encoded IV/nonce,
|
| 89 |
+
and hex-encoded authentication tag.
|
| 90 |
+
"""
|
| 91 |
+
nonce = os.urandom(self.NONCE_LENGTH)
|
| 92 |
+
|
| 93 |
+
aesgcm = Cipher(
|
| 94 |
+
algorithms.AES(self.key),
|
| 95 |
+
modes.GCM(nonce),
|
| 96 |
+
backend=default_backend()
|
| 97 |
+
).encryptor()
|
| 98 |
+
|
| 99 |
+
encrypted_data = aesgcm.update(data.encode('utf-8')) + aesgcm.finalize()
|
| 100 |
+
tag = aesgcm.tag
|
| 101 |
+
|
| 102 |
+
return {
|
| 103 |
+
'data': base64.b64encode(encrypted_data).decode('utf-8'),
|
| 104 |
+
'nonce': binascii.hexlify(nonce).decode('utf-8'),
|
| 105 |
+
'tag': binascii.hexlify(tag).decode('utf-8')
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
def decrypt(self, encrypted_data: str, nonce: str, tag: str) -> str:
|
| 109 |
+
"""
|
| 110 |
+
Decrypts an AES-256-GCM encrypted string.
|
| 111 |
+
|
| 112 |
+
Args:
|
| 113 |
+
encrypted_data: The base64-encoded ciphertext.
|
| 114 |
+
nonce: The hex-encoded nonce/IV.
|
| 115 |
+
tag: The hex-encoded authentication tag.
|
| 116 |
+
|
| 117 |
+
Returns:
|
| 118 |
+
The decrypted plaintext string.
|
| 119 |
+
|
| 120 |
+
Raises:
|
| 121 |
+
ValueError: If nonce, tag, or data format is invalid.
|
| 122 |
+
InvalidTag: If the authentication tag fails validation.
|
| 123 |
+
"""
|
| 124 |
+
try:
|
| 125 |
+
nonce_bytes = binascii.unhexlify(nonce)
|
| 126 |
+
tag_bytes = binascii.unhexlify(tag)
|
| 127 |
+
cipher_bytes = base64.b64decode(encrypted_data)
|
| 128 |
+
except (binascii.Error, ValueError) as e:
|
| 129 |
+
raise ValueError(f'Invalid data format: {e}')
|
| 130 |
+
|
| 131 |
+
aesgcm = Cipher(
|
| 132 |
+
algorithms.AES(self.key),
|
| 133 |
+
modes.GCM(nonce_bytes, tag_bytes),
|
| 134 |
+
backend=default_backend()
|
| 135 |
+
).decryptor()
|
| 136 |
+
|
| 137 |
+
try:
|
| 138 |
+
decrypted_data = aesgcm.update(cipher_bytes) + aesgcm.finalize()
|
| 139 |
+
return decrypted_data.decode('utf-8')
|
| 140 |
+
except InvalidTag:
|
| 141 |
+
raise InvalidTag("Authentication tag validation failed. Data may be corrupted or key is incorrect.")
|
| 142 |
+
|
| 143 |
+
def encrypt_file(self, source_path: str, destination_path: str) -> Dict[str, str]:
|
| 144 |
+
"""
|
| 145 |
+
Encrypts a file using AES-256-GCM with a streaming approach.
|
| 146 |
+
|
| 147 |
+
Args:
|
| 148 |
+
source_path: Path to the file to be encrypted.
|
| 149 |
+
destination_path: Path where the encrypted file will be saved.
|
| 150 |
+
|
| 151 |
+
Returns:
|
| 152 |
+
A dictionary containing the hex-encoded IV/nonce and authentication tag.
|
| 153 |
+
"""
|
| 154 |
+
nonce = os.urandom(self.NONCE_LENGTH)
|
| 155 |
+
|
| 156 |
+
encryptor = Cipher(
|
| 157 |
+
algorithms.AES(self.key),
|
| 158 |
+
modes.GCM(nonce),
|
| 159 |
+
backend=default_backend()
|
| 160 |
+
).encryptor()
|
| 161 |
+
|
| 162 |
+
try:
|
| 163 |
+
with open(source_path, 'rb') as fp_source, open(destination_path, 'wb') as fp_dest:
|
| 164 |
+
fp_dest.write(nonce)
|
| 165 |
+
|
| 166 |
+
chunk_size = 8192
|
| 167 |
+
while True:
|
| 168 |
+
chunk = fp_source.read(chunk_size)
|
| 169 |
+
if not chunk:
|
| 170 |
+
break
|
| 171 |
+
encrypted_chunk = encryptor.update(chunk)
|
| 172 |
+
fp_dest.write(encrypted_chunk)
|
| 173 |
+
|
| 174 |
+
encryptor.finalize()
|
| 175 |
+
tag = encryptor.tag
|
| 176 |
+
fp_dest.write(tag)
|
| 177 |
+
|
| 178 |
+
return {
|
| 179 |
+
'nonce': binascii.hexlify(nonce).decode('utf-8'),
|
| 180 |
+
'tag': binascii.hexlify(tag).decode('utf-8')
|
| 181 |
+
}
|
| 182 |
+
except FileNotFoundError as e:
|
| 183 |
+
raise ValueError(f"File not found: {e.filename}") from e
|
| 184 |
+
except Exception as e:
|
| 185 |
+
raise IOError(f"File encryption failed: {e}") from e
|
| 186 |
+
|
| 187 |
+
def decrypt_file(self, source_path: str, destination_path: str) -> None:
|
| 188 |
+
"""
|
| 189 |
+
Decrypts an AES-256-GCM encrypted file.
|
| 190 |
+
|
| 191 |
+
Args:
|
| 192 |
+
source_path: Path to the encrypted file.
|
| 193 |
+
destination_path: Path where the decrypted file will be saved.
|
| 194 |
+
"""
|
| 195 |
+
try:
|
| 196 |
+
with open(source_path, 'rb') as fp_source, open(destination_path, 'wb') as fp_dest:
|
| 197 |
+
nonce = fp_source.read(self.NONCE_LENGTH)
|
| 198 |
+
if len(nonce) != self.NONCE_LENGTH:
|
| 199 |
+
raise ValueError("Incomplete or invalid file format: Nonce is missing.")
|
| 200 |
+
|
| 201 |
+
fp_source.seek(-self.TAG_LENGTH, os.SEEK_END)
|
| 202 |
+
tag = fp_source.read(self.TAG_LENGTH)
|
| 203 |
+
if len(tag) != self.TAG_LENGTH:
|
| 204 |
+
raise ValueError("Incomplete or invalid file format: Tag is missing.")
|
| 205 |
+
|
| 206 |
+
fp_source.seek(self.NONCE_LENGTH, os.SEEK_SET) # Rewind to the start of the ciphertext
|
| 207 |
+
|
| 208 |
+
decryptor = Cipher(
|
| 209 |
+
algorithms.AES(self.key),
|
| 210 |
+
modes.GCM(nonce, tag),
|
| 211 |
+
backend=default_backend()
|
| 212 |
+
).decryptor()
|
| 213 |
+
|
| 214 |
+
chunk_size = 8192
|
| 215 |
+
encrypted_file_size = os.path.getsize(source_path) - self.NONCE_LENGTH - self.TAG_LENGTH
|
| 216 |
+
|
| 217 |
+
bytes_read = 0
|
| 218 |
+
while bytes_read < encrypted_file_size:
|
| 219 |
+
chunk_to_read = min(chunk_size, encrypted_file_size - bytes_read)
|
| 220 |
+
chunk = fp_source.read(chunk_to_read)
|
| 221 |
+
|
| 222 |
+
decrypted_chunk = decryptor.update(chunk)
|
| 223 |
+
fp_dest.write(decrypted_chunk)
|
| 224 |
+
bytes_read += len(chunk)
|
| 225 |
+
|
| 226 |
+
decryptor.finalize()
|
| 227 |
+
|
| 228 |
+
except InvalidTag as e:
|
| 229 |
+
raise IOError(f"File decryption failed. The authentication tag is invalid, suggesting the file was corrupted or tampered with. Error: {e}") from e
|
| 230 |
+
except FileNotFoundError as e:
|
| 231 |
+
raise ValueError(f"File not found: {e.filename}") from e
|
| 232 |
+
except Exception as e:
|
| 233 |
+
raise IOError(f"File decryption failed due to an unexpected error: {e}") from e
|
fundaments/postgresql.py
ADDED
|
@@ -0,0 +1,179 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments: A Secure Python Architecture
|
| 2 |
+
# Copyright 2008-2025 - Volkan Kücükbudak
|
| 3 |
+
# Apache License V. 2
|
| 4 |
+
# Repo: https://github.com/VolkanSah/PyFundaments
|
| 5 |
+
# fundaments/postgresql.py
|
| 6 |
+
import os
|
| 7 |
+
import logging
|
| 8 |
+
import asyncpg
|
| 9 |
+
import ssl
|
| 10 |
+
from urllib.parse import urlparse, urlencode, parse_qs, urlunparse
|
| 11 |
+
from typing import Optional
|
| 12 |
+
|
| 13 |
+
logging.basicConfig(
|
| 14 |
+
level=logging.INFO,
|
| 15 |
+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
| 16 |
+
)
|
| 17 |
+
logger = logging.getLogger(__name__)
|
| 18 |
+
|
| 19 |
+
_db_pool: Optional[asyncpg.Pool] = None
|
| 20 |
+
|
| 21 |
+
def enforce_cloud_security(dsn_url: str) -> str:
|
| 22 |
+
"""
|
| 23 |
+
Enforces security settings for cloud environments.
|
| 24 |
+
- Ensures SSL mode is at least 'require'
|
| 25 |
+
- Removes unsupported options for cloud providers (e.g. statement_timeout for Neon)
|
| 26 |
+
- Sets connect_timeout and keepalives_idle defaults
|
| 27 |
+
"""
|
| 28 |
+
parsed = urlparse(dsn_url)
|
| 29 |
+
query_params = parse_qs(parsed.query)
|
| 30 |
+
|
| 31 |
+
# Enforce SSL (at least 'require')
|
| 32 |
+
sslmode = query_params.get('sslmode', ['prefer'])[0].lower()
|
| 33 |
+
if sslmode not in ['require', 'verify-ca', 'verify-full']:
|
| 34 |
+
query_params['sslmode'] = ['require']
|
| 35 |
+
|
| 36 |
+
# Set timeouts and keep-alives if not present
|
| 37 |
+
if 'connect_timeout' not in query_params:
|
| 38 |
+
query_params['connect_timeout'] = ['5']
|
| 39 |
+
if 'keepalives_idle' not in query_params:
|
| 40 |
+
query_params['keepalives_idle'] = ['60']
|
| 41 |
+
|
| 42 |
+
# Remove statement_timeout option for Neon
|
| 43 |
+
if 'neon.tech' in parsed.netloc:
|
| 44 |
+
if 'options' in query_params:
|
| 45 |
+
options_clean = []
|
| 46 |
+
for opt in query_params['options']:
|
| 47 |
+
if 'statement_timeout' not in opt:
|
| 48 |
+
options_clean.append(opt)
|
| 49 |
+
if options_clean:
|
| 50 |
+
query_params['options'] = options_clean
|
| 51 |
+
else:
|
| 52 |
+
query_params.pop('options')
|
| 53 |
+
logger.info("Removed unsupported 'statement_timeout' option for Neon.tech.")
|
| 54 |
+
# Optionally, set a supported option for Neon (usually none)
|
| 55 |
+
|
| 56 |
+
# TODO: Extend here for further providers...
|
| 57 |
+
|
| 58 |
+
# Rebuild DSN
|
| 59 |
+
new_query = urlencode(query_params, doseq=True)
|
| 60 |
+
new_url = parsed._replace(query=new_query)
|
| 61 |
+
return urlunparse(new_url)
|
| 62 |
+
|
| 63 |
+
def mask_dsn(dsn_url: str) -> str:
|
| 64 |
+
"""
|
| 65 |
+
Masks username/password from DSN so they are not exposed in logs.
|
| 66 |
+
"""
|
| 67 |
+
parsed = urlparse(dsn_url)
|
| 68 |
+
safe_netloc = f"{parsed.hostname}:{parsed.port}" if parsed.port else parsed.hostname
|
| 69 |
+
return parsed._replace(netloc=safe_netloc).geturl()
|
| 70 |
+
|
| 71 |
+
async def ssl_runtime_check(conn: asyncpg.Connection):
|
| 72 |
+
"""
|
| 73 |
+
Performs a cloud-aware SSL runtime check on an active connection.
|
| 74 |
+
For Neon/Supabase (or unknown cloud) only log a warning if pg_stat_ssl is unavailable.
|
| 75 |
+
"""
|
| 76 |
+
dsn = os.getenv("DATABASE_URL", "")
|
| 77 |
+
try:
|
| 78 |
+
ssl_status = await conn.fetchval("""
|
| 79 |
+
SELECT CASE WHEN ssl THEN 'active' ELSE 'INACTIVE' END
|
| 80 |
+
FROM pg_stat_ssl WHERE pid = pg_backend_pid()
|
| 81 |
+
""")
|
| 82 |
+
if ssl_status != 'active':
|
| 83 |
+
logger.critical("CRITICAL ERROR: SSL connection is not active!")
|
| 84 |
+
raise RuntimeError("SSL connection failed")
|
| 85 |
+
logger.info("SSL connection is active.")
|
| 86 |
+
except Exception as e:
|
| 87 |
+
# Cloud: If pg_stat_ssl is not available, don't fail hard.
|
| 88 |
+
if "neon.tech" in dsn or "supabase" in dsn:
|
| 89 |
+
logger.warning("SSL check via pg_stat_ssl not possible (cloud restriction). Assuming SSL is active due to sslmode=require.")
|
| 90 |
+
else:
|
| 91 |
+
logger.critical(f"SSL runtime check failed: {e}")
|
| 92 |
+
raise
|
| 93 |
+
|
| 94 |
+
async def init_db_pool(dsn_url: Optional[str] = None) -> Optional[asyncpg.Pool]:
|
| 95 |
+
"""Initializes the asynchronous database connection pool."""
|
| 96 |
+
global _db_pool
|
| 97 |
+
if _db_pool:
|
| 98 |
+
return _db_pool
|
| 99 |
+
|
| 100 |
+
if not dsn_url:
|
| 101 |
+
dsn_url = os.getenv("DATABASE_URL") or os.getenv("PG_DSN")
|
| 102 |
+
if not dsn_url:
|
| 103 |
+
logger.warning("No DATABASE_URL or PG_DSN found. Skipping DB pool initialization.")
|
| 104 |
+
return None
|
| 105 |
+
|
| 106 |
+
# Enforce cloud security and remove unsupported options
|
| 107 |
+
secured_dsn = enforce_cloud_security(dsn_url)
|
| 108 |
+
|
| 109 |
+
# ⚠ WARNING: This logs full credentials — keep only for secure DEV debugging
|
| 110 |
+
logger.debug(f"[DEV ONLY] Full DSN used for DB connection: {secured_dsn}")
|
| 111 |
+
|
| 112 |
+
# Always log a masked DSN for production safety
|
| 113 |
+
logger.info(f"DSN used for DB connection (masked): {mask_dsn(secured_dsn)}")
|
| 114 |
+
|
| 115 |
+
ssl_context = None
|
| 116 |
+
if 'sslmode=verify-full' in secured_dsn:
|
| 117 |
+
ssl_context = ssl.create_default_context()
|
| 118 |
+
|
| 119 |
+
try:
|
| 120 |
+
logger.info("Initializing secure database pool...")
|
| 121 |
+
_db_pool = await asyncpg.create_pool(
|
| 122 |
+
dsn=secured_dsn,
|
| 123 |
+
min_size=1,
|
| 124 |
+
max_size=10,
|
| 125 |
+
timeout=5,
|
| 126 |
+
command_timeout=30,
|
| 127 |
+
ssl=ssl_context
|
| 128 |
+
)
|
| 129 |
+
# Post-init checks
|
| 130 |
+
async with _db_pool.acquire() as conn:
|
| 131 |
+
await ssl_runtime_check(conn)
|
| 132 |
+
logger.info("Secure database pool initialized.")
|
| 133 |
+
return _db_pool
|
| 134 |
+
except Exception as e:
|
| 135 |
+
logger.critical(f"Pool initialization failed: {str(e)}")
|
| 136 |
+
_db_pool = None
|
| 137 |
+
return None # Fallback: allow app to run without DB
|
| 138 |
+
|
| 139 |
+
async def close_db_pool():
|
| 140 |
+
"""Gracefully closes the database connection pool."""
|
| 141 |
+
global _db_pool
|
| 142 |
+
if _db_pool:
|
| 143 |
+
await _db_pool.close()
|
| 144 |
+
_db_pool = None
|
| 145 |
+
logger.info("Database pool closed successfully.")
|
| 146 |
+
|
| 147 |
+
async def execute_secured_query(query: str, *params, fetch_method='fetch'):
|
| 148 |
+
"""
|
| 149 |
+
Executes a parameterized query with integrated security checks.
|
| 150 |
+
"""
|
| 151 |
+
global _db_pool
|
| 152 |
+
if not _db_pool:
|
| 153 |
+
raise RuntimeError("Database pool not initialized")
|
| 154 |
+
|
| 155 |
+
try:
|
| 156 |
+
async with _db_pool.acquire() as conn:
|
| 157 |
+
if fetch_method == 'fetch':
|
| 158 |
+
return await conn.fetch(query, *params)
|
| 159 |
+
elif fetch_method == 'fetchrow':
|
| 160 |
+
return await conn.fetchrow(query, *params)
|
| 161 |
+
elif fetch_method == 'execute':
|
| 162 |
+
return await conn.execute(query, *params)
|
| 163 |
+
else:
|
| 164 |
+
raise ValueError("Invalid fetch_method")
|
| 165 |
+
except asyncpg.PostgresError as e:
|
| 166 |
+
error_type = "Security violation" if getattr(e, 'sqlstate', None) == '42501' else "Database error"
|
| 167 |
+
|
| 168 |
+
if os.getenv('APP_ENV') == 'production':
|
| 169 |
+
logger.error(f"{error_type} [Code: {getattr(e, 'sqlstate', '?')}]")
|
| 170 |
+
else:
|
| 171 |
+
logger.error(f"{error_type}: {e}")
|
| 172 |
+
|
| 173 |
+
# Neon: Reconnect if connection terminated (optional)
|
| 174 |
+
if getattr(e, 'sqlstate', None) == '08006' and 'neon.tech' in (os.getenv("DATABASE_URL") or ''):
|
| 175 |
+
logger.warning("Neon.tech connection terminated. Restarting pool...")
|
| 176 |
+
await close_db_pool()
|
| 177 |
+
await init_db_pool(os.getenv("DATABASE_URL"))
|
| 178 |
+
|
| 179 |
+
raise
|
fundaments/security.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments: A Secure Python Architecture
|
| 2 |
+
# Copyright 2008-2025 - Volkan Kücükbudak
|
| 3 |
+
# Apache License V. 2
|
| 4 |
+
# Repo: https://github.com/VolkanSah/PyFundaments
|
| 5 |
+
# fundaments/security.py
|
| 6 |
+
# A central security manager that orchestrates core security functions.
|
| 7 |
+
# This class acts as a single, trusted interface for the application to
|
| 8 |
+
# interact with all underlying security fundamentals.
|
| 9 |
+
|
| 10 |
+
import logging
|
| 11 |
+
from typing import Dict, Any, Optional
|
| 12 |
+
|
| 13 |
+
# VORHER: Diese direkten Imports sind überflüssig,
|
| 14 |
+
# da die Instanzen der Klassen über den 'services'-Dictionary
|
| 15 |
+
# im __init__-Konstruktor übergeben werden.
|
| 16 |
+
# Sie sind auch nicht für die Typisierung notwendig,
|
| 17 |
+
# da wir 'Optional[ClassName]' verwenden, was ausreicht.
|
| 18 |
+
# from fundaments.encryption import Encryption
|
| 19 |
+
# from fundaments.access_control import AccessControl
|
| 20 |
+
# from fundaments.postgresql import execute_secured_query
|
| 21 |
+
# from fundaments.user_handler import UserHandler
|
| 22 |
+
|
| 23 |
+
logger = logging.getLogger('security')
|
| 24 |
+
|
| 25 |
+
class Security:
|
| 26 |
+
def __init__(self, services: Dict[str, Any]):
|
| 27 |
+
# VORHER: Hier werden die Instanzen aus dem übergebenen Dictionary entnommen.
|
| 28 |
+
self.user_handler: Optional[UserHandler] = services.get("user_handler")
|
| 29 |
+
self.access_control: Optional[AccessControl] = services.get("access_control")
|
| 30 |
+
self.encryption: Optional[Encryption] = services.get("encryption")
|
| 31 |
+
|
| 32 |
+
# VORHER: Bei fehlenden kritischen Diensten wird die Anwendung sofort gestoppt.
|
| 33 |
+
# Dies ist eine strikte, aber unflexible Regel.
|
| 34 |
+
if not self.user_handler:
|
| 35 |
+
logger.critical("Security manager init failed: UserHandler service missing.")
|
| 36 |
+
raise RuntimeError("UserHandler service missing")
|
| 37 |
+
|
| 38 |
+
if not self.access_control:
|
| 39 |
+
logger.critical("Security manager init failed: AccessControl service missing.")
|
| 40 |
+
raise RuntimeError("AccessControl service missing")
|
| 41 |
+
|
| 42 |
+
# VORHER: Bei einem nicht-kritischen Dienst wird nur eine Warnung ausgegeben.
|
| 43 |
+
# Die Anwendung läuft weiter.
|
| 44 |
+
if not self.encryption:
|
| 45 |
+
logger.warning("Encryption service not available. Encryption/decryption features will be disabled.")
|
| 46 |
+
|
| 47 |
+
logger.info("Security manager initialized and ready.")
|
| 48 |
+
|
| 49 |
+
# VORHER: Hier fehlt die Überprüfung, ob 'self.user_handler' None ist.
|
| 50 |
+
# Wenn der Dienst nicht initialisiert wurde (wie oben), würde dies einen AttributeError auslösen.
|
| 51 |
+
async def user_login(self, username: str, password: str, request_data: dict) -> bool:
|
| 52 |
+
logger.info(f"Attempting login for user: {username}")
|
| 53 |
+
# UserHandler.login() is synchronous (SQLite) — no await.
|
| 54 |
+
# The old 'await' raised TypeError: bool can't be awaited.
|
| 55 |
+
if self.user_handler.login(username, password, request_data):
|
| 56 |
+
return self.user_handler.validate_session(request_data)
|
| 57 |
+
return False
|
| 58 |
+
|
| 59 |
+
# VORHER: Hier fehlt die Überprüfung, ob 'self.access_control' None ist.
|
| 60 |
+
async def check_permission(self, user_id: int, permission_name: str) -> bool:
|
| 61 |
+
"""Checks permission for ANY user_id (not just the constructor one)."""
|
| 62 |
+
logger.debug(f"Checking permission '{permission_name}' for user ID {user_id}")
|
| 63 |
+
return await self.access_control.has_permission(permission_name, user_id=user_id)
|
| 64 |
+
|
| 65 |
+
# VORHER: Hier wird geprüft, ob 'self.encryption' None ist, was korrekt ist.
|
| 66 |
+
def encrypt_data(self, data: str) -> Dict[str, str]:
|
| 67 |
+
if not self.encryption:
|
| 68 |
+
raise RuntimeError("Encryption service not initialized.")
|
| 69 |
+
logger.debug("Encrypting data.")
|
| 70 |
+
return self.encryption.encrypt(data)
|
| 71 |
+
|
| 72 |
+
# VORHER: Hier wird geprüft, ob 'self.encryption' None ist, was korrekt ist.
|
| 73 |
+
def decrypt_data(self, encrypted_data: str, nonce: str, tag: str) -> Optional[str]:
|
| 74 |
+
if not self.encryption:
|
| 75 |
+
logger.error("Encryption service not initialized. Cannot decrypt data.")
|
| 76 |
+
return None
|
| 77 |
+
logger.debug("Decrypting data.")
|
| 78 |
+
try:
|
| 79 |
+
return self.encryption.decrypt(encrypted_data, nonce, tag)
|
| 80 |
+
except Exception as e:
|
| 81 |
+
logger.error(f"Decryption failed: {e}")
|
| 82 |
+
return None
|
fundaments/user_handler.py
ADDED
|
@@ -0,0 +1,370 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments: A Secure Python Architecture
|
| 2 |
+
# Copyright 2008-2025 - Volkan Kücükbudak
|
| 3 |
+
# Apache License V. 2
|
| 4 |
+
# Repo: https://github.com/VolkanSah/PyFundaments
|
| 5 |
+
# root/fundaments/user_handler.py
|
| 6 |
+
# A Python module for handling user authentication and session management.
|
| 7 |
+
|
| 8 |
+
import sqlite3
|
| 9 |
+
import uuid
|
| 10 |
+
from datetime import datetime, timedelta
|
| 11 |
+
from passlib.hash import pbkdf2_sha256
|
| 12 |
+
import os
|
| 13 |
+
|
| 14 |
+
|
| 15 |
+
class Database:
|
| 16 |
+
"""
|
| 17 |
+
Handles the SQLite database connection and initialization.
|
| 18 |
+
Supports dynamic path selection via environment variables with a
|
| 19 |
+
fallback to the local application directory.
|
| 20 |
+
"""
|
| 21 |
+
def __init__(self, db_name="cms_database.db"):
|
| 22 |
+
# 1. Attempt to load the database path from an environment variable
|
| 23 |
+
# This allows for flexible configuration in production/Docker environments
|
| 24 |
+
env_path = os.getenv("SQLITE_PATH")
|
| 25 |
+
|
| 26 |
+
if env_path:
|
| 27 |
+
# Use the absolute path provided by the environment variable
|
| 28 |
+
full_db_path = os.path.abspath(env_path)
|
| 29 |
+
else:
|
| 30 |
+
# Fallback logic: Locate the 'app' directory relative to this script
|
| 31 |
+
# Expected structure: root/fun/user_handler.py -> root/app/
|
| 32 |
+
base_path = os.path.dirname(os.path.abspath(__file__))
|
| 33 |
+
app_dir = os.path.join(base_path, "..", "app")
|
| 34 |
+
full_db_path = os.path.join(app_dir, db_name)
|
| 35 |
+
|
| 36 |
+
# 2. Ensure the target directory exists before attempting to connect
|
| 37 |
+
# SQLite can create the file, but not the parent folders.
|
| 38 |
+
db_dir = os.path.dirname(full_db_path)
|
| 39 |
+
if db_dir and not os.path.exists(db_dir):
|
| 40 |
+
os.makedirs(db_dir)
|
| 41 |
+
|
| 42 |
+
# Initialize the connection and cursor
|
| 43 |
+
self.conn = sqlite3.connect(full_db_path)
|
| 44 |
+
self.cursor = self.conn.cursor()
|
| 45 |
+
|
| 46 |
+
# Log the active database path for debugging purposes
|
| 47 |
+
print(f"Database connected to: {full_db_path}")
|
| 48 |
+
|
| 49 |
+
def execute(self, query, params=None):
|
| 50 |
+
if params is None:
|
| 51 |
+
params = []
|
| 52 |
+
self.cursor.execute(query, params)
|
| 53 |
+
self.conn.commit()
|
| 54 |
+
|
| 55 |
+
def fetchone(self, query, params=None):
|
| 56 |
+
if params is None:
|
| 57 |
+
params = []
|
| 58 |
+
self.cursor.execute(query, params)
|
| 59 |
+
return self.cursor.fetchone()
|
| 60 |
+
|
| 61 |
+
def fetchall(self, query, params=None):
|
| 62 |
+
if params is None:
|
| 63 |
+
params = []
|
| 64 |
+
self.cursor.execute(query, params)
|
| 65 |
+
return self.cursor.fetchall()
|
| 66 |
+
|
| 67 |
+
def close(self):
|
| 68 |
+
self.conn.close()
|
| 69 |
+
|
| 70 |
+
def setup_tables(self):
|
| 71 |
+
"""
|
| 72 |
+
Creates the necessary tables for users and sessions.
|
| 73 |
+
"""
|
| 74 |
+
self.execute("""
|
| 75 |
+
CREATE TABLE IF NOT EXISTS users (
|
| 76 |
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
| 77 |
+
username TEXT UNIQUE NOT NULL,
|
| 78 |
+
password TEXT NOT NULL,
|
| 79 |
+
is_admin INTEGER NOT NULL DEFAULT 0,
|
| 80 |
+
account_locked INTEGER NOT NULL DEFAULT 0,
|
| 81 |
+
failed_login_attempts INTEGER NOT NULL DEFAULT 0
|
| 82 |
+
)
|
| 83 |
+
""")
|
| 84 |
+
self.execute("""
|
| 85 |
+
CREATE TABLE IF NOT EXISTS sessions (
|
| 86 |
+
id TEXT PRIMARY KEY,
|
| 87 |
+
user_id INTEGER NOT NULL,
|
| 88 |
+
ip_address TEXT,
|
| 89 |
+
user_agent TEXT,
|
| 90 |
+
last_activity TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
| 91 |
+
FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE
|
| 92 |
+
)
|
| 93 |
+
""")
|
| 94 |
+
|
| 95 |
+
class Security:
|
| 96 |
+
"""
|
| 97 |
+
Handles secure password hashing and session regeneration.
|
| 98 |
+
Using passlib for robust and secure password management.
|
| 99 |
+
"""
|
| 100 |
+
@staticmethod
|
| 101 |
+
def hash_password(password: str) -> str:
|
| 102 |
+
"""Hashes a password using PBKDF2 with SHA256."""
|
| 103 |
+
return pbkdf2_sha256.hash(password)
|
| 104 |
+
|
| 105 |
+
@staticmethod
|
| 106 |
+
def verify_password(password: str, hashed_password: str) -> bool:
|
| 107 |
+
"""Verifies a password against a stored hash."""
|
| 108 |
+
return pbkdf2_sha256.verify(password, hashed_password)
|
| 109 |
+
|
| 110 |
+
@staticmethod
|
| 111 |
+
def regenerate_session(session_id: str):
|
| 112 |
+
"""
|
| 113 |
+
Simulates regenerating a session ID to prevent session fixation.
|
| 114 |
+
In a real web framework, this would be a framework-specific function.
|
| 115 |
+
"""
|
| 116 |
+
print(f"Session regenerated. Old ID: {session_id}")
|
| 117 |
+
new_session_id = str(uuid.uuid4())
|
| 118 |
+
print(f"New ID: {new_session_id}")
|
| 119 |
+
return new_session_id
|
| 120 |
+
|
| 121 |
+
class UserHandler:
|
| 122 |
+
"""
|
| 123 |
+
Handles user login, logout, and session validation.
|
| 124 |
+
This class mirrors the logic from the user's PHP User class.
|
| 125 |
+
"""
|
| 126 |
+
def __init__(self, db: Database):
|
| 127 |
+
self.db = db
|
| 128 |
+
# A simple in-memory session store for this example
|
| 129 |
+
self._session = {}
|
| 130 |
+
|
| 131 |
+
def login(self, username: str, password: str, request_data: dict) -> bool:
|
| 132 |
+
"""
|
| 133 |
+
Logs in the user by verifying credentials and storing a new session.
|
| 134 |
+
:param username: The user's username.
|
| 135 |
+
:param password: The user's plain-text password.
|
| 136 |
+
:param request_data: A dictionary containing 'ip_address' and 'user_agent'.
|
| 137 |
+
:return: True if login is successful, False otherwise.
|
| 138 |
+
"""
|
| 139 |
+
try:
|
| 140 |
+
# Step 1: Find the user in the database
|
| 141 |
+
user_data = self.db.fetchone("SELECT id, username, password, is_admin, account_locked, failed_login_attempts FROM users WHERE username = ?", (username,))
|
| 142 |
+
if user_data is None:
|
| 143 |
+
print(f"Login failed: Username '{username}' not found.")
|
| 144 |
+
return False
|
| 145 |
+
|
| 146 |
+
user = {
|
| 147 |
+
'id': user_data[0],
|
| 148 |
+
'username': user_data[1],
|
| 149 |
+
'password': user_data[2],
|
| 150 |
+
'is_admin': user_data[3],
|
| 151 |
+
'account_locked': user_data[4],
|
| 152 |
+
'failed_login_attempts': user_data[5]
|
| 153 |
+
}
|
| 154 |
+
|
| 155 |
+
# Check if account is locked
|
| 156 |
+
if user['account_locked'] == 1:
|
| 157 |
+
print(f"Login failed: Account for '{username}' is locked.")
|
| 158 |
+
return False
|
| 159 |
+
|
| 160 |
+
# Step 2: Verify the password
|
| 161 |
+
if Security.verify_password(password, user['password']):
|
| 162 |
+
print(f"Login successful for user: '{username}'")
|
| 163 |
+
|
| 164 |
+
# Reset failed login attempts on success
|
| 165 |
+
self.reset_failed_attempts(username)
|
| 166 |
+
|
| 167 |
+
# Step 3: Create a new session record in the database
|
| 168 |
+
session_id = str(uuid.uuid4())
|
| 169 |
+
ip_address = request_data.get('ip_address', 'unknown')
|
| 170 |
+
user_agent = request_data.get('user_agent', 'unknown')
|
| 171 |
+
|
| 172 |
+
self.db.execute(
|
| 173 |
+
"INSERT INTO sessions (id, user_id, ip_address, user_agent) VALUES (?, ?, ?, ?)",
|
| 174 |
+
(session_id, user['id'], ip_address, user_agent)
|
| 175 |
+
)
|
| 176 |
+
|
| 177 |
+
# Step 4: Store session data in the in-memory session (or a session store)
|
| 178 |
+
self._session = {
|
| 179 |
+
'session_id': session_id,
|
| 180 |
+
'user_id': user['id'],
|
| 181 |
+
'username': user['username'],
|
| 182 |
+
'is_admin': user['is_admin']
|
| 183 |
+
}
|
| 184 |
+
|
| 185 |
+
# Security: Regenerate session ID
|
| 186 |
+
self._session['session_id'] = Security.regenerate_session(session_id)
|
| 187 |
+
return True
|
| 188 |
+
else:
|
| 189 |
+
print(f"Login failed: Incorrect password for user '{username}'.")
|
| 190 |
+
# Increment failed login attempts
|
| 191 |
+
self.increment_failed_attempts(username)
|
| 192 |
+
return False
|
| 193 |
+
|
| 194 |
+
except sqlite3.Error as e:
|
| 195 |
+
print(f"Database error during login: {e}")
|
| 196 |
+
return False
|
| 197 |
+
|
| 198 |
+
def logout(self) -> bool:
|
| 199 |
+
"""
|
| 200 |
+
Logs out the current user by deleting the session from the database.
|
| 201 |
+
:return: True if logout is successful, False otherwise.
|
| 202 |
+
"""
|
| 203 |
+
if 'user_id' not in self._session:
|
| 204 |
+
print("No active session to log out.")
|
| 205 |
+
return False
|
| 206 |
+
|
| 207 |
+
try:
|
| 208 |
+
# Step 1: Delete the session from the database
|
| 209 |
+
self.db.execute("DELETE FROM sessions WHERE user_id = ?", (self._session['user_id'],))
|
| 210 |
+
|
| 211 |
+
# Step 2: Clear the in-memory session data
|
| 212 |
+
self._session.clear()
|
| 213 |
+
print("User logged out successfully.")
|
| 214 |
+
return True
|
| 215 |
+
except sqlite3.Error as e:
|
| 216 |
+
print(f"Database error during logout: {e}")
|
| 217 |
+
return False
|
| 218 |
+
|
| 219 |
+
def is_logged_in(self) -> bool:
|
| 220 |
+
"""
|
| 221 |
+
Checks if the current user is logged in.
|
| 222 |
+
:return: True if a valid session exists, False otherwise.
|
| 223 |
+
"""
|
| 224 |
+
if 'user_id' not in self._session:
|
| 225 |
+
return False
|
| 226 |
+
|
| 227 |
+
try:
|
| 228 |
+
# Check for the session in the database
|
| 229 |
+
session_data = self.db.fetchone(
|
| 230 |
+
"SELECT * FROM sessions WHERE id = ? AND user_id = ?",
|
| 231 |
+
(self._session['session_id'], self._session['user_id'])
|
| 232 |
+
)
|
| 233 |
+
return session_data is not None
|
| 234 |
+
except sqlite3.Error as e:
|
| 235 |
+
print(f"Database error during is_logged_in check: {e}")
|
| 236 |
+
return False
|
| 237 |
+
|
| 238 |
+
def is_admin(self) -> bool:
|
| 239 |
+
"""
|
| 240 |
+
Checks if the logged-in user is an admin.
|
| 241 |
+
:return: True if the user is an admin, False otherwise.
|
| 242 |
+
"""
|
| 243 |
+
return self._session.get('is_admin', 0) == 1
|
| 244 |
+
|
| 245 |
+
def validate_session(self, request_data: dict) -> bool:
|
| 246 |
+
"""
|
| 247 |
+
Validates the current session against IP address and user agent.
|
| 248 |
+
:param request_data: A dictionary containing 'ip_address' and 'user_agent'.
|
| 249 |
+
:return: True if the session is valid, False otherwise.
|
| 250 |
+
"""
|
| 251 |
+
if not self.is_logged_in():
|
| 252 |
+
return False
|
| 253 |
+
|
| 254 |
+
try:
|
| 255 |
+
ip_address = request_data.get('ip_address', 'unknown')
|
| 256 |
+
user_agent = request_data.get('user_agent', 'unknown')
|
| 257 |
+
|
| 258 |
+
session_data = self.db.fetchone(
|
| 259 |
+
"SELECT * FROM sessions WHERE id = ? AND user_id = ? AND ip_address = ? AND user_agent = ?",
|
| 260 |
+
(self._session['session_id'], self._session['user_id'], ip_address, user_agent)
|
| 261 |
+
)
|
| 262 |
+
return session_data is not None
|
| 263 |
+
except sqlite3.Error as e:
|
| 264 |
+
print(f"Database error during session validation: {e}")
|
| 265 |
+
return False
|
| 266 |
+
|
| 267 |
+
def lock_account(self, username: str):
|
| 268 |
+
"""
|
| 269 |
+
Locks a user account.
|
| 270 |
+
:param username: The username of the account to lock.
|
| 271 |
+
"""
|
| 272 |
+
try:
|
| 273 |
+
self.db.execute("UPDATE users SET account_locked = 1 WHERE username = ?", (username,))
|
| 274 |
+
print(f"Account for '{username}' has been locked.")
|
| 275 |
+
except sqlite3.Error as e:
|
| 276 |
+
print(f"Database error while locking account: {e}")
|
| 277 |
+
|
| 278 |
+
def reset_failed_attempts(self, username: str):
|
| 279 |
+
"""
|
| 280 |
+
Resets failed login attempts for a user.
|
| 281 |
+
:param username: The username of the account.
|
| 282 |
+
"""
|
| 283 |
+
try:
|
| 284 |
+
self.db.execute("UPDATE users SET failed_login_attempts = 0 WHERE username = ?", (username,))
|
| 285 |
+
except sqlite3.Error as e:
|
| 286 |
+
print(f"Database error while resetting failed attempts: {e}")
|
| 287 |
+
|
| 288 |
+
def increment_failed_attempts(self, username: str):
|
| 289 |
+
"""
|
| 290 |
+
Increments failed login attempts and locks the account if a threshold is met.
|
| 291 |
+
:param username: The username of the account.
|
| 292 |
+
"""
|
| 293 |
+
try:
|
| 294 |
+
# Get the current failed attempts
|
| 295 |
+
user_data = self.db.fetchone("SELECT failed_login_attempts FROM users WHERE username = ?", (username,))
|
| 296 |
+
if user_data:
|
| 297 |
+
attempts = user_data[0] + 1
|
| 298 |
+
self.db.execute(
|
| 299 |
+
"UPDATE users SET failed_login_attempts = ? WHERE username = ?",
|
| 300 |
+
(attempts, username)
|
| 301 |
+
)
|
| 302 |
+
print(f"Failed login attempts for '{username}': {attempts}")
|
| 303 |
+
|
| 304 |
+
# Check for threshold (e.g., 5 attempts)
|
| 305 |
+
if attempts >= 5:
|
| 306 |
+
self.lock_account(username)
|
| 307 |
+
|
| 308 |
+
except sqlite3.Error as e:
|
| 309 |
+
print(f"Database error while incrementing failed attempts: {e}")
|
| 310 |
+
|
| 311 |
+
# --- Example Usage ---
|
| 312 |
+
if __name__ == "__main__":
|
| 313 |
+
db = Database()
|
| 314 |
+
db.setup_tables()
|
| 315 |
+
user_handler = UserHandler(db)
|
| 316 |
+
|
| 317 |
+
# Clean up old test data if it exists
|
| 318 |
+
db.execute("DELETE FROM users WHERE username IN (?, ?)", ("testuser", "adminuser"))
|
| 319 |
+
db.execute("DELETE FROM sessions")
|
| 320 |
+
|
| 321 |
+
# 1. Register a new user and an admin user
|
| 322 |
+
hashed_password = Security.hash_password("secure_password_123")
|
| 323 |
+
db.execute("INSERT INTO users (username, password) VALUES (?, ?)", ("testuser", hashed_password))
|
| 324 |
+
db.execute("INSERT INTO users (username, password, is_admin) VALUES (?, ?, 1)", ("adminuser", hashed_password))
|
| 325 |
+
|
| 326 |
+
print("--- Test 1: Successful Login ---")
|
| 327 |
+
# Simulate a web request
|
| 328 |
+
request_data = {
|
| 329 |
+
'ip_address': '192.168.1.100',
|
| 330 |
+
'user_agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
| 331 |
+
}
|
| 332 |
+
|
| 333 |
+
login_success = user_handler.login("testuser", "secure_password_123", request_data)
|
| 334 |
+
print(f"Login attempt status: {login_success}")
|
| 335 |
+
print(f"Is user logged in? {user_handler.is_logged_in()}")
|
| 336 |
+
print(f"Is user an admin? {user_handler.is_admin()}")
|
| 337 |
+
print(f"Is session valid? {user_handler.validate_session(request_data)}")
|
| 338 |
+
print("-" * 20)
|
| 339 |
+
|
| 340 |
+
# 2. Simulate a logout
|
| 341 |
+
print("--- Test 2: Logout ---")
|
| 342 |
+
user_handler.logout()
|
| 343 |
+
print(f"Is user logged in after logout? {user_handler.is_logged_in()}")
|
| 344 |
+
print("-" * 20)
|
| 345 |
+
|
| 346 |
+
# 3. Simulate a failed login and account lock
|
| 347 |
+
print("--- Test 3: Failed Login and Account Lock ---")
|
| 348 |
+
# Log in with the wrong password multiple times
|
| 349 |
+
for i in range(6):
|
| 350 |
+
user_handler.login("testuser", "wrong_password", request_data)
|
| 351 |
+
|
| 352 |
+
# Now, try to log in with the correct password. It should fail because the account is locked.
|
| 353 |
+
print("\nAttempting to log in with correct password after lock:")
|
| 354 |
+
login_attempt_after_lock = user_handler.login("testuser", "secure_password_123", request_data)
|
| 355 |
+
print(f"Login attempt status: {login_attempt_after_lock}")
|
| 356 |
+
print("-" * 20)
|
| 357 |
+
|
| 358 |
+
# 4. Reset failed attempts for a new login
|
| 359 |
+
print("--- Test 4: Resetting failed attempts ---")
|
| 360 |
+
user_handler.reset_failed_attempts("testuser")
|
| 361 |
+
login_attempt_after_reset = user_handler.login("testuser", "secure_password_123", request_data)
|
| 362 |
+
print(f"Login attempt status after reset: {login_attempt_after_reset}")
|
| 363 |
+
|
| 364 |
+
db.close()
|
| 365 |
+
|
| 366 |
+
# Optional: Clean up the database file after the run
|
| 367 |
+
# OLD
|
| 368 |
+
# os.remove("cms_database.db")
|
| 369 |
+
# NEW : Clean up the database file after the run:
|
| 370 |
+
# os.remove(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "app", "cms_database.db"))
|
main.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments: A Secure Python Architecture
|
| 2 |
+
# Copyright 2008-2025 - Volkan Kücükbudak
|
| 3 |
+
# Apache License V. 2
|
| 4 |
+
# Repo: https://github.com/VolkanSah/PyFundaments
|
| 5 |
+
# main.py
|
| 6 |
+
# This is the main entry point of the application.
|
| 7 |
+
# It now handles asynchronous initialization of the fundament modules.
|
| 8 |
+
import sys
|
| 9 |
+
import logging
|
| 10 |
+
import asyncio
|
| 11 |
+
import os
|
| 12 |
+
from typing import Dict, Any, Optional
|
| 13 |
+
|
| 14 |
+
import importlib.util
|
| 15 |
+
import datetime
|
| 16 |
+
|
| 17 |
+
if 'fundaments' in sys.modules:
|
| 18 |
+
del sys.modules['fundaments']
|
| 19 |
+
|
| 20 |
+
# We import our core modules from the "fundaments" directory.
|
| 21 |
+
try:
|
| 22 |
+
from fundaments.config_handler import config_service
|
| 23 |
+
from fundaments.postgresql import init_db_pool, close_db_pool
|
| 24 |
+
from fundaments.encryption import Encryption
|
| 25 |
+
from fundaments.access_control import AccessControl
|
| 26 |
+
from fundaments.user_handler import UserHandler
|
| 27 |
+
from fundaments.security import Security
|
| 28 |
+
from fundaments.debug import PyFundamentsDebug
|
| 29 |
+
except ImportError as e:
|
| 30 |
+
print(f"Error: Failed to import a fundament module: {e}")
|
| 31 |
+
print("Please ensure the modules and dependencies are present.")
|
| 32 |
+
sys.exit(1)
|
| 33 |
+
# Debug run
|
| 34 |
+
debug = PyFundamentsDebug()
|
| 35 |
+
debug.run()
|
| 36 |
+
|
| 37 |
+
# Logger configuration - conditional based on ENV
|
| 38 |
+
log_level = os.getenv('LOG_LEVEL', 'INFO').upper()
|
| 39 |
+
log_to_tmp = os.getenv('LOG_TO_TMP', 'false').lower() == 'true'
|
| 40 |
+
enable_public_logs = os.getenv('ENABLE_PUBLIC_LOGS', 'true').lower() == 'true'
|
| 41 |
+
|
| 42 |
+
if enable_public_logs:
|
| 43 |
+
if log_to_tmp:
|
| 44 |
+
log_file = '/tmp/pyfundaments.log'
|
| 45 |
+
logging.basicConfig(
|
| 46 |
+
level=getattr(logging, log_level),
|
| 47 |
+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
|
| 48 |
+
handlers=[
|
| 49 |
+
logging.FileHandler(log_file),
|
| 50 |
+
logging.StreamHandler()
|
| 51 |
+
]
|
| 52 |
+
)
|
| 53 |
+
else:
|
| 54 |
+
logging.basicConfig(
|
| 55 |
+
level=getattr(logging, log_level),
|
| 56 |
+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
| 57 |
+
)
|
| 58 |
+
else:
|
| 59 |
+
# Silent mode - only critical errors
|
| 60 |
+
logging.basicConfig(level=logging.CRITICAL)
|
| 61 |
+
|
| 62 |
+
logger = logging.getLogger('main_app_loader')
|
| 63 |
+
|
| 64 |
+
async def initialize_fundaments() -> Dict[str, Any]:
|
| 65 |
+
"""
|
| 66 |
+
Initializes core application services conditionally based on available ENV variables.
|
| 67 |
+
Only loads services for which the required configuration is present.
|
| 68 |
+
"""
|
| 69 |
+
logger.info("Starting conditional initialization of fundament modules...")
|
| 70 |
+
|
| 71 |
+
fundaments = {
|
| 72 |
+
"config": config_service
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
# --- Database Initialization (PostgreSQL) ---
|
| 76 |
+
# Only initialize if DATABASE_URL is available
|
| 77 |
+
database_url = config_service.get("DATABASE_URL")
|
| 78 |
+
if database_url and database_url != "your_database_dsn_here":
|
| 79 |
+
try:
|
| 80 |
+
db_service = await init_db_pool(database_url)
|
| 81 |
+
if db_service is not None:
|
| 82 |
+
fundaments["db"] = db_service
|
| 83 |
+
logger.info("Database service initialized.")
|
| 84 |
+
else:
|
| 85 |
+
# init_db_pool() returns None instead of raising on failure
|
| 86 |
+
logger.warning("Database pool could not be built, continuing without DB.")
|
| 87 |
+
fundaments["db"] = None
|
| 88 |
+
except Exception as e:
|
| 89 |
+
logger.warning(f"Database initialization failed, continuing without DB: {e}")
|
| 90 |
+
fundaments["db"] = None
|
| 91 |
+
else:
|
| 92 |
+
logger.info("No valid DATABASE_URL found, skipping database initialization.")
|
| 93 |
+
fundaments["db"] = None
|
| 94 |
+
|
| 95 |
+
# --- Encryption Initialization ---
|
| 96 |
+
# Only initialize if both encryption keys are available
|
| 97 |
+
master_key = config_service.get("MASTER_ENCRYPTION_KEY")
|
| 98 |
+
persistent_salt = config_service.get("PERSISTENT_ENCRYPTION_SALT")
|
| 99 |
+
|
| 100 |
+
if master_key and persistent_salt and master_key != "your_256_bit_key_here":
|
| 101 |
+
try:
|
| 102 |
+
encryption_service = Encryption(master_key=master_key, salt=persistent_salt)
|
| 103 |
+
fundaments["encryption"] = encryption_service
|
| 104 |
+
logger.info("Encryption service initialized.")
|
| 105 |
+
except Exception as e:
|
| 106 |
+
logger.warning(f"Encryption initialization failed, continuing without encryption: {e}")
|
| 107 |
+
fundaments["encryption"] = None
|
| 108 |
+
else:
|
| 109 |
+
logger.info("Encryption keys not found or using defaults, skipping encryption initialization.")
|
| 110 |
+
fundaments["encryption"] = None
|
| 111 |
+
|
| 112 |
+
# --- Access Control Initialization ---
|
| 113 |
+
# Only initialize if we have a database connection
|
| 114 |
+
if fundaments["db"] is not None:
|
| 115 |
+
try:
|
| 116 |
+
access_control_service = AccessControl()
|
| 117 |
+
fundaments["access_control"] = access_control_service
|
| 118 |
+
logger.info("Access Control service initialized.")
|
| 119 |
+
except Exception as e:
|
| 120 |
+
logger.warning(f"Access Control initialization failed: {e}")
|
| 121 |
+
fundaments["access_control"] = None
|
| 122 |
+
else:
|
| 123 |
+
logger.info("No database available, skipping Access Control initialization.")
|
| 124 |
+
fundaments["access_control"] = None
|
| 125 |
+
|
| 126 |
+
# --- User Handler Initialization ---
|
| 127 |
+
# UserHandler is SQLite-based (own Database class, see user_handler.py)
|
| 128 |
+
# and does NOT use the asyncpg pool — wiring it with the Postgres pool
|
| 129 |
+
# was a bug (pool has no fetchone/execute-sync interface).
|
| 130 |
+
# SQLite path: SQLITE_PATH env or fallback next to app/ (HF Spaces: read-only FS
|
| 131 |
+
# may fail — caught below, app continues without user_handler).
|
| 132 |
+
try:
|
| 133 |
+
from fundaments.user_handler import Database as UserDB
|
| 134 |
+
user_db = UserDB()
|
| 135 |
+
user_db.setup_tables()
|
| 136 |
+
user_handler_service = UserHandler(user_db)
|
| 137 |
+
fundaments["user_handler"] = user_handler_service
|
| 138 |
+
logger.info("User Handler service initialized (SQLite store).")
|
| 139 |
+
except Exception as e:
|
| 140 |
+
logger.warning(f"User Handler initialization failed: {e}")
|
| 141 |
+
fundaments["user_handler"] = None
|
| 142 |
+
|
| 143 |
+
# --- Security Manager Initialization ---
|
| 144 |
+
# Only initialize if we have the required sub-services
|
| 145 |
+
available_services = {k: v for k, v in fundaments.items() if v is not None and k != "config"}
|
| 146 |
+
|
| 147 |
+
if len(available_services) >= 1: # At least one service beyond config
|
| 148 |
+
try:
|
| 149 |
+
# Filter out None services for Security manager
|
| 150 |
+
fundament_services = {
|
| 151 |
+
k: v for k, v in {
|
| 152 |
+
"user_handler": fundaments.get("user_handler"),
|
| 153 |
+
"access_control": fundaments.get("access_control"),
|
| 154 |
+
"encryption": fundaments.get("encryption")
|
| 155 |
+
}.items() if v is not None
|
| 156 |
+
}
|
| 157 |
+
|
| 158 |
+
if fundament_services: # Only if we have actual services
|
| 159 |
+
security_service = Security(fundament_services)
|
| 160 |
+
fundaments["security"] = security_service
|
| 161 |
+
logger.info("Security manager initialized.")
|
| 162 |
+
else:
|
| 163 |
+
logger.info("No services available for Security manager, skipping initialization.")
|
| 164 |
+
fundaments["security"] = None
|
| 165 |
+
except Exception as e:
|
| 166 |
+
logger.warning(f"Security manager initialization failed: {e}")
|
| 167 |
+
fundaments["security"] = None
|
| 168 |
+
else:
|
| 169 |
+
logger.info("Insufficient services for Security manager, skipping initialization.")
|
| 170 |
+
fundaments["security"] = None
|
| 171 |
+
|
| 172 |
+
# Log what was actually initialized
|
| 173 |
+
initialized_services = [k for k, v in fundaments.items() if v is not None]
|
| 174 |
+
logger.info(f"Successfully initialized services: {', '.join(initialized_services)}")
|
| 175 |
+
|
| 176 |
+
return fundaments
|
| 177 |
+
|
| 178 |
+
async def main():
|
| 179 |
+
"""
|
| 180 |
+
The main asynchronous function of the application.
|
| 181 |
+
"""
|
| 182 |
+
logger.info("Starting main.py...")
|
| 183 |
+
|
| 184 |
+
fundaments = await initialize_fundaments()
|
| 185 |
+
|
| 186 |
+
try:
|
| 187 |
+
# Load the actual app logic here.
|
| 188 |
+
# This is where your 'app/app.py' would be imported and run.
|
| 189 |
+
logger.info("Fundament modules are ready for the app logic.")
|
| 190 |
+
# Example:
|
| 191 |
+
from app.app import start_application
|
| 192 |
+
await start_application(fundaments)
|
| 193 |
+
|
| 194 |
+
finally:
|
| 195 |
+
# Ensure the database pool is closed gracefully on exit (if it was initialized)
|
| 196 |
+
if fundaments.get("db") is not None:
|
| 197 |
+
await close_db_pool()
|
| 198 |
+
logger.info("Database pool closed.")
|
| 199 |
+
logger.info("Application shut down.")
|
| 200 |
+
|
| 201 |
+
if __name__ == "__main__":
|
| 202 |
+
asyncio.run(main())
|
requirements.txt
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# PyFundaments Core Dependencies (always needed)
|
| 2 |
+
asyncpg
|
| 3 |
+
python-dotenv
|
| 4 |
+
passlib
|
| 5 |
+
cryptography
|
| 6 |
+
|
| 7 |
+
# MCP Hub Dependencies (needed for app/mcp.py)
|
| 8 |
+
# Code imports mcp.server.fastmcp — the mcp SDK, NOT the fastmcp package.
|
| 9 |
+
mcp>=1.10,<2
|
| 10 |
+
httpx
|
| 11 |
+
quart
|
| 12 |
+
hypercorn
|
| 13 |
+
|
| 14 |
+
# Internal SQLite IPC (needed for app/db_sync.py)
|
| 15 |
+
aiosqlite
|
| 16 |
+
|
| 17 |
+
# HTTP client operations (API calls, webhooks)
|
| 18 |
+
requests
|
| 19 |
+
|
| 20 |
+
# Additional dependencies for specific app types:
|
| 21 |
+
# Uncomment what your application needs
|
| 22 |
+
|
| 23 |
+
# File operations (ML, data processing, file uploads)
|
| 24 |
+
#aiofiles
|
| 25 |
+
|
| 26 |
+
# Web applications (Flask-based apps, APIs)
|
| 27 |
+
#Flask[async]
|
| 28 |
+
|
| 29 |
+
# Discord bots
|
| 30 |
+
#discord.py
|
| 31 |
+
#PyNaCl
|
| 32 |
+
|
| 33 |
+
# Alternative PostgreSQL driver (if not using asyncpg)
|
| 34 |
+
#psycopg2-binary
|