File size: 6,469 Bytes
329dc8c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 | """
scripts/check_secrets.py — Sentinel-AI
======================================
Fail fast when a live credential or an unignored generated artifact is about to
be shared.
Checks:
1. High-confidence cloud key patterns in source/docs (never prints the key).
2. ``.env`` files are present but covered by ``.gitignore``.
3. Generated index artifacts are covered by ``.gitignore``.
Run from anywhere: python scripts/check_secrets.py
Exit code 0 = clean, 1 = findings.
"""
from __future__ import annotations
import fnmatch
import os
import re
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
GITIGNORE = ROOT / ".gitignore"
SKIP_DIRS = {
".git",
".venv",
"venv",
"env",
"node_modules",
"__pycache__",
".pytest_cache",
".mypy_cache",
".ruff_cache",
".tanstack",
".output",
".vite",
".wrangler",
"dist",
"dist-ssr",
"build",
"Data",
"models_cache",
"index_artifacts",
"backup_fullimage",
}
TEXT_SUFFIXES = {
".py", ".pyi", ".ps1", ".sh", ".bat", ".cmd", ".ts", ".tsx", ".js",
".jsx", ".json", ".md", ".txt", ".yaml", ".yml", ".toml", ".ini",
".cfg", ".conf", ".env", ".example", ".html", ".css", ".sql",
}
# (label, compiled pattern). Patterns intentionally require realistic key
# lengths so documentation placeholders are not flagged.
PATTERNS = [
("Google API key", re.compile(r"AIza[0-9A-Za-z_-]{35}")),
("Groq API key", re.compile(r"gsk_[0-9A-Za-z]{20,}")),
("OpenAI-style key", re.compile(r"\bsk-[0-9A-Za-z_-]{20,}")),
("AWS access key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")),
("Cohere key assignment", re.compile(
r"(?i)cohere[_-]?api[_-]?key\s*[=:]\s*['\"]?([0-9A-Za-z]{20,})"
)),
("Generic secret assignment", re.compile(
r"(?i)(?:api[_-]?key|secret|access[_-]?token|password)"
r"\s*[=:]\s*['\"]([0-9A-Za-z_\-]{24,})['\"]"
)),
]
REQUIRED_IGNORES = [
".env",
"backend/.env",
"backend/index_artifacts/celeb_index.faiss",
"backend/index_artifacts/metadata.json",
"backend/index_artifacts/celeb_index.order.json",
"backend/index_artifacts/celeb_index.preprocessing.json",
]
def _read_gitignore() -> list[str]:
if not GITIGNORE.is_file():
return []
lines = []
for raw in GITIGNORE.read_text(encoding="utf-8").splitlines():
line = raw.strip()
if not line or line.startswith("#"):
continue
lines.append(line)
return lines
def _is_ignored(relative_path: str, patterns: list[str]) -> bool:
"""Small gitignore evaluator (enough for this repository's patterns)."""
ignored = False
path = relative_path.replace("\\", "/").lstrip("./")
for pattern in patterns:
negated = pattern.startswith("!")
candidate = pattern[1:] if negated else pattern
candidate = candidate.rstrip("/")
has_slash = "/" in candidate
target = path if has_slash else Path(path).name
if fnmatch.fnmatch(target, candidate) or fnmatch.fnmatch(path, candidate):
ignored = not negated
return ignored
def _iter_files():
for dirpath, dirnames, filenames in os.walk(ROOT):
dirnames[:] = [
name for name in dirnames
if name not in SKIP_DIRS and not name.startswith(".git")
]
for filename in filenames:
path = Path(dirpath) / filename
if path.stat().st_size > 2 * 1024 * 1024:
continue
if path.suffix.lower() in TEXT_SUFFIXES or filename in {
"Dockerfile", "Makefile", "Procfile", "gitignore",
}:
yield path
def _redact(value: str) -> str:
return value[:4] + "..." + value[-2:] if len(value) > 8 else "***"
def _scan_patterns() -> list[str]:
findings: list[str] = []
for path in _iter_files():
relative = path.relative_to(ROOT).as_posix()
# Real env files are supposed to contain secrets; they are checked for
# ignore coverage below, not pattern-matched.
if path.name == ".env" or path.name.startswith(".env.") and path.name != ".env.example":
continue
if path.name == ".env.example":
continue
try:
text = path.read_text(encoding="utf-8", errors="ignore")
except OSError:
continue
for line_number, line in enumerate(text.splitlines(), start=1):
for label, pattern in PATTERNS:
match = pattern.search(line)
if not match:
continue
value = match.group(1) if match.groups() else match.group(0)
findings.append(
f"{relative}:{line_number}: {label} ({_redact(value)})"
)
return findings
def _check_ignores(patterns: list[str]) -> tuple[list[str], list[str]]:
problems: list[str] = []
notes: list[str] = []
for relative in REQUIRED_IGNORES:
if not _is_ignored(relative, patterns):
problems.append(f"{relative} is NOT covered by .gitignore")
for env_file in (ROOT / ".env", ROOT / "backend" / ".env"):
if env_file.is_file():
relative = env_file.relative_to(ROOT).as_posix()
if not _is_ignored(relative, patterns):
problems.append(f"{relative} exists but is NOT covered by .gitignore")
else:
notes.append(f"{relative} present and gitignored")
return problems, notes
def main() -> int:
patterns = _read_gitignore()
findings = _scan_patterns()
ignore_problems, notes = _check_ignores(patterns)
status = 0
if findings:
status = 1
print("[FAIL] Possible live credentials in non-env files:")
for finding in findings:
print(f" - {finding}")
else:
print("[OK] No high-confidence key patterns in source/docs.")
if ignore_problems:
status = 1
print("[FAIL] Ignore coverage:")
for problem in ignore_problems:
print(f" - {problem}")
else:
print("[OK] Required secrets/artifacts are covered by .gitignore.")
for note in notes:
print(f" note: {note}")
if status:
print("\nRotate any real key that was exposed, then remove it from the file.")
print("Never commit .env or generated index artifacts.")
return status
if __name__ == "__main__":
sys.exit(main())
|