SushantGautam commited on
Commit
d98bfdd
Β·
1 Parent(s): c79afe8

fix: persist embedded Hatchet Postgres in a fixed dir instead of leaking temp dirs

Browse files

Each uvx run created a fresh tempfile.mkdtemp() for the embedded Postgres
cluster and never removed it, leaking ~218M per run (37 runs β‰ˆ 8GB found
in $TMPDIR). Use a persistent cluster at ~/.simpleaudit-studio/embedded-pg/
(override with SIMPLEAUDIT_EMBEDDED_PG_DIR) so initdb runs once, and
register an atexit hook so the engine stops cleanly even if the caller
forgets. Also align demo bootstrap username assertions with the studio
default.

.env.example CHANGED
@@ -19,7 +19,7 @@ DJANGO_ALLOWED_HOSTS=localhost,127.0.0.1
19
  LOG_LEVEL=INFO
20
 
21
  # --- Bootstrap (idempotent; run automatically by the web container) ----------
22
- BOOTSTRAP_ADMIN_USERNAME=admin
23
  BOOTSTRAP_ADMIN_EMAIL=admin@example.local
24
  BOOTSTRAP_ADMIN_PASSWORD=change-me
25
  BOOTSTRAP_PROJECT_NAME=Default
 
19
  LOG_LEVEL=INFO
20
 
21
  # --- Bootstrap (idempotent; run automatically by the web container) ----------
22
+ BOOTSTRAP_ADMIN_USERNAME=studio
23
  BOOTSTRAP_ADMIN_EMAIL=admin@example.local
24
  BOOTSTRAP_ADMIN_PASSWORD=change-me
25
  BOOTSTRAP_PROJECT_NAME=Default
.env.local.example CHANGED
@@ -22,7 +22,7 @@ LOG_LEVEL=DEBUG
22
 
23
  # --- Bootstrap ----------------------------------------------------------------
24
  # `python manage.py bootstrap_platform` reads these on first run (idempotent).
25
- BOOTSTRAP_ADMIN_USERNAME=admin
26
  BOOTSTRAP_ADMIN_EMAIL=admin@example.local
27
  BOOTSTRAP_ADMIN_PASSWORD=localdevpass123
28
  BOOTSTRAP_PROJECT_NAME=Default
 
22
 
23
  # --- Bootstrap ----------------------------------------------------------------
24
  # `python manage.py bootstrap_platform` reads these on first run (idempotent).
25
+ BOOTSTRAP_ADMIN_USERNAME=studio
26
  BOOTSTRAP_ADMIN_EMAIL=admin@example.local
27
  BOOTSTRAP_ADMIN_PASSWORD=localdevpass123
28
  BOOTSTRAP_PROJECT_NAME=Default
config/settings.py CHANGED
@@ -237,7 +237,7 @@ WORKOS_ENABLED = bool(WORKOS_CLIENT_ID and WORKOS_API_KEY)
237
  APP_BASE_URL = os.environ.get("APP_BASE_URL", "http://localhost:8000").rstrip("/")
238
 
239
  DEMO_MODE = env_bool("DEMO_MODE", False)
240
- DEMO_USERNAME = os.environ.get("DEMO_USERNAME", "admin")
241
  DEMO_PASSWORD = os.environ.get("DEMO_PASSWORD", "admin123")
242
 
243
  # Framing policy:
 
237
  APP_BASE_URL = os.environ.get("APP_BASE_URL", "http://localhost:8000").rstrip("/")
238
 
239
  DEMO_MODE = env_bool("DEMO_MODE", False)
240
+ DEMO_USERNAME = os.environ.get("DEMO_USERNAME", "studio")
241
  DEMO_PASSWORD = os.environ.get("DEMO_PASSWORD", "admin123")
242
 
243
  # Framing policy:
deploy/e2e_smoke.py CHANGED
@@ -70,7 +70,7 @@ class Client:
70
  def main():
71
  base_url = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8000"
72
  env = _load_env()
73
- username = env.get("BOOTSTRAP_ADMIN_USERNAME", "admin")
74
  password = env.get("BOOTSTRAP_ADMIN_PASSWORD", "")
75
  if not password:
76
  print("FATAL: BOOTSTRAP_ADMIN_PASSWORD not set in .env")
 
70
  def main():
71
  base_url = sys.argv[1] if len(sys.argv) > 1 else "http://localhost:8000"
72
  env = _load_env()
73
+ username = env.get("BOOTSTRAP_ADMIN_USERNAME", "studio")
74
  password = env.get("BOOTSTRAP_ADMIN_PASSWORD", "")
75
  if not password:
76
  print("FATAL: BOOTSTRAP_ADMIN_PASSWORD not set in .env")
deploy/seed_bulk.py CHANGED
@@ -133,7 +133,7 @@ RUN_STATUS_DISTRIBUTION = (
133
 
134
  def main():
135
  project = Project.objects.get(id=1)
136
- admin = User.objects.get(username="admin")
137
 
138
  # ─── 1. Model Endpoints (15) ───────────────────────────────────────────────
139
  print(f"Model endpoints before: {ModelEndpoint.objects.count()}")
 
133
 
134
  def main():
135
  project = Project.objects.get(id=1)
136
+ admin = User.objects.get(username="studio")
137
 
138
  # ─── 1. Model Endpoints (15) ───────────────────────────────────────────────
139
  print(f"Model endpoints before: {ModelEndpoint.objects.count()}")
docker-compose.yml CHANGED
@@ -177,7 +177,7 @@ services:
177
  # collection, then serve with gunicorn. The seed is safe on every start:
178
  # it skips anything already present. Set SEED_ON_BOOT=false to opt out.
179
  # Set SEED_DEMO_AUDITS=false to skip demo audit runs only.
180
- command: 'sh -c "python manage.py migrate --noinput && python manage.py bootstrap_platform --username ${BOOTSTRAP_ADMIN_USERNAME:-admin} --email ${BOOTSTRAP_ADMIN_EMAIL:-admin@example.local} --password ${BOOTSTRAP_ADMIN_PASSWORD:?Set BOOTSTRAP_ADMIN_PASSWORD in .env} --project-name ${BOOTSTRAP_PROJECT_NAME:-Default} && if [ "${SEED_ON_BOOT:-true}" != "false" ]; then SEED_ARGS=\"\"; if [ \"${SEED_DEMO_AUDITS:-true}\" = \"false\" ]; then SEED_ARGS=\"--skip-demo-audits\"; fi; python manage.py seed_platform $SEED_ARGS || echo ''WARNING: seed_platform failed - continuing without seed data''; fi && python manage.py collectstatic --noinput && gunicorn config.wsgi:application --bind 0.0.0.0:8000 --workers 2"'
181
  healthcheck:
182
  test: ["CMD", "curl", "-fsS", "http://localhost:8000/healthz"]
183
  interval: 30s
 
177
  # collection, then serve with gunicorn. The seed is safe on every start:
178
  # it skips anything already present. Set SEED_ON_BOOT=false to opt out.
179
  # Set SEED_DEMO_AUDITS=false to skip demo audit runs only.
180
+ command: 'sh -c "python manage.py migrate --noinput && python manage.py bootstrap_platform --username ${BOOTSTRAP_ADMIN_USERNAME:-studio} --email ${BOOTSTRAP_ADMIN_EMAIL:-admin@example.local} --password ${BOOTSTRAP_ADMIN_PASSWORD:?Set BOOTSTRAP_ADMIN_PASSWORD in .env} --project-name ${BOOTSTRAP_PROJECT_NAME:-Default} && if [ "${SEED_ON_BOOT:-true}" != "false" ]; then SEED_ARGS=\"\"; if [ \"${SEED_DEMO_AUDITS:-true}\" = \"false\" ]; then SEED_ARGS=\"--skip-demo-audits\"; fi; python manage.py seed_platform $SEED_ARGS || echo ''WARNING: seed_platform failed - continuing without seed data''; fi && python manage.py collectstatic --noinput && gunicorn config.wsgi:application --bind 0.0.0.0:8000 --workers 2"'
181
  healthcheck:
182
  test: ["CMD", "curl", "-fsS", "http://localhost:8000/healthz"]
183
  interval: 30s
e2e/test_ui.py CHANGED
@@ -7,7 +7,7 @@ with bootstrap_platform already run.
7
  Run locally:
8
  docker compose up -d
9
  docker compose exec web python manage.py bootstrap_platform \
10
- --username admin --email admin@example.com --password e2etestpass123 \
11
  --project-name Research
12
  .venv/bin/python e2e/test_ui.py
13
  """
@@ -22,7 +22,7 @@ except ImportError:
22
  sys.exit(1)
23
 
24
  BASE_URL = "http://localhost:8000"
25
- ADMIN_USER = "admin"
26
  ADMIN_PASS = "e2etestpass123"
27
 
28
 
 
7
  Run locally:
8
  docker compose up -d
9
  docker compose exec web python manage.py bootstrap_platform \
10
+ --username studio --email studio@example.local --password e2etestpass123 \
11
  --project-name Research
12
  .venv/bin/python e2e/test_ui.py
13
  """
 
22
  sys.exit(1)
23
 
24
  BASE_URL = "http://localhost:8000"
25
+ ADMIN_USER = "studio"
26
  ADMIN_PASS = "e2etestpass123"
27
 
28
 
infra/management/commands/bootstrap_platform.py CHANGED
@@ -11,7 +11,7 @@ class Command(BaseCommand):
11
  help = "Create initial admin user and default project if missing."
12
 
13
  def add_arguments(self, parser):
14
- parser.add_argument("--username", default=os.environ.get("BOOTSTRAP_ADMIN_USERNAME", "admin"))
15
  parser.add_argument("--email", default=os.environ.get("BOOTSTRAP_ADMIN_EMAIL", "admin@example.local"))
16
  parser.add_argument("--password", default=os.environ.get("BOOTSTRAP_ADMIN_PASSWORD", ""))
17
  parser.add_argument("--project-name", default=os.environ.get("BOOTSTRAP_PROJECT_NAME", "Default"))
 
11
  help = "Create initial admin user and default project if missing."
12
 
13
  def add_arguments(self, parser):
14
+ parser.add_argument("--username", default=os.environ.get("BOOTSTRAP_ADMIN_USERNAME", "studio"))
15
  parser.add_argument("--email", default=os.environ.get("BOOTSTRAP_ADMIN_EMAIL", "admin@example.local"))
16
  parser.add_argument("--password", default=os.environ.get("BOOTSTRAP_ADMIN_PASSWORD", ""))
17
  parser.add_argument("--project-name", default=os.environ.get("BOOTSTRAP_PROJECT_NAME", "Default"))
infra/minimal_config.py CHANGED
@@ -10,9 +10,9 @@ config:
10
 
11
  from __future__ import annotations
12
 
 
13
  import logging
14
  import os
15
- import tempfile
16
  import threading
17
  from typing import Any
18
 
@@ -28,11 +28,25 @@ def is_minimal_config() -> bool:
28
  return os.environ.get("SIMPLEAUDIT_MINIMAL", "").strip() == "1"
29
 
30
 
 
 
 
 
 
 
 
 
 
 
 
 
 
31
  def start_embedded_hatchet() -> Any:
32
  """Start an embedded Hatchet engine and return the client.
33
 
34
  The sidecar binary downloads on first use (~53 MB) and caches at
35
- ~/.hatchet/embedded/. Subsequent starts are faster.
 
36
 
37
  Returns the Hatchet client instance. Raises on failure.
38
  """
@@ -43,8 +57,7 @@ def start_embedded_hatchet() -> Any:
43
 
44
  from hatchet_sdk import ClientConfig, EmbeddedHatchetConfig, Hatchet
45
 
46
- # Use a fresh temp dir per session to avoid stale postmaster.pid locks
47
- data_dir = tempfile.mkdtemp(prefix="simpleaudit-hatchet-pg-")
48
  logger.info("Embedded Hatchet data dir: %s", data_dir)
49
 
50
  config = ClientConfig(
@@ -76,3 +89,7 @@ def stop_embedded_hatchet() -> None:
76
  def get_embedded_client() -> Any | None:
77
  """Return the running embedded Hatchet client, or None."""
78
  return _embedded_client
 
 
 
 
 
10
 
11
  from __future__ import annotations
12
 
13
+ import atexit
14
  import logging
15
  import os
 
16
  import threading
17
  from typing import Any
18
 
 
28
  return os.environ.get("SIMPLEAUDIT_MINIMAL", "").strip() == "1"
29
 
30
 
31
+ def _data_dir() -> str:
32
+ """Return (and create) the persistent data dir for the embedded Postgres.
33
+
34
+ A fixed location avoids re-running initdb (~218M, ~15s) on every start.
35
+ Override with SIMPLEAUDIT_EMBEDDED_PG_DIR if needed.
36
+ """
37
+ d = os.path.expanduser(
38
+ os.environ.get("SIMPLEAUDIT_EMBEDDED_PG_DIR", "~/.simpleaudit-studio/embedded-pg")
39
+ )
40
+ os.makedirs(d, exist_ok=True)
41
+ return d
42
+
43
+
44
  def start_embedded_hatchet() -> Any:
45
  """Start an embedded Hatchet engine and return the client.
46
 
47
  The sidecar binary downloads on first use (~53 MB) and caches at
48
+ ~/.hatchet/embedded/. The Postgres cluster persists in
49
+ ~/.simpleaudit-studio/embedded-pg/, so only the first start is slow.
50
 
51
  Returns the Hatchet client instance. Raises on failure.
52
  """
 
57
 
58
  from hatchet_sdk import ClientConfig, EmbeddedHatchetConfig, Hatchet
59
 
60
+ data_dir = _data_dir()
 
61
  logger.info("Embedded Hatchet data dir: %s", data_dir)
62
 
63
  config = ClientConfig(
 
89
  def get_embedded_client() -> Any | None:
90
  """Return the running embedded Hatchet client, or None."""
91
  return _embedded_client
92
+
93
+
94
+ # Best-effort clean shutdown even if the caller forgets to stop explicitly.
95
+ atexit.register(stop_embedded_hatchet)
infra/tests/test_bootstrap.py CHANGED
@@ -39,7 +39,7 @@ class BootstrapTests(TestCase):
39
  for _ in range(2):
40
  call_command(
41
  "bootstrap_platform",
42
- username="admin",
43
  email="admin@example.local",
44
  password="admin-pass-123",
45
  project_name="Default",
@@ -57,12 +57,12 @@ class BootstrapTests(TestCase):
57
  with _safe_env():
58
  call_command(
59
  "bootstrap_platform",
60
- username="admin",
61
  email="admin@example.local",
62
  password="admin-pass-123",
63
  project_name="Default",
64
  )
65
 
66
- user = User.objects.get(username="admin")
67
  self.assertFalse(user.is_superuser)
68
  self.assertTrue(user.is_staff)
 
39
  for _ in range(2):
40
  call_command(
41
  "bootstrap_platform",
42
+ username="studio",
43
  email="admin@example.local",
44
  password="admin-pass-123",
45
  project_name="Default",
 
57
  with _safe_env():
58
  call_command(
59
  "bootstrap_platform",
60
+ username="studio",
61
  email="admin@example.local",
62
  password="admin-pass-123",
63
  project_name="Default",
64
  )
65
 
66
+ user = User.objects.get(username="studio")
67
  self.assertFalse(user.is_superuser)
68
  self.assertTrue(user.is_staff)
infra/tests/test_minimal_config.py CHANGED
@@ -26,12 +26,12 @@ class TestDemoBootSequence(TestCase):
26
  from model_registry.models import ModelEndpoint
27
 
28
  user, project = bootstrap_admin_and_default_project(
29
- username="admin",
30
  email="admin@localhost",
31
  password="admin12345",
32
  project_name="Demo Project",
33
  )
34
- self.assertEqual(user.username, "admin")
35
  self.assertEqual(project.name, "Demo Project")
36
 
37
  call_command("seed_platform", project=project.id, verbosity=0)
@@ -56,7 +56,7 @@ class TestDemoBootSequence(TestCase):
56
  from model_registry.models import ModelEndpoint
57
 
58
  _, project = bootstrap_admin_and_default_project(
59
- username="admin", email="admin@localhost", password="admin12345",
60
  project_name="Demo Project",
61
  )
62
  call_command("seed_platform", project=project.id, verbosity=0)
@@ -85,6 +85,30 @@ class TestEmbeddedHatchetLifecycle(TestCase):
85
  (~15s first run) but verify the full lifecycle works.
86
  """
87
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
88
  def test_start_and_stop(self):
89
  from infra.minimal_config import start_embedded_hatchet, stop_embedded_hatchet
90
 
 
26
  from model_registry.models import ModelEndpoint
27
 
28
  user, project = bootstrap_admin_and_default_project(
29
+ username="studio",
30
  email="admin@localhost",
31
  password="admin12345",
32
  project_name="Demo Project",
33
  )
34
+ self.assertEqual(user.username, "studio")
35
  self.assertEqual(project.name, "Demo Project")
36
 
37
  call_command("seed_platform", project=project.id, verbosity=0)
 
56
  from model_registry.models import ModelEndpoint
57
 
58
  _, project = bootstrap_admin_and_default_project(
59
+ username="studio", email="admin@localhost", password="admin12345",
60
  project_name="Demo Project",
61
  )
62
  call_command("seed_platform", project=project.id, verbosity=0)
 
85
  (~15s first run) but verify the full lifecycle works.
86
  """
87
 
88
+ def test_data_dir_is_persistent(self):
89
+ """The embedded Postgres data dir is a fixed location, not a temp dir."""
90
+ from infra.minimal_config import _data_dir
91
+
92
+ d1 = _data_dir()
93
+ d2 = _data_dir()
94
+ self.assertEqual(d1, d2)
95
+ self.assertTrue(os.path.isdir(d1))
96
+ # Must not be a per-run temp dir (the old leak: 37 x 218M in $TMPDIR)
97
+ self.assertNotIn("simpleaudit-hatchet-pg-", os.path.basename(d1))
98
+
99
+ def test_data_dir_env_override(self):
100
+ """SIMPLEAUDIT_EMBEDDED_PG_DIR overrides the default location."""
101
+ import tempfile as _tempfile
102
+
103
+ from infra.minimal_config import _data_dir
104
+
105
+ with patch.dict(os.environ, {"SIMPLEAUDIT_EMBEDDED_PG_DIR": "/tmp/custom-pg-dir-test"}):
106
+ d = _data_dir()
107
+ self.assertEqual(d, "/tmp/custom-pg-dir-test")
108
+ self.assertTrue(os.path.isdir(d))
109
+ import shutil
110
+ shutil.rmtree("/tmp/custom-pg-dir-test", ignore_errors=True)
111
+
112
  def test_start_and_stop(self):
113
  from infra.minimal_config import start_embedded_hatchet, stop_embedded_hatchet
114
 
infra/tests/test_seed_platform.py CHANGED
@@ -35,7 +35,7 @@ def _safe_env():
35
  class SeedPlatformTests(TestCase):
36
  def setUp(self):
37
  self.user = User.objects.create_superuser(
38
- username="admin", email="admin@example.local", password="pass12345"
39
  )
40
  self.project = Project.objects.create(name="Default", slug="default")
41
  ProjectMembership.objects.create(
 
35
  class SeedPlatformTests(TestCase):
36
  def setUp(self):
37
  self.user = User.objects.create_superuser(
38
+ username="studio", email="admin@example.local", password="pass12345"
39
  )
40
  self.project = Project.objects.create(name="Default", slug="default")
41
  ProjectMembership.objects.create(
simpleaudit_studio/cli.py CHANGED
@@ -99,7 +99,7 @@ def main() -> None:
99
  # Give the web server a moment to bind
100
  time.sleep(1)
101
 
102
- username = os.environ.get("BOOTSTRAP_ADMIN_USERNAME", "admin")
103
  password = os.environ.get("BOOTSTRAP_ADMIN_PASSWORD", "admin12345")
104
 
105
  print("β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”")
@@ -140,7 +140,7 @@ def _seed_demo_data() -> None:
140
  from django.core.management import call_command
141
 
142
  # Respect env vars (set by Dockerfile for HF Space, or defaults for local)
143
- username = os.environ.get("BOOTSTRAP_ADMIN_USERNAME", "admin")
144
  email = os.environ.get("BOOTSTRAP_ADMIN_EMAIL", "admin@localhost")
145
  password = os.environ.get("BOOTSTRAP_ADMIN_PASSWORD", "admin12345")
146
  project_name = os.environ.get("BOOTSTRAP_PROJECT_NAME", "Demo Project")
 
99
  # Give the web server a moment to bind
100
  time.sleep(1)
101
 
102
+ username = os.environ.get("BOOTSTRAP_ADMIN_USERNAME", "studio")
103
  password = os.environ.get("BOOTSTRAP_ADMIN_PASSWORD", "admin12345")
104
 
105
  print("β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”")
 
140
  from django.core.management import call_command
141
 
142
  # Respect env vars (set by Dockerfile for HF Space, or defaults for local)
143
+ username = os.environ.get("BOOTSTRAP_ADMIN_USERNAME", "studio")
144
  email = os.environ.get("BOOTSTRAP_ADMIN_EMAIL", "admin@localhost")
145
  password = os.environ.get("BOOTSTRAP_ADMIN_PASSWORD", "admin12345")
146
  project_name = os.environ.get("BOOTSTRAP_PROJECT_NAME", "Demo Project")
uv.lock ADDED
The diff for this file is too large to render. See raw diff