Fix: bootstrap sequence, sqlite database detection, and migrations

.gitignore

@@ -14,6 +14,7 @@ __pycache__/
 attstore/
 
 .DS_Store/
+*.DS_Store
 node_modules/
 
 # Documentation (pulled during build or via download script)

app.py

@@ -22,6 +22,7 @@ jwt = JWTManager()
 
 # Create the app
 app = Flask(__name__)
+os.makedirs(app.instance_path, exist_ok=True)
 # Ensure templates auto-reload for development
 app.config['TEMPLATES_AUTO_RELOAD'] = True
 app.jinja_env.auto_reload = True
@@ -51,17 +52,13 @@ jwt.init_app(app)
 migrate = Migrate(app, db)
 
 # Import config and setup
-from config import setup_config, create_default_admin_user
+from config import setup_config
 setup_config(app)
 
-with app.app_context():
-    # Import models to ensure tables are created
-    import models
-    db.create_all()
-    
-    # Note: Default admin user creation moved to main.py after migrations
-    
-    # Import and register blueprints
+# Import models so SQLAlchemy metadata is populated for migrations
+import models  # noqa: E402,F401
+
+def register_blueprints(app):
     from api.evidence import evidence_bp
     from api.login import login_bp
     from api.syslog import syslog_bp
@@ -84,6 +81,8 @@ with app.app_context():
     app.register_blueprint(saml_config_bp)
     app.register_blueprint(saml_bp)
 
+register_blueprints(app)
+
 # Public health endpoint (no auth)
 @app.get('/healthz')
 def healthz():

main.py

@@ -1,6 +1,11 @@
-from app import app
-from config import PORT
 import logging
+import os
+
+from sqlalchemy import inspect
+from sqlalchemy.engine.url import make_url
+
+from app import app, db
+from config import PORT
 from werkzeug.serving import WSGIRequestHandler
 # --- Alembic migration runner ---
 from alembic import command
@@ -18,6 +23,44 @@ def run_migrations():
         command.upgrade(alembic_cfg, "head")
 
 
+def ensure_sqlite_database():
+    """Bootstrap SQLite DB files before Alembic runs."""
+    db_uri = app.config.get("SQLALCHEMY_DATABASE_URI", "")
+    if not db_uri.startswith("sqlite"):
+        return
+
+    url = make_url(db_uri)
+    database = url.database
+    if not database:
+        return
+
+    if os.path.isabs(database):
+        db_path = database
+    else:
+        db_path = os.path.join(app.instance_path, database)
+
+    os.makedirs(os.path.dirname(db_path), exist_ok=True)
+
+    bootstrap_reason = None
+    if not os.path.exists(db_path):
+        bootstrap_reason = "file missing"
+    else:
+        with app.app_context():
+            inspector = inspect(db.engine)
+            tables = inspector.get_table_names()
+            if 'users' not in tables:
+                bootstrap_reason = "schema missing"
+
+    if not bootstrap_reason:
+        logging.info(f"SQLite database already initialized at {db_path}")
+        return
+
+    logging.info(f"Bootstrapping SQLite database at {db_path} ({bootstrap_reason})")
+    with app.app_context():
+        db.create_all()
+    logging.info("SQLite database initialization complete")
+
+
 # Filter out specific paths from Werkzeug logs
 class FilteredWSGIRequestHandler(WSGIRequestHandler):
     def log(self, type, message, *args):
@@ -26,31 +69,13 @@ class FilteredWSGIRequestHandler(WSGIRequestHandler):
             return
         return super().log(type, message, *args)
 
-import os
-from app import db
-
-def db_file_exists():
-    # Only works for SQLite
-    db_uri = app.config.get("SQLALCHEMY_DATABASE_URI", "")
-    if db_uri.startswith("sqlite:///"):
-        db_path = db_uri.replace("sqlite:///", "")
-        return os.path.exists(db_path)
-    return True  # For non-SQLite, always run migrations
-
 if __name__ == '__main__':
     from config import create_default_admin_user
-    
-    if not db_file_exists():
-        with app.app_context():
-            db.create_all()
-            print("Database created, skipping migrations.")
-            # Create default admin user for new database
-            create_default_admin_user(app)
-    else:
-        run_migrations()  # Ensure DB is migrated before serving requests
-        # Create default admin user after migrations
-        with app.app_context():
-            create_default_admin_user(app)
-    
+
+    ensure_sqlite_database()
+    run_migrations()  # Always ensure DB schema is up to date before serving
+    with app.app_context():
+        create_default_admin_user(app)
+
     # Use filtered request handler
     app.run(host='0.0.0.0', port=PORT, debug=True, request_handler=FilteredWSGIRequestHandler)

migrations/versions/d2485e8b5d85_add_saml_support.py

@@ -7,6 +7,7 @@ Create Date: 2025-12-08 11:21:20.267870
 """
 from alembic import op
 import sqlalchemy as sa
+from sqlalchemy import inspect
 
 
 # revision identifiers, used by Alembic.
@@ -17,20 +18,148 @@ depends_on = None
 
 
 def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    with op.batch_alter_table('users', schema=None) as batch_op:
-        batch_op.add_column(sa.Column('tenant_id', sa.Integer(), nullable=True))
-        batch_op.create_index('ix_users_tenant_id', ['tenant_id'], unique=False)
-        batch_op.create_foreign_key(None, 'tenants', ['tenant_id'], ['id'])
-
-    # ### end Alembic commands ###
+    """Ensure all SAML/tenant tables and relationships exist."""
+    bind = op.get_bind()
+    ensure_tenants(bind)
+    ensure_saml_configurations(bind)
+    ensure_external_identities(bind)
+    ensure_users_tenant_relationship(bind)
 
 
 def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
     with op.batch_alter_table('users', schema=None) as batch_op:
-        batch_op.drop_constraint(None, type_='foreignkey')
+        batch_op.drop_constraint('fk_users_tenant_id_tenants', type_='foreignkey')
         batch_op.drop_index('ix_users_tenant_id')
         batch_op.drop_column('tenant_id')
 
-    # ### end Alembic commands ###
+    op.drop_table('external_identities')
+    op.drop_table('saml_configurations')
+    op.drop_table('tenants')
+
+
+def ensure_tenants(bind):
+    inspector = inspect(bind)
+    tables = inspector.get_table_names()
+    if 'tenants' not in tables:
+        op.create_table(
+            'tenants',
+            sa.Column('id', sa.Integer(), primary_key=True),
+            sa.Column('name', sa.String(length=128), nullable=False),
+            sa.Column('slug', sa.String(length=64), nullable=False, unique=True),
+            sa.Column('description', sa.Text(), nullable=True),
+            sa.Column('active', sa.Boolean(), nullable=False, server_default=sa.sql.expression.true()),
+            sa.Column('allow_password_auth', sa.Boolean(), nullable=False, server_default=sa.sql.expression.true()),
+            sa.Column('created_at', sa.DateTime(), nullable=True),
+            sa.Column('updated_at', sa.DateTime(), nullable=True),
+        )
+        op.create_index('ix_tenants_slug', 'tenants', ['slug'], unique=False)
+        op.create_index('ix_tenants_active', 'tenants', ['active'], unique=False)
+    else:
+        existing_indexes = {idx['name'] for idx in inspector.get_indexes('tenants')}
+        if 'ix_tenants_slug' not in existing_indexes:
+            op.create_index('ix_tenants_slug', 'tenants', ['slug'], unique=False)
+        if 'ix_tenants_active' not in existing_indexes:
+            op.create_index('ix_tenants_active', 'tenants', ['active'], unique=False)
+
+
+def ensure_saml_configurations(bind):
+    inspector = inspect(bind)
+    tables = inspector.get_table_names()
+    if 'saml_configurations' not in tables:
+        op.create_table(
+            'saml_configurations',
+            sa.Column('id', sa.Integer(), primary_key=True),
+            sa.Column('tenant_id', sa.Integer(), sa.ForeignKey('tenants.id', name='fk_saml_configurations_tenant_id'), nullable=False),
+            sa.Column('name', sa.String(length=128), nullable=False),
+            sa.Column('slug', sa.String(length=64), nullable=False),
+            sa.Column('enabled', sa.Boolean(), nullable=False, server_default=sa.sql.expression.true()),
+            sa.Column('metadata_xml', sa.Text(), nullable=False),
+            sa.Column('idp_entity_id', sa.String(length=512), nullable=False),
+            sa.Column('idp_sso_url', sa.String(length=512), nullable=False),
+            sa.Column('idp_slo_url', sa.String(length=512), nullable=True),
+            sa.Column('idp_certificate', sa.Text(), nullable=False),
+            sa.Column('sp_entity_id', sa.String(length=512), nullable=True),
+            sa.Column('acs_url', sa.String(length=512), nullable=True),
+            sa.Column('slo_url', sa.String(length=512), nullable=True),
+            sa.Column('attribute_mapping', sa.JSON(), nullable=True),
+            sa.Column('auto_provision_users', sa.Boolean(), nullable=False, server_default=sa.sql.expression.true()),
+            sa.Column('allow_idp_initiated', sa.Boolean(), nullable=False, server_default=sa.sql.expression.true()),
+            sa.Column('created_at', sa.DateTime(), nullable=True),
+            sa.Column('updated_at', sa.DateTime(), nullable=True),
+            sa.Column('created_by', sa.Integer(), sa.ForeignKey('users.id', name='fk_saml_configurations_created_by'), nullable=True),
+            sa.UniqueConstraint('tenant_id', 'slug', name='uq_saml_tenant_slug'),
+        )
+        op.create_index('ix_saml_configurations_tenant_id', 'saml_configurations', ['tenant_id'], unique=False)
+        op.create_index('ix_saml_configurations_enabled', 'saml_configurations', ['enabled'], unique=False)
+    else:
+        existing_indexes = {idx['name'] for idx in inspector.get_indexes('saml_configurations')}
+        if 'ix_saml_configurations_tenant_id' not in existing_indexes:
+            op.create_index('ix_saml_configurations_tenant_id', 'saml_configurations', ['tenant_id'], unique=False)
+        if 'ix_saml_configurations_enabled' not in existing_indexes:
+            op.create_index('ix_saml_configurations_enabled', 'saml_configurations', ['enabled'], unique=False)
+
+        existing_uniques = {uc['name'] for uc in inspector.get_unique_constraints('saml_configurations')}
+        if 'uq_saml_tenant_slug' not in existing_uniques:
+            op.create_unique_constraint('uq_saml_tenant_slug', 'saml_configurations', ['tenant_id', 'slug'])
+
+
+def ensure_external_identities(bind):
+    inspector = inspect(bind)
+    tables = inspector.get_table_names()
+    if 'external_identities' not in tables:
+        op.create_table(
+            'external_identities',
+            sa.Column('id', sa.Integer(), primary_key=True),
+            sa.Column('provider', sa.String(length=64), nullable=False),
+            sa.Column('provider_id', sa.Integer(), nullable=True),
+            sa.Column('external_id', sa.String(length=255), nullable=False),
+            sa.Column('tenant_id', sa.Integer(), sa.ForeignKey('tenants.id', name='fk_external_identities_tenant_id'), nullable=True),
+            sa.Column('user_id', sa.Integer(), sa.ForeignKey('users.id', name='fk_external_identities_user_id'), nullable=False),
+            sa.Column('attributes', sa.JSON(), nullable=True),
+            sa.Column('last_login_at', sa.DateTime(), nullable=True),
+            sa.Column('created_at', sa.DateTime(), nullable=True),
+            sa.Column('updated_at', sa.DateTime(), nullable=True),
+            sa.UniqueConstraint('provider', 'provider_id', 'external_id', name='uq_external_identity'),
+        )
+        op.create_index('ix_external_identities_user_id', 'external_identities', ['user_id'], unique=False)
+        op.create_index('ix_external_identities_tenant_id', 'external_identities', ['tenant_id'], unique=False)
+    else:
+        existing_indexes = {idx['name'] for idx in inspector.get_indexes('external_identities')}
+        if 'ix_external_identities_user_id' not in existing_indexes:
+            op.create_index('ix_external_identities_user_id', 'external_identities', ['user_id'], unique=False)
+        if 'ix_external_identities_tenant_id' not in existing_indexes:
+            op.create_index('ix_external_identities_tenant_id', 'external_identities', ['tenant_id'], unique=False)
+
+        existing_uniques = {uc['name'] for uc in inspector.get_unique_constraints('external_identities')}
+        if 'uq_external_identity' not in existing_uniques:
+            op.create_unique_constraint('uq_external_identity', 'external_identities', ['provider', 'provider_id', 'external_id'])
+
+
+def ensure_users_tenant_relationship(bind):
+    inspector = inspect(bind)
+    columns = {col['name'] for col in inspector.get_columns('users')}
+    existing_indexes = {idx['name'] for idx in inspector.get_indexes('users')}
+    existing_fks = inspector.get_foreign_keys('users')
+
+    fk_exists = any(
+        fk.get('name') == 'fk_users_tenant_id_tenants' or
+        (fk.get('referred_table') == 'tenants' and fk.get('constrained_columns') == ['tenant_id'])
+        for fk in existing_fks
+    )
+    needs_column = 'tenant_id' not in columns
+    needs_fk = not fk_exists
+
+    if needs_column or needs_fk:
+        with op.batch_alter_table('users', schema=None) as batch_op:
+            if needs_column:
+                batch_op.add_column(sa.Column('tenant_id', sa.Integer(), nullable=True))
+            if needs_fk:
+                batch_op.create_foreign_key(
+                    'fk_users_tenant_id_tenants',
+                    'tenants',
+                    ['tenant_id'],
+                    ['id']
+                )
+
+    if 'ix_users_tenant_id' not in existing_indexes:
+        op.create_index('ix_users_tenant_id', 'users', ['tenant_id'], unique=False)