Spaces:

Leon4gr45
/

powerbi

Paused

powerbi / .github /workflows /superset-python-integrationtest.yml

Upload folder using huggingface_hub (part 2)

c2a126e verified 2 months ago

9.24 kB

# Python integration tests
name: Python-Integration

# Least-privilege default for GITHUB_TOKEN. Jobs that need more (e.g. OIDC for
# codecov uploads) opt in via their own job-level `permissions:` block.
permissions:
  contents: read

on:
  push:
    branches:
      - "master"
      - "[0-9].[0-9]*"
  pull_request:
    types: [synchronize, opened, reopened, ready_for_review]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

jobs:
  changes:
    runs-on: ubuntu-26.04
    timeout-minutes: 10
    permissions:
      contents: read
      pull-requests: read
    outputs:
      python: ${{ steps.check.outputs.python }}
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Check for file changes
        id: check
        uses: ./.github/actions/change-detector/
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

  test-mysql:
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-26.04
    timeout-minutes: 45
    permissions:
      id-token: write
    env:
      PYTHONPATH: ${{ github.workspace }}
      SUPERSET_CONFIG: tests.integration_tests.superset_test_config
      REDIS_PORT: 16379
      SUPERSET__SQLALCHEMY_DATABASE_URI: |
        mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
    services:
      mysql:
        image: ghcr.io/apache/superset/ci/mysql:8.0
        env:
          MYSQL_ROOT_PASSWORD: root
        ports:
          - 13306:3306
        options: >-
          --health-cmd="mysqladmin ping --silent"
          --health-interval=10s
          --health-timeout=5s
          --health-retries=5
      redis:
        image: ghcr.io/apache/superset/ci/redis:7-alpine
        options: --entrypoint redis-server
        ports:
          - 16379:6379
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          submodules: recursive
      - name: Setup Python
        uses: ./.github/actions/setup-backend/
      - name: Setup MySQL
        uses: ./.github/actions/cached-dependencies
        with:
          run: setup-mysql
      - name: Start Celery worker
        uses: ./.github/actions/cached-dependencies
        with:
          run: celery-worker
      - name: Python integration tests (MySQL)
        run: |
          ./scripts/python_tests.sh
      - name: Upload code coverage
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
        with:
          flags: python,mysql
          verbose: true
          use_oidc: true
          slug: apache/superset
      - name: Generate database diagnostics for docs
        env:
          SUPERSET_CONFIG: tests.integration_tests.superset_test_config
          SUPERSET__SQLALCHEMY_DATABASE_URI: |
            mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
        run: |
          python -c "
          import json
          from superset.app import create_app
          from superset.db_engine_specs.lib import generate_yaml_docs
          app = create_app()
          with app.app_context():
              docs = generate_yaml_docs()
              # Wrap in the expected format
              output = {
                  'generated': '$(date -Iseconds)',
                  'databases': docs
              }
              with open('databases-diagnostics.json', 'w') as f:
                  json.dump(output, f, indent=2, default=str)
              print(f'Generated diagnostics for {len(docs)} databases')
          "
      - name: Upload database diagnostics artifact
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: database-diagnostics
          path: databases-diagnostics.json
          retention-days: 7
  test-postgres:
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-26.04
    timeout-minutes: 45
    permissions:
      id-token: write
    strategy:
      matrix:
        # Full version spread on push (master/release) + nightly; current only
        # on PRs to cut runner cost (cross-version breaks are caught at merge).
        python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') || fromJSON('["current", "next"]') }}
    env:
      PYTHONPATH: ${{ github.workspace }}
      SUPERSET_CONFIG: tests.integration_tests.superset_test_config
      REDIS_PORT: 16379
      SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
    services:
      postgres:
        image: ghcr.io/apache/superset/ci/postgres:17-alpine
        env:
          POSTGRES_USER: superset
          POSTGRES_PASSWORD: superset
        ports:
          # Use custom ports for services to avoid accidentally connecting to
          # GitHub action runner's default installations
          - 15432:5432
      redis:
        image: ghcr.io/apache/superset/ci/redis:7-alpine
        ports:
          - 16379:6379
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          submodules: recursive
      - name: Setup Python
        uses: ./.github/actions/setup-backend/
        with:
          python-version: ${{ matrix.python-version }}
      - name: Setup Postgres
        uses: ./.github/actions/cached-dependencies
        with:
          run: |
            setup-postgres
      - name: Start Celery worker
        uses: ./.github/actions/cached-dependencies
        with:
          run: celery-worker
      - name: Python integration tests (PostgreSQL)
        run: |
          ./scripts/python_tests.sh
      - name: Upload code coverage
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
        with:
          flags: python,postgres
          verbose: true
          use_oidc: true
          slug: apache/superset

  test-sqlite:
    needs: changes
    if: needs.changes.outputs.python == 'true'
    runs-on: ubuntu-26.04
    timeout-minutes: 45
    permissions:
      id-token: write
    env:
      PYTHONPATH: ${{ github.workspace }}
      SUPERSET_CONFIG: tests.integration_tests.superset_test_config
      REDIS_PORT: 16379
      SUPERSET__SQLALCHEMY_DATABASE_URI: |
        sqlite:///${{ github.workspace }}/.temp/superset.db?check_same_thread=true
      SUPERSET__SQLALCHEMY_EXAMPLES_URI: |
        sqlite:///${{ github.workspace }}/.temp/examples.db?check_same_thread=true
    services:
      redis:
        image: ghcr.io/apache/superset/ci/redis:7-alpine
        ports:
          - 16379:6379
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
          submodules: recursive
      - name: Setup Python
        uses: ./.github/actions/setup-backend/
      - name: Install dependencies
        uses: ./.github/actions/cached-dependencies
        with:
          run: |
            # sqlite needs this working directory
            mkdir ${{ github.workspace }}/.temp
      - name: Start Celery worker
        uses: ./.github/actions/cached-dependencies
        with:
          run: celery-worker
      - name: Python integration tests (SQLite)
        run: |
          ./scripts/python_tests.sh
      - name: Upload code coverage
        uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
        with:
          flags: python,sqlite
          verbose: true
          use_oidc: true
          slug: apache/superset

  # Stable required-status-check anchor for the matrix-based test-postgres job.
  # It is gated on change detection, so on non-Python PRs it is skipped and
  # never produces its `test-postgres (current)` context (a job-level skip
  # happens before matrix expansion). This always-running job reports a single
  # context branch protection can require: it passes when test-postgres
  # succeeded or was skipped, and fails only on a real failure.
  test-postgres-required:
    needs: [changes, test-postgres]
    if: always()
    runs-on: ubuntu-26.04
    timeout-minutes: 5
    steps:
      - name: Check test-postgres result
        env:
          RESULT: ${{ needs.test-postgres.result }}
        run: |
          if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then
            echo "test-postgres did not pass (result: $RESULT)"
            exit 1
          fi
          echo "test-postgres result: $RESULT"
  actions-timeline:
    needs: [test-mysql, test-postgres, test-sqlite, test-postgres-required]
    if: always()
    runs-on: ubuntu-26.04
    permissions:
      actions: read
    steps:
      - uses: Kesin11/actions-timeline@57fc93f20c6da7fbc14063c6d24a2a5627c799ad # v3.2.0
        with:
          expand-composite-actions: true

	# Python integration tests
	name: Python-Integration

	# Least-privilege default for GITHUB_TOKEN. Jobs that need more (e.g. OIDC for
	# codecov uploads) opt in via their own job-level `permissions:` block.
	permissions:
	contents: read

	on:
	push:
	branches:
	- "master"
	- "[0-9].[0-9]*"
	pull_request:
	types: [synchronize, opened, reopened, ready_for_review]

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.run_id }}
	cancel-in-progress: true

	jobs:
	changes:
	runs-on: ubuntu-26.04
	timeout-minutes: 10
	permissions:
	contents: read
	pull-requests: read
	outputs:
	python: ${{ steps.check.outputs.python }}
	steps:
	- name: Checkout
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	- name: Check for file changes
	id: check
	uses: ./.github/actions/change-detector/
	with:
	token: ${{ secrets.GITHUB_TOKEN }}

	test-mysql:
	needs: changes
	if: needs.changes.outputs.python == 'true'
	runs-on: ubuntu-26.04
	timeout-minutes: 45
	permissions:
	id-token: write
	env:
	PYTHONPATH: ${{ github.workspace }}
	SUPERSET_CONFIG: tests.integration_tests.superset_test_config
	REDIS_PORT: 16379
	SUPERSET__SQLALCHEMY_DATABASE_URI: \|
	mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
	services:
	mysql:
	image: ghcr.io/apache/superset/ci/mysql:8.0
	env:
	MYSQL_ROOT_PASSWORD: root
	ports:
	- 13306:3306
	options: >-
	--health-cmd="mysqladmin ping --silent"
	--health-interval=10s
	--health-timeout=5s
	--health-retries=5
	redis:
	image: ghcr.io/apache/superset/ci/redis:7-alpine
	options: --entrypoint redis-server
	ports:
	- 16379:6379
	steps:
	- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	submodules: recursive
	- name: Setup Python
	uses: ./.github/actions/setup-backend/
	- name: Setup MySQL
	uses: ./.github/actions/cached-dependencies
	with:
	run: setup-mysql
	- name: Start Celery worker
	uses: ./.github/actions/cached-dependencies
	with:
	run: celery-worker
	- name: Python integration tests (MySQL)
	run: \|
	./scripts/python_tests.sh
	- name: Upload code coverage
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
	with:
	flags: python,mysql
	verbose: true
	use_oidc: true
	slug: apache/superset
	- name: Generate database diagnostics for docs
	env:
	SUPERSET_CONFIG: tests.integration_tests.superset_test_config
	SUPERSET__SQLALCHEMY_DATABASE_URI: \|
	mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true
	run: \|
	python -c "
	import json
	from superset.app import create_app
	from superset.db_engine_specs.lib import generate_yaml_docs
	app = create_app()
	with app.app_context():
	docs = generate_yaml_docs()
	# Wrap in the expected format
	output = {
	'generated': '$(date -Iseconds)',
	'databases': docs
	}
	with open('databases-diagnostics.json', 'w') as f:
	json.dump(output, f, indent=2, default=str)
	print(f'Generated diagnostics for {len(docs)} databases')
	"
	- name: Upload database diagnostics artifact
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
	with:
	name: database-diagnostics
	path: databases-diagnostics.json
	retention-days: 7
	test-postgres:
	needs: changes
	if: needs.changes.outputs.python == 'true'
	runs-on: ubuntu-26.04
	timeout-minutes: 45
	permissions:
	id-token: write
	strategy:
	matrix:
	# Full version spread on push (master/release) + nightly; current only
	# on PRs to cut runner cost (cross-version breaks are caught at merge).
	python-version: ${{ github.event_name == 'pull_request' && fromJSON('["current"]') \|\| fromJSON('["current", "next"]') }}
	env:
	PYTHONPATH: ${{ github.workspace }}
	SUPERSET_CONFIG: tests.integration_tests.superset_test_config
	REDIS_PORT: 16379
	SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
	services:
	postgres:
	image: ghcr.io/apache/superset/ci/postgres:17-alpine
	env:
	POSTGRES_USER: superset
	POSTGRES_PASSWORD: superset
	ports:
	# Use custom ports for services to avoid accidentally connecting to
	# GitHub action runner's default installations
	- 15432:5432
	redis:
	image: ghcr.io/apache/superset/ci/redis:7-alpine
	ports:
	- 16379:6379
	steps:
	- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	submodules: recursive
	- name: Setup Python
	uses: ./.github/actions/setup-backend/
	with:
	python-version: ${{ matrix.python-version }}
	- name: Setup Postgres
	uses: ./.github/actions/cached-dependencies
	with:
	run: \|
	setup-postgres
	- name: Start Celery worker
	uses: ./.github/actions/cached-dependencies
	with:
	run: celery-worker
	- name: Python integration tests (PostgreSQL)
	run: \|
	./scripts/python_tests.sh
	- name: Upload code coverage
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
	with:
	flags: python,postgres
	verbose: true
	use_oidc: true
	slug: apache/superset

	test-sqlite:
	needs: changes
	if: needs.changes.outputs.python == 'true'
	runs-on: ubuntu-26.04
	timeout-minutes: 45
	permissions:
	id-token: write
	env:
	PYTHONPATH: ${{ github.workspace }}
	SUPERSET_CONFIG: tests.integration_tests.superset_test_config
	REDIS_PORT: 16379
	SUPERSET__SQLALCHEMY_DATABASE_URI: \|
	sqlite:///${{ github.workspace }}/.temp/superset.db?check_same_thread=true
	SUPERSET__SQLALCHEMY_EXAMPLES_URI: \|
	sqlite:///${{ github.workspace }}/.temp/examples.db?check_same_thread=true
	services:
	redis:
	image: ghcr.io/apache/superset/ci/redis:7-alpine
	ports:
	- 16379:6379
	steps:
	- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with:
	persist-credentials: false
	submodules: recursive
	- name: Setup Python
	uses: ./.github/actions/setup-backend/
	- name: Install dependencies
	uses: ./.github/actions/cached-dependencies
	with:
	run: \|
	# sqlite needs this working directory
	mkdir ${{ github.workspace }}/.temp
	- name: Start Celery worker
	uses: ./.github/actions/cached-dependencies
	with:
	run: celery-worker
	- name: Python integration tests (SQLite)
	run: \|
	./scripts/python_tests.sh
	- name: Upload code coverage
	uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7.0.0
	with:
	flags: python,sqlite
	verbose: true
	use_oidc: true
	slug: apache/superset

	# Stable required-status-check anchor for the matrix-based test-postgres job.
	# It is gated on change detection, so on non-Python PRs it is skipped and
	# never produces its `test-postgres (current)` context (a job-level skip
	# happens before matrix expansion). This always-running job reports a single
	# context branch protection can require: it passes when test-postgres
	# succeeded or was skipped, and fails only on a real failure.
	test-postgres-required:
	needs: [changes, test-postgres]
	if: always()
	runs-on: ubuntu-26.04
	timeout-minutes: 5
	steps:
	- name: Check test-postgres result
	env:
	RESULT: ${{ needs.test-postgres.result }}
	run: \|
	if [ "$RESULT" != "success" ] && [ "$RESULT" != "skipped" ]; then
	echo "test-postgres did not pass (result: $RESULT)"
	exit 1
	fi
	echo "test-postgres result: $RESULT"

	actions-timeline:
	needs: [test-mysql, test-postgres, test-sqlite, test-postgres-required]
	if: always()
	runs-on: ubuntu-26.04
	permissions:
	actions: read
	steps:
	- uses: Kesin11/actions-timeline@57fc93f20c6da7fbc14063c6d24a2a5627c799ad # v3.2.0
	with:
	expand-composite-actions: true