Spaces:
Paused
Paused
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| import argparse | |
| import json | |
| import os | |
| import re | |
| import subprocess | |
| import time | |
| from typing import List, Optional | |
| from urllib.error import HTTPError, URLError | |
| from urllib.request import Request, urlopen | |
| # The change detector gates the entire CI matrix, so a single transient GitHub | |
| # API hiccup should not fail the build. Retry server errors and network blips | |
| # with exponential backoff before giving up. | |
| MAX_RETRIES: int = 4 | |
| RETRY_BACKOFF_SECONDS: int = 2 | |
| REQUEST_TIMEOUT_SECONDS: int = 30 | |
| # GitHub returns 429 when throttling, which is transient and worth retrying | |
| # alongside 5xx server errors. It also returns 403 for two very different | |
| # reasons — a secondary rate limit, and a token missing the required scope — | |
| # so 403 is only retried when the response headers show throttling. Retrying a | |
| # scope failure just delays the error and buries its cause. | |
| RETRYABLE_STATUS_CODES: frozenset[int] = frozenset({429}) | |
| # Define patterns for each group of files you're interested in | |
| PATTERNS = { | |
| "python": [ | |
| r"^\.github/workflows/.*python", | |
| r"^\.github/workflows/scheduled-docker-image-refresh\.yml$", | |
| r"^docker-compose-image-tag\.yml$", | |
| r"^tests/", | |
| r"^superset/", | |
| r"^scripts/", | |
| r"^setup\.py", | |
| r"^pyproject\.toml$", | |
| r"^requirements/.+\.txt", | |
| r"^pyproject\.toml", | |
| r"^.pylintrc", | |
| ], | |
| "frontend": [ | |
| r"^\.github/workflows/.*(bashlib|frontend|e2e)", | |
| r"^superset-frontend/", | |
| ], | |
| "docker": [ | |
| r"^Dockerfile$", | |
| r"^docker.*", | |
| ], | |
| "docs": [ | |
| r"^docs/", | |
| ], | |
| "superset-extensions-cli": [ | |
| r"^\.github/workflows/superset-extensions-cli\.yml", | |
| r"^superset-extensions-cli/", | |
| r"^superset-core/", | |
| ], | |
| } | |
| GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") | |
| def _is_rate_limited(err: HTTPError) -> bool: | |
| """Whether a 403 is GitHub throttling rather than a missing token scope.""" | |
| headers = getattr(err, "headers", None) | |
| if headers is None: | |
| return False | |
| try: | |
| return ( | |
| headers.get("x-ratelimit-remaining") == "0" | |
| or headers.get("retry-after") is not None | |
| ) | |
| except AttributeError: | |
| return False | |
| def _api_error_detail(err: object) -> str: | |
| """The API's own explanation, e.g. ``Resource not accessible by integration``. | |
| Without this a 403 is indistinguishable from a rate limit in the CI log. | |
| """ | |
| try: | |
| body = err.read().decode("utf-8", "replace") # type: ignore[attr-defined] | |
| except Exception: # noqa: BLE001 # pylint: disable=broad-except | |
| return "" | |
| try: | |
| detail = json.loads(body).get("message") or body | |
| except (ValueError, AttributeError): | |
| detail = body | |
| # Bound the output so a large/HTML error page can't flood the CI log. | |
| return detail[:500] | |
| def fetch_files_github_api(url: str): # type: ignore | |
| """Fetches data using GitHub API, retrying on transient failures.""" | |
| req = Request(url) # noqa: S310 | |
| req.add_header("Authorization", f"Bearer {GITHUB_TOKEN}") | |
| req.add_header("Accept", "application/vnd.github.v3+json") | |
| print(f"Fetching from {url}") | |
| for attempt in range(1, MAX_RETRIES + 1): | |
| try: | |
| with urlopen(req, timeout=REQUEST_TIMEOUT_SECONDS) as response: # noqa: S310 | |
| body = response.read() | |
| return json.loads(body) | |
| except (HTTPError, URLError) as err: | |
| # Retry transient failures: network errors (URLError has no status | |
| # code), 5xx server errors, and GitHub rate-limit responses. Other | |
| # 4xx client errors are deterministic, so re-raise immediately. Also | |
| # re-raise once the retry budget is exhausted. | |
| status = getattr(err, "code", None) | |
| is_transient = ( | |
| status is None | |
| or status >= 500 | |
| or status in RETRYABLE_STATUS_CODES | |
| or (status == 403 and _is_rate_limited(err)) # type: ignore[arg-type] | |
| ) | |
| if not is_transient or attempt == MAX_RETRIES: | |
| if detail := _api_error_detail(err): | |
| print(f"GitHub API error {status}: {detail}") | |
| raise | |
| wait = RETRY_BACKOFF_SECONDS * 2 ** (attempt - 1) | |
| print( | |
| f"Attempt {attempt}/{MAX_RETRIES} failed ({err}); " | |
| f"retrying in {wait}s..." | |
| ) | |
| time.sleep(wait) | |
| def fetch_changed_files_pr(repo: str, pr_number: str) -> List[str]: | |
| """Fetches files changed in a PR using the GitHub API.""" | |
| # NOTE: limited to 100 files ideally should page-through but instead resorting | |
| # to assuming we should trigger when 100 files have been touched | |
| url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files?per_page=100" | |
| files = fetch_files_github_api(url) | |
| return [file_info["filename"] for file_info in files] | |
| def fetch_changed_files_push(repo: str, sha: str) -> List[str]: | |
| """Fetches files changed in the last commit for push events using GitHub API.""" | |
| # Fetch commit details to get the parent SHA | |
| commit_url = f"https://api.github.com/repos/{repo}/commits/{sha}" | |
| commit_data = fetch_files_github_api(commit_url) | |
| if "parents" not in commit_data or len(commit_data["parents"]) < 1: | |
| raise RuntimeError("No parent commit found for comparison.") | |
| parent_sha = commit_data["parents"][0]["sha"] | |
| # Compare the current commit against its parent | |
| compare_url = f"https://api.github.com/repos/{repo}/compare/{parent_sha}...{sha}" | |
| comparison_data = fetch_files_github_api(compare_url) | |
| return [file["filename"] for file in comparison_data["files"]] | |
| def detect_changes(files: List[str], check_patterns: List) -> bool: # type: ignore | |
| """Detects if any of the specified files match the provided patterns.""" | |
| for file in files: | |
| for pattern in check_patterns: | |
| if re.match(pattern, file): | |
| return True | |
| return False | |
| def print_files(files: List[str]) -> None: | |
| print("\n".join([f"- {s}" for s in files])) | |
| def is_int(s: str) -> bool: | |
| return bool(re.match(r"^-?\d+$", s)) | |
| def main(event_type: str, sha: str, repo: str) -> None: | |
| """Main function to check for file changes based on event context.""" | |
| print("SHA:", sha) | |
| print("EVENT_TYPE", event_type) | |
| files: Optional[List[str]] = [] | |
| if event_type == "pull_request": | |
| pr_number = os.getenv("GITHUB_REF", "").split("/")[-2] | |
| if is_int(pr_number): | |
| files = fetch_changed_files_pr(repo, pr_number) | |
| print("PR files:") | |
| print_files(files) | |
| elif event_type == "push": | |
| files = fetch_changed_files_push(repo, sha) | |
| print("Files touched since previous commit:") | |
| print_files(files) | |
| elif event_type in ("workflow_dispatch", "schedule"): | |
| # Manual or cron-triggered runs aren't tied to a specific diff, so | |
| # treat every group as changed. `files = None` makes the loop below | |
| # short-circuit to True for every group via `files is None or ...`. | |
| print(f"{event_type} run, assuming all changed") | |
| files = None | |
| else: | |
| raise ValueError(f"Unsupported event type: {event_type}") | |
| changes_detected = {} | |
| for group, regex_patterns in PATTERNS.items(): | |
| patterns_compiled = [re.compile(p) for p in regex_patterns] | |
| changes_detected[group] = files is None or detect_changes( | |
| files, patterns_compiled | |
| ) | |
| # Output results | |
| output_path = os.getenv("GITHUB_OUTPUT") or "/tmp/GITHUB_OUTPUT.txt" # noqa: S108 | |
| with open(output_path, "a") as f: | |
| for check, changed in changes_detected.items(): | |
| # NOTE: as noted above, we assume that if 100 files are touched, we should | |
| # trigger all checks. This is a workaround for the GitHub API limit of 100 | |
| # files. Using >= 99 because off-by-one errors are not uncommon | |
| if changed or (files is not None and len(files) >= 99): | |
| print(f"{check}=true", file=f) | |
| print(f"Triggering group: {check}") | |
| def get_git_sha() -> str: | |
| return os.getenv("GITHUB_SHA") or subprocess.check_output( # noqa: S603 | |
| ["git", "rev-parse", "HEAD"] # noqa: S603, S607 | |
| ).strip().decode("utf-8") | |
| if __name__ == "__main__": | |
| parser = argparse.ArgumentParser( | |
| description="Detect file changes based on event context" | |
| ) | |
| parser.add_argument( | |
| "--event-type", | |
| default=os.getenv("GITHUB_EVENT_NAME") or "push", | |
| help="The type of event that triggered the workflow", | |
| ) | |
| parser.add_argument( | |
| "--sha", | |
| default=get_git_sha(), | |
| help="The commit SHA for push events or PR head SHA", | |
| ) | |
| parser.add_argument( | |
| "--repo", | |
| default=os.getenv("GITHUB_REPOSITORY") or "apache/superset", | |
| help="GitHub repository in the format owner/repo", | |
| ) | |
| args = parser.parse_args() | |
| main(args.event_type, args.sha, args.repo) | |