Buckets:
AnonymousSpottedToad/candidate_bucket / codebase /python /AmberLee2427_nancy-brain /nancy_brain /env_import.py
| """Helpers to import GitHub-backed dependencies from a conda environment.yml.""" | |
| from __future__ import annotations | |
| from pathlib import Path | |
| import re | |
| import time | |
| from typing import Any, Optional | |
| from urllib.parse import urlparse | |
| import requests | |
| import yaml | |
| PRIORITY_URL_KEYS = ("Source", "Source Code", "Homepage", "Repository") | |
| def _is_skippable_pip_spec(spec: str) -> bool: | |
| lowered = spec.strip().lower() | |
| if not lowered: | |
| return True | |
| if lowered == ".": | |
| return True | |
| if lowered.startswith("-e") or lowered.startswith("-r"): | |
| return True | |
| if lowered.startswith("http") or lowered.startswith("git+"): | |
| return True | |
| if lowered.startswith("--"): | |
| return True | |
| return False | |
| def _package_name_from_pip_spec(spec: str) -> Optional[str]: | |
| if _is_skippable_pip_spec(spec): | |
| return None | |
| package_name = re.split(r"[=<>!~\[]", spec, maxsplit=1)[0].strip() | |
| if not package_name or package_name.startswith("-"): | |
| return None | |
| return package_name | |
| def _normalize_github_url(raw_url: str) -> Optional[str]: | |
| if not isinstance(raw_url, str): | |
| return None | |
| cleaned = raw_url.strip() | |
| if not cleaned: | |
| return None | |
| if cleaned.lower().startswith("git+"): | |
| cleaned = cleaned[4:] | |
| # Handle SCM-style refs like git@github.com:owner/repo.git | |
| ssh_match = re.search(r"github\.com[:/]+([^/\s]+)/([^/\s#?]+)", cleaned, flags=re.IGNORECASE) | |
| if ssh_match: | |
| owner = ssh_match.group(1) | |
| repo = ssh_match.group(2) | |
| else: | |
| if "://" not in cleaned and cleaned.startswith("github.com/"): | |
| cleaned = f"https://{cleaned}" | |
| parsed = urlparse(cleaned) | |
| host = parsed.netloc.lower() | |
| if host.startswith("www."): | |
| host = host[4:] | |
| if host != "github.com": | |
| return None | |
| parts = [part for part in parsed.path.split("/") if part] | |
| if len(parts) < 2: | |
| return None | |
| owner, repo = parts[0], parts[1] | |
| repo = re.sub(r"\.git$", "", repo, flags=re.IGNORECASE) | |
| if not owner or not repo: | |
| return None | |
| return f"https://github.com/{owner}/{repo}" | |
| def _extract_github_url(payload: dict[str, Any]) -> Optional[str]: | |
| info = payload.get("info") or {} | |
| project_urls = info.get("project_urls") or {} | |
| if not isinstance(project_urls, dict): | |
| return None | |
| seen_keys: set[str] = set() | |
| for key in PRIORITY_URL_KEYS: | |
| seen_keys.add(key) | |
| value = project_urls.get(key) | |
| if isinstance(value, str) and "github.com" in value.lower(): | |
| normalized = _normalize_github_url(value) | |
| if normalized: | |
| return normalized | |
| for key, value in project_urls.items(): | |
| if key in seen_keys: | |
| continue | |
| if isinstance(value, str) and "github.com" in value.lower(): | |
| normalized = _normalize_github_url(value) | |
| if normalized: | |
| return normalized | |
| # Fallback: check info.home_page (older packages only have this field) | |
| home_page = info.get("home_page") or "" | |
| if isinstance(home_page, str) and "github.com" in home_page.lower(): | |
| normalized = _normalize_github_url(home_page) | |
| if normalized: | |
| return normalized | |
| return None | |
| def _repo_name_from_github_url(github_url: str) -> str: | |
| parsed = urlparse(github_url) | |
| parts = [part for part in parsed.path.split("/") if part] | |
| owner, repo = parts[0], parts[1] | |
| return f"{owner}_{repo}".lower().replace("-", "_") | |
| def _iter_existing_urls(config: dict[str, Any]) -> set[str]: | |
| urls: set[str] = set() | |
| for entries in config.values(): | |
| if not isinstance(entries, list): | |
| continue | |
| for entry in entries: | |
| if not isinstance(entry, dict): | |
| continue | |
| url = entry.get("url") | |
| if not isinstance(url, str) or not url.strip(): | |
| continue | |
| if "github.com" in url.lower(): | |
| normalized = _normalize_github_url(url) | |
| if normalized: | |
| urls.add(normalized) | |
| else: | |
| urls.add(url.strip()) | |
| else: | |
| urls.add(url.strip()) | |
| return urls | |
| def _version_from_pip_spec(spec: str) -> Optional[str]: | |
| """Return the exact version from a ``==`` pinned pip spec, or None.""" | |
| m = re.search(r"==([^\s,;]+)", spec) | |
| return m.group(1).strip() if m else None | |
| def _parse_requirements_lines(lines: list[str]) -> list[str]: | |
| """Return pip requirement specs parsed from raw requirements.txt lines.""" | |
| specs: list[str] = [] | |
| for line in lines: | |
| line = re.sub(r"\s*#.*$", "", line).strip() | |
| if not line: | |
| continue | |
| # Skip -r/-c file includes and bare flags like --extra-index-url | |
| if line.startswith("-r ") or line.startswith("-c ") or line.startswith("--"): | |
| continue | |
| specs.append(line) | |
| return specs | |
| def _load_toml(path: Path) -> dict: | |
| """Load a TOML file using tomllib (3.11+) or the tomli back-compat package.""" | |
| try: | |
| import tomllib # type: ignore[import] | |
| except ImportError: | |
| try: | |
| import tomli as tomllib # type: ignore[no-redef] | |
| except ImportError: | |
| raise ImportError( | |
| "tomllib (Python 3.11+) or tomli is required to parse pyproject.toml. " | |
| "Install with: pip install tomli" | |
| ) | |
| with open(path, "rb") as f: | |
| return tomllib.load(f) | |
| def _parse_pyproject_deps(path: Path) -> tuple[list[str], Optional[str]]: | |
| """ | |
| Extract dependency specifiers from a pyproject.toml. | |
| Returns ``(specs, project_name)`` where *project_name* may be None. | |
| Handles PEP 621 ``[project].dependencies`` and Poetry | |
| ``[tool.poetry.dependencies]``. | |
| """ | |
| data = _load_toml(path) | |
| specs: list[str] = [] | |
| # PEP 621 | |
| project_deps = data.get("project", {}).get("dependencies", []) | |
| if isinstance(project_deps, list): | |
| specs.extend([d for d in project_deps if isinstance(d, str)]) | |
| # Poetry | |
| poetry_deps = data.get("tool", {}).get("poetry", {}).get("dependencies", {}) | |
| if isinstance(poetry_deps, dict): | |
| for pkg, constraint in poetry_deps.items(): | |
| if pkg.lower() == "python": | |
| continue | |
| if isinstance(constraint, str): | |
| if constraint.startswith("=="): | |
| specs.append(f"{pkg}{constraint}") | |
| elif re.match(r"^\d", constraint): | |
| specs.append(f"{pkg}=={constraint}") | |
| else: | |
| specs.append(pkg) | |
| elif isinstance(constraint, dict): | |
| version = constraint.get("version", "") | |
| if version.startswith("=="): | |
| specs.append(f"{pkg}{version}") | |
| elif re.match(r"^\d", version): | |
| specs.append(f"{pkg}=={version}") | |
| else: | |
| specs.append(pkg) | |
| project_name: Optional[str] = data.get("project", {}).get("name") or data.get("tool", {}).get("poetry", {}).get( | |
| "name" | |
| ) | |
| return specs, project_name | |
| def _import_package_list( | |
| pip_packages: list[str], | |
| effective_category: str, | |
| output_file_path: Path, | |
| dry_run: bool, | |
| pin_versions: bool, | |
| ) -> dict: | |
| """ | |
| Core PyPI lookup loop shared by all ``import_from_*`` functions. | |
| Returns dict: {added, skipped_no_github, skipped_duplicate, errors} | |
| """ | |
| if output_file_path.exists(): | |
| with open(output_file_path, "r", encoding="utf-8") as f: | |
| repositories = yaml.safe_load(f) or {} | |
| if not isinstance(repositories, dict): | |
| raise ValueError("repositories.yml must contain a top-level mapping") | |
| else: | |
| repositories = {} | |
| existing_urls = _iter_existing_urls(repositories) | |
| category_entries = repositories.setdefault(effective_category, []) | |
| if not isinstance(category_entries, list): | |
| raise ValueError(f"Category '{effective_category}' in repositories.yml must be a list") | |
| added = 0 | |
| skipped_no_github = 0 | |
| skipped_duplicate = 0 | |
| errors: list[str] = [] | |
| for raw_spec in pip_packages: | |
| package_name = _package_name_from_pip_spec(raw_spec) | |
| if not package_name: | |
| continue | |
| try: | |
| response = requests.get(f"https://pypi.org/pypi/{package_name}/json", timeout=10) | |
| response.raise_for_status() | |
| payload = response.json() | |
| except Exception as exc: | |
| errors.append(f"{package_name}: {exc}") | |
| time.sleep(0.1) | |
| continue | |
| github_url = _extract_github_url(payload) | |
| if not github_url: | |
| skipped_no_github += 1 | |
| time.sleep(0.1) | |
| continue | |
| if github_url in existing_urls: | |
| skipped_duplicate += 1 | |
| time.sleep(0.1) | |
| continue | |
| entry: dict[str, Any] = { | |
| "name": _repo_name_from_github_url(github_url), | |
| "url": github_url, | |
| "description": f"{package_name} - source from PyPI project_urls", | |
| } | |
| if pin_versions: | |
| version = _version_from_pip_spec(raw_spec) | |
| if version: | |
| entry["ref"] = version | |
| category_entries.append(entry) | |
| existing_urls.add(github_url) | |
| added += 1 | |
| time.sleep(0.1) | |
| if not dry_run: | |
| output_file_path.parent.mkdir(parents=True, exist_ok=True) | |
| with open(output_file_path, "w", encoding="utf-8") as f: | |
| yaml.dump(repositories, f, default_flow_style=False, sort_keys=False) | |
| return { | |
| "added": added, | |
| "skipped_no_github": skipped_no_github, | |
| "skipped_duplicate": skipped_duplicate, | |
| "errors": errors, | |
| } | |
| def import_from_env( | |
| env_file: str | Path, | |
| category: str | None, | |
| output_path: str | Path, | |
| dry_run: bool = False, | |
| pin_versions: bool = False, | |
| ) -> dict: | |
| """ | |
| Parse a conda ``environment.yml`` and add GitHub-backed packages to | |
| ``repositories.yml``. | |
| Returns dict: {added, skipped_no_github, skipped_duplicate, errors} | |
| """ | |
| env_file_path = Path(env_file) | |
| with open(env_file_path, "r", encoding="utf-8") as f: | |
| env = yaml.safe_load(f) or {} | |
| if not isinstance(env, dict): | |
| raise ValueError("environment.yml must contain a top-level mapping") | |
| effective_category = category or env.get("name", "imported") | |
| pip_packages: list[str] = [] | |
| for dep in env.get("dependencies", []): | |
| if isinstance(dep, dict) and "pip" in dep: | |
| pip_entries = dep.get("pip") or [] | |
| if isinstance(pip_entries, list): | |
| pip_packages.extend([e for e in pip_entries if isinstance(e, str)]) | |
| return _import_package_list(pip_packages, effective_category, Path(output_path), dry_run, pin_versions) | |
| def import_from_requirements( | |
| req_file: str | Path, | |
| category: str | None, | |
| output_path: str | Path, | |
| dry_run: bool = False, | |
| pin_versions: bool = False, | |
| ) -> dict: | |
| """ | |
| Parse a ``requirements.txt`` (or ``.in``) file and add GitHub-backed | |
| packages to ``repositories.yml``. | |
| Returns dict: {added, skipped_no_github, skipped_duplicate, errors} | |
| """ | |
| req_path = Path(req_file) | |
| lines = req_path.read_text(encoding="utf-8").splitlines() | |
| pip_packages = _parse_requirements_lines(lines) | |
| effective_category = category or req_path.stem | |
| return _import_package_list(pip_packages, effective_category, Path(output_path), dry_run, pin_versions) | |
| def import_from_pyproject( | |
| toml_file: str | Path, | |
| category: str | None, | |
| output_path: str | Path, | |
| dry_run: bool = False, | |
| pin_versions: bool = False, | |
| ) -> dict: | |
| """ | |
| Parse a ``pyproject.toml`` (PEP 621 or Poetry) and add GitHub-backed | |
| packages to ``repositories.yml``. | |
| Returns dict: {added, skipped_no_github, skipped_duplicate, errors} | |
| """ | |
| toml_path = Path(toml_file) | |
| pip_packages, project_name = _parse_pyproject_deps(toml_path) | |
| effective_category = category or project_name or toml_path.stem | |
| return _import_package_list(pip_packages, effective_category, Path(output_path), dry_run, pin_versions) | |
| def import_from_file( | |
| file_path: str | Path, | |
| category: str | None = None, | |
| output_path: str | Path = "config/repositories.yml", | |
| dry_run: bool = False, | |
| pin_versions: bool = False, | |
| ) -> dict: | |
| """ | |
| Auto-detect the dependency file format and import GitHub-backed packages | |
| into ``repositories.yml``. | |
| Supported formats: | |
| - conda ``environment.yml`` / ``*.yaml`` | |
| - ``requirements.txt`` / ``requirements.in`` / ``*.txt`` | |
| - ``pyproject.toml`` | |
| Returns dict: {added, skipped_no_github, skipped_duplicate, errors} | |
| """ | |
| path = Path(file_path) | |
| name = path.name.lower() | |
| suffix = path.suffix.lower() | |
| if name == "pyproject.toml" or suffix == ".toml": | |
| return import_from_pyproject(path, category, output_path, dry_run, pin_versions) | |
| elif suffix in (".txt", ".in") or "requirements" in name: | |
| return import_from_requirements(path, category, output_path, dry_run, pin_versions) | |
| elif suffix in (".yml", ".yaml"): | |
| return import_from_env(path, category, output_path, dry_run, pin_versions) | |
| else: | |
| raise ValueError( | |
| f"Unrecognised file format: {path.name}. " "Supported: environment.yml, requirements.txt, pyproject.toml" | |
| ) | |
Xet Storage Details
- Size:
- 13.5 kB
- Xet hash:
- ce66fd394f14b4e7eab503054cd93a5a02b272b4cd4ff5248d91838c8ff2efff
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.