| """Read-only, SHA-pinned GitHub scope reports. |
| |
| The complete Git tree supplies exhaustive inventory facts. Only a bounded subset |
| of files is inspected for source structure; these are NEVER called a full code |
| audit. Nothing from repository content is interpreted as instructions. |
| """ |
| from __future__ import annotations |
|
|
| from collections import Counter |
| import json |
| import re |
| from pathlib import PurePosixPath |
|
|
| from github_explorer_v223 import ExplorerError, safe_path, scope_stats |
|
|
| MAX_SCOPES = 12 |
| LIMITS = {"overview": 4, "structure": 12, "deep": 24} |
| TEXT_LIMIT = 120000 |
| TOTAL_TEXT = {"overview": 45000, "structure": 160000, "deep": 300000} |
| BINARY_EXTS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".ico", ".pdf", |
| ".zip", ".tar", ".gz", ".7z", ".jar", ".wasm", ".ttf", ".otf", |
| ".mp3", ".mp4", ".ogg", ".wav", ".db", ".sqlite", ".lock"} |
| SENSITIVE = re.compile(r"(?i)(^|/)(?:\.env(?:\..*)?|id_rsa|id_ed25519|" |
| r".*(?:private[_-]?key|credentials|secret[_-]?token).*)$") |
| LANGUAGES = {".py": "Python", ".js": "JavaScript", ".mjs": "JavaScript", |
| ".ts": "TypeScript", ".tsx": "TypeScript", ".jsx": "JavaScript", |
| ".kt": "Kotlin", ".java": "Java", ".rs": "Rust", ".go": "Go", |
| ".c": "C", ".h": "C header", ".cpp": "C++", ".cs": "C#", |
| ".html": "HTML", ".css": "CSS", ".scss": "SCSS", ".sh": "Shell", |
| ".json": "JSON", ".toml": "TOML", ".yml": "YAML", |
| ".yaml": "YAML", ".xml": "XML", ".md": "Markdown", |
| ".rst": "reStructuredText", ".sql": "SQL", ".swift": "Swift"} |
| SYMBOL = re.compile(r"^\s*(?:async\s+)?(?:def|class|function|export\s+(?:default\s+)?(?:async\s+)?function|" |
| r"export\s+class|interface|struct|fn|fun)\s+([A-Za-z_$][\w$]*)\b", re.M) |
| IMPORT = re.compile(r"^\s*(?:from\s+([\w.]+)\s+import|import\s+([\w., ]+)|" |
| r"(?:const|let|var)\s+.+?\s*=\s*require\(['\"]([^'\"]+)['\"]\)|" |
| r"import\s+.*?\s+from\s+['\"]([^'\"]+)['\"])\s*", re.M) |
| TASKS = re.compile(r"\b(?:TODO|FIXME|XXX)\b") |
| SAFE_HEADER = re.compile(r"(?i)(?:token|password|secret|api[_-]?key|private[_-]?key|bearer|credential)\s*[:=]") |
|
|
|
|
| def selection_paths(value): |
| """Validate an explicit list, a single path, or command-delimited paths.""" |
| if isinstance(value, str): |
| value = value.strip() |
| if value.startswith("["): |
| try: |
| value = json.loads(value) |
| except ValueError as exc: |
| raise ExplorerError("Report paths must be a JSON array or | separated paths") from exc |
| else: |
| value = value.split("|") |
| if not isinstance(value, list) or not 1 <= len(value) <= MAX_SCOPES: |
| raise ExplorerError("Choose between 1 and 12 folders/files") |
| normalized = [] |
| for item in value: |
| if not isinstance(item, str): |
| raise ExplorerError("Report paths must be strings") |
| path = safe_path(item.strip()) |
| if path not in normalized: |
| normalized.append(path) |
| return normalized |
|
|
|
|
| def file_kind(path): |
| name = PurePosixPath(path).name |
| extension = PurePosixPath(path).suffix.lower() |
| if name.lower() in ("dockerfile", "makefile"): |
| return "Build/configuration" |
| if name.lower().startswith("readme") or extension in (".md", ".rst"): |
| return "Documentation" |
| if name.lower().startswith("test_") or name.lower().endswith((".test.js", ".spec.js", ".test.ts", ".spec.ts")): |
| return "Automated test" |
| if "/tests/" in "/" + path + "/" or "/test/" in "/" + path + "/": |
| return "Test/support" |
| if name.lower().endswith(".lock") or extension in (".toml", ".yaml", ".yml", ".json", ".xml"): |
| return "Configuration / data" |
| if extension in (".png", ".jpg", ".jpeg", ".webp", ".gif", ".svg", ".mp3", ".mp4"): |
| return "Asset" |
| return LANGUAGES.get(extension, "Unknown/other") + " source or resource" |
|
|
|
|
| def safe_heading(text): |
| value = " ".join(str(text or "").split())[:180] |
| return "" if SAFE_HEADER.search(value) or len(value) < 8 else value |
|
|
|
|
| def structure(path, body, depth): |
| lines = body.splitlines() |
| result = {"language": LANGUAGES.get(PurePosixPath(path).suffix.lower(), "Other"), |
| "lineCount": len(lines), "purposeEvidence": "", "symbols": [], |
| "imports": [], "todoMarkers": 0, "analysisType": "static-source-structure"} |
| |
| for line in lines[:20]: |
| stripped = line.strip() |
| if stripped.startswith(("# ", "// ", "* ", '"""', "'''")): |
| value = safe_heading(stripped.lstrip("#/*'\" ").strip()) |
| if value and not value.lower().startswith(("copyright", "license", "http", "spdx")): |
| result["purposeEvidence"] = value |
| break |
| if depth == "overview": |
| return result |
| seen_symbols = set() |
| for match in SYMBOL.finditer(body): |
| name = match.group(1) |
| if name not in seen_symbols and len(result["symbols"]) < (30 if depth == "deep" else 14): |
| result["symbols"].append({"name": name, "line": body.count("\n", 0, match.start()) + 1}) |
| seen_symbols.add(name) |
| seen_imports = set() |
| for match in IMPORT.finditer(body): |
| name = next((x for x in match.groups() if x), "").strip() |
| name = name.split(",")[0].strip() |
| if name and len(name) <= 100 and name not in seen_imports and len(result["imports"]) < 20: |
| result["imports"].append(name) |
| seen_imports.add(name) |
| result["todoMarkers"] = len(TASKS.findall(body)) |
| return result |
|
|
|
|
| def internal_import_refs(file, full_index): |
| """Match declared imports to actual verified repository paths, not guesses.""" |
| base = PurePosixPath(file["path"]).parent |
| links = [] |
| for ref in file["imports"]: |
| targets = [] |
| if ref.startswith("."): |
| joined = str(base / ref) |
| if ".." in joined.split("/"): |
| continue |
| targets.append(joined) |
| else: |
| targets.append(ref.replace(".", "/")) |
| for prefix in targets: |
| for target in (prefix, prefix + ".py", prefix + ".js", prefix + ".ts", |
| prefix + ".mjs", prefix + "/__init__.py", prefix + "/index.js"): |
| if target in full_index and target != file["path"]: |
| if not any(item["target"] == target for item in links): |
| links.append({"importRef": ref, "target": target}) |
| break |
| return links[:20] |
|
|
|
|
| def build_report(snapshot, requested, depth="structure", read_file=None): |
| if depth not in LIMITS: |
| raise ExplorerError("Report depth must be overview, structure or deep") |
| paths = selection_paths(requested) |
| entries = snapshot.get("items") |
| if snapshot.get("complete") is not True or not isinstance(entries, list): |
| raise ExplorerError("Complete verified Git tree is required for a repository report") |
| index = {x["path"]: x for x in entries} |
| scopes = [] |
| union = {} |
| explicit_files = [] |
| for path in paths: |
| if path: |
| node = index.get(path) |
| if not node: |
| raise ExplorerError("Selected scope is missing from verified source: " + path) |
| else: |
| node = {"type": "dir"} |
| if node["type"] == "dir": |
| stats = scope_stats(snapshot, path) |
| prefix = path + "/" if path else "" |
| subset = [x for x in entries if x["type"] == "file" and x["path"].startswith(prefix)] |
| scopes.append({"path": path or "/", "type": "dir", |
| "files": stats["recursiveFiles"], "directories": stats["recursiveDirectories"], |
| "gitBlobBytes": stats["gitBlobBytes"], "sizeComplete": stats["sizeComplete"]}) |
| elif node["type"] == "file": |
| subset = [node] |
| explicit_files.append(path) |
| scopes.append({"path": path, "type": "file", "files": 1, |
| "directories": 0, "gitBlobBytes": node["sizeBytes"], |
| "sizeComplete": isinstance(node["sizeBytes"], int)}) |
| else: |
| raise ExplorerError("Unknown scope type") |
| for item in subset: |
| union[item["path"]] = item |
| selected = list(union.values()) |
| known_sizes = all(isinstance(x.get("sizeBytes"), int) for x in selected) |
| extensions = Counter(LANGUAGES.get(PurePosixPath(x["path"]).suffix.lower(), "Other") for x in selected) |
| purposes = Counter(file_kind(x["path"]) for x in selected) |
| top = Counter(x["path"].split("/")[0] if "/" in x["path"] else "(root files)" for x in selected) |
| |
| |
| def priority(item): |
| path = item["path"] |
| base = PurePosixPath(path).name.lower() |
| rank = (0 if path in explicit_files else |
| 1 if base.startswith(("readme", "architecture", "overview")) else |
| 2 if base in ("pyproject.toml", "package.json", "requirements.txt", "dockerfile") else |
| 3 if "/docs/" in "/" + path + "/" else 4) |
| return (rank, path.count("/"), path.casefold()) |
| sorted_candidates = sorted(selected, key=priority) |
| |
| |
| chosen = [] |
| chosen_paths = set() |
| def add_candidate(item): |
| if item["path"] not in chosen_paths: |
| chosen.append(item) |
| chosen_paths.add(item["path"]) |
| for item in sorted_candidates: |
| if item["path"] in explicit_files: |
| add_candidate(item) |
| for root in paths: |
| if root and index[root]["type"] == "file": |
| continue |
| prefix = root + "/" if root else "" |
| grouped = {} |
| for item in sorted_candidates: |
| path = item["path"] |
| if not path.startswith(prefix): |
| continue |
| remainder = path[len(prefix):] |
| group = remainder.split("/")[0] if "/" in remainder else "(direct files)" |
| grouped.setdefault(group, item) |
| for item in grouped.values(): |
| add_candidate(item) |
| for item in sorted_candidates: |
| add_candidate(item) |
| budget = TOTAL_TEXT[depth] |
| scanned = [] |
| skipped = Counter() |
| for item in chosen: |
| if len(scanned) >= LIMITS[depth]: |
| break |
| path = item["path"] |
| suffix = PurePosixPath(path).suffix.lower() |
| if SENSITIVE.search(path) or suffix in BINARY_EXTS: |
| skipped["sensitive or non-text paths"] += 1 |
| continue |
| size = item.get("sizeBytes") |
| if not isinstance(size, int) or size > TEXT_LIMIT or size > budget: |
| skipped["oversize / unknown size / byte budget"] += 1 |
| continue |
| if read_file is None: |
| skipped["source reader not provided"] += 1 |
| continue |
| try: |
| record = read_file(path, snapshot["sourceSha"], min(TEXT_LIMIT, budget)) |
| except Exception: |
| record = None |
| if not isinstance(record, dict) or not isinstance(record.get("body"), str): |
| skipped["unavailable UTF-8 source"] += 1 |
| continue |
| if record.get("sha") != item.get("sha"): |
| skipped["Git blob SHA mismatch"] += 1 |
| continue |
| body = record["body"] |
| if len(body.encode("utf-8")) > budget: |
| skipped["source byte budget"] += 1 |
| continue |
| budget -= len(body.encode("utf-8")) |
| inspected = structure(path, body, depth) |
| scanned.append({"path": path, "blobSha": item["sha"], "sizeBytes": size, |
| "kind": file_kind(path), **inspected}) |
| for file in scanned: |
| file["verifiedLocalImportMatches"] = internal_import_refs(file, index) |
| scanned_paths = {x["path"] for x in scanned} |
| return {"schema": "swrlz-github-scope-report-v1", "repository": snapshot["repository"], |
| "branch": snapshot["branch"], "sourceSha": snapshot["sourceSha"], |
| "depth": depth, "scopes": scopes, |
| "inventory": {"complete": True, "files": len(selected), |
| "gitBlobBytes": sum(x["sizeBytes"] for x in selected) if known_sizes else None, |
| "sizeComplete": known_sizes, |
| "languageCounts": dict(extensions.most_common(16)), |
| "roleCounts": dict(purposes.most_common(12)), |
| "topLevelCounts": dict(top.most_common(18))}, |
| "inspection": {"limit": LIMITS[depth], "inspected": len(scanned), |
| "totalFiles": len(selected), "notInspected": len(selected)-len(scanned), |
| "explicitFileCount": len(explicit_files), |
| "explicitFilesNotInspected": [p for p in explicit_files if p not in scanned_paths], |
| "skippedReasons": dict(skipped), |
| "complete": len(scanned) == len(selected), |
| "note": "Complete Git-tree scope inventory; source descriptions and symbols cover only listed inspected UTF-8 files. Purpose and dependency roles are inferred, not execution-verified."}, |
| "files": scanned, "readOnly": True} |
|
|
|
|
| def markdown_report(data, limit=15000): |
| """Bounded plain Markdown for explicit Chat @GitHub report commands.""" |
| inv, cov = data["inventory"], data["inspection"] |
| out = ["## GitHub source report · " + data["repository"], |
| "Branch: " + data["branch"] + " · commit " + data["sourceSha"], |
| "Depth: " + data["depth"], |
| "Scopes: " + ", ".join(x["path"] for x in data["scopes"]), |
| "Verified Git-tree inventory: " + str(inv["files"]) + " unique files", |
| "Source contents inspected: " + str(cov["inspected"]) + "/" + str(cov["totalFiles"]), |
| cov["note"], |
| "### Source categories"] |
| out += [("- " + str(k) + ": " + str(v)) for k,v in inv["languageCounts"].items()] |
| out.append("### Inspected files") |
| for item in data["files"]: |
| out.append("- " + item["path"] + " (" + item["kind"] + ", " + str(item["lineCount"]) + |
| " lines, Git blob " + item["blobSha"][:12] + ")") |
| if item["purposeEvidence"]: |
| out.append(" - Opening comment: " + item["purposeEvidence"]) |
| if item["symbols"]: |
| out.append(" - Declared symbols: " + ", ".join(s["name"] + ":" + str(s["line"]) |
| for s in item["symbols"][:15])) |
| if item["imports"]: |
| out.append(" - Import references: " + ", ".join(item["imports"][:15])) |
| if item["verifiedLocalImportMatches"]: |
| out.append(" - Confirmed paths matching imports (static): " + ", ".join(x["target"] for x in item["verifiedLocalImportMatches"])) |
| if item["todoMarkers"]: |
| out.append(" - TODO/FIXME markers: " + str(item["todoMarkers"])) |
| if cov["explicitFilesNotInspected"]: |
| out.append("Requested files not inspected: " + ", ".join(cov["explicitFilesNotInspected"])) |
| out.append("Source inspection is static, not a behavioral or security audit.") |
| text = "\n".join(out) |
| return text[:limit] + ("\n(Report display truncated; narrow scope in GitHub Explorer for more.)" |
| if len(text) > limit else "") |
|
|
