const CODE_EXTENSIONS = new Set([ ".py", ".pyi", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".c", ".h", ".cc", ".cpp", ".cxx", ".hpp", ".rs", ".go", ".zig", ".java", ".cs", ".lua", ".rb", ".php", ".swift", ".kt", ".scala", ".sh", ]); const SKIP_DIRS = new Set([ ".git", ".hg", ".svn", "__pycache__", "node_modules", "target", "build", "dist", "venv", ".venv", ".tox", ".mypy_cache", ".pytest_cache", ]); const SECONDARY_DIRS = new Set([ "test", "tests", "__tests__", "spec", "example", "examples", "benchmark", "benchmarks", "fixtures", ]); const TEST_DIRS = new Set(["test", "tests", "__tests__", "spec"]); const TEST_INFIXES = [".test.", ".spec.", "_test.", "_spec."]; export function isTestPath(relPath) { const parts = relPath.toLowerCase().split("/"); const baseName = parts[parts.length - 1]; if (parts.slice(0, -1).some((dir) => TEST_DIRS.has(dir))) { return true; } return /^tests?_/.test(baseName) || TEST_INFIXES.some((infix) => baseName.includes(infix)); } const REPO_HOSTS =new Set(["github.com", "www.github.com", "gitingest.com", "www.gitingest.com"]); const RAW_HOST = "raw.githubusercontent.com"; const MINIFIED_SUFFIXES = [".min.js", ".min.mjs", ".min.cjs", ".bundle.js"]; const SAFE_SEGMENT = /^[A-Za-z0-9_.-]+$/; const MAX_FILE_BYTES = 131072; const MAX_SCORED_FILES = 200; const FETCH_CONCURRENCY = 12; export class RepoError extends Error { constructor(code, message) { super(message); this.code = code; } } function describeRepo(owner, rawRepo, ref = "", subpath = "", blobPath = "") { const repo = rawRepo.endsWith(".git") ? rawRepo.slice(0, -4) : rawRepo; if (!SAFE_SEGMENT.test(owner) || !SAFE_SEGMENT.test(repo)) { return null; } return { owner, repo, ref, subpath, blobPath, slug: `${owner}/${repo}` }; } function stripScheme(text) { return text.replace(/^https?:\/\//i, ""); } function cleanPath(text) { const cut = text.search(/[?#]/); const sliced = cut >= 0 ? text.slice(0, cut) : text; return sliced.replace(/^\/+|\/+$/g, ""); } function describeFromSegments(segments) { const [owner, repo, kind, ref, ...rest] = segments; if (kind === "tree" && ref) { return describeRepo(owner, repo, ref, rest.join("/")); } if (kind === "blob" && ref && rest.length > 0) { return describeRepo(owner, repo, ref, "", rest.join("/")); } return describeRepo(owner, repo); } function describeRawSegments(segments) { const [owner, repo, ref, ...rest] = segments; if (!ref || rest.length === 0) { return null; } return describeRepo(owner, repo, ref, "", rest.join("/")); } function describeRefSuffix(segments) { const [owner, repoWithRef] = segments; const atIndex = repoWithRef.indexOf("@"); if (atIndex <= 0) { return describeRepo(owner, repoWithRef); } return describeRepo(owner, repoWithRef.slice(0, atIndex), repoWithRef.slice(atIndex + 1)); } export function parseRepoSpec(rawInput) { const trimmed = rawInput.trim(); if (!trimmed || /\s/.test(trimmed)) { return null; } const withoutScheme = stripScheme(trimmed); const firstSlash = withoutScheme.indexOf("/"); if (firstSlash <= 0) { return null; } const host = withoutScheme.slice(0, firstSlash).toLowerCase(); const pathText = REPO_HOSTS.has(host) || host === RAW_HOST ? withoutScheme.slice(firstSlash + 1) : withoutScheme; const rawParts = cleanPath(pathText).split("/").filter(Boolean); const segments = rawParts.map(decodeURIComponent); if (segments.length < 2) { return null; } if (host === RAW_HOST) { return describeRawSegments(segments); } if (segments.length === 2) { return describeRefSuffix(segments); } return describeFromSegments(segments); } export function isFullRepoUrl(rawText) { const text = stripScheme(rawText.trim()).toLowerCase(); const host = text.slice(0, text.indexOf("/")); return (REPO_HOSTS.has(host) || host === RAW_HOST) && parseRepoSpec(rawText) !== null; } export function specLabel(spec) { return spec.ref ? `${spec.slug}@${spec.ref}` : spec.slug; } function fileExtension(relPath) { const baseName = relPath.slice(relPath.lastIndexOf("/") + 1); const dotIndex = baseName.lastIndexOf("."); return dotIndex <= 0 ? "" : baseName.slice(dotIndex).toLowerCase(); } function hasSkippedDirectory(segments) { return segments.slice(0, -1).some( (dir) => dir.startsWith(".") || dir.endsWith(".egg-info") || SKIP_DIRS.has(dir), ); } function isScorablePath(relPath, byteSize, subpath) { const lowerPath = relPath.toLowerCase(); if (MINIFIED_SUFFIXES.some((suffix) => lowerPath.endsWith(suffix))) { return false; } if (byteSize !== null && (byteSize <= 0 || byteSize > MAX_FILE_BYTES)) { return false; } if (subpath && relPath !== subpath && !relPath.startsWith(`${subpath}/`)) { return false; } const segments = relPath.split("/").filter(Boolean); if (segments.length === 0 || hasSkippedDirectory(segments)) { return false; } return CODE_EXTENSIONS.has(fileExtension(relPath)); } function pathPriority(relPath, focusPath) { if (focusPath && relPath === focusPath) { return -10; } const parts = relPath.toLowerCase().split("/"); const inSecondary = parts.slice(0, -1).some((dir) => SECONDARY_DIRS.has(dir)); return (inSecondary ? 100 : 0) + parts.length; } function selectEntries(entries, spec, focusPath) { const scorable = entries.filter((entry) => isScorablePath(entry.path, entry.size, spec.subpath)); scorable.sort((left, right) => { const rankDiff = pathPriority(left.path, focusPath) - pathPriority(right.path, focusPath); return rankDiff !== 0 ? rankDiff : left.path.localeCompare(right.path); }); return scorable.slice(0, MAX_SCORED_FILES); } export function selectLocalEntries(entries) { return selectEntries(entries, { subpath: "" }, ""); } async function guardedFetch(url, signal, headers = {}) { try { return await fetch(url, { signal, headers }); } catch (error) { if (error.name === "AbortError") { throw error; } throw new RepoError("offline", "Network request failed"); } } function sizeOf(value) { return typeof value === "number" ? value : null; } async function fetchApiTree(spec, ref, signal) { const treeRef = encodeURIComponent(ref); const url = `https://api.github.com/repos/${spec.slug}/git/trees/${treeRef}?recursive=1`; const response = await guardedFetch(url, signal); if (response.status === 404) { throw new RepoError("not_found", `${spec.slug} not found`); } if (response.status === 403 || response.status === 429) { return { entries: null, rateLimited: true }; } if (!response.ok) { return { entries: null, rateLimited: false }; } const body = await response.json(); if (body.truncated || !Array.isArray(body.tree)) { return { entries: null, rateLimited: false }; } const entries = body.tree .filter((node) => node.type === "blob" && typeof node.path === "string") .map((node) => ({ path: node.path, size: sizeOf(node.size) })); return { entries, rateLimited: false }; } async function fetchJsdelivrTree(spec, ref, signal) { const url = `https://data.jsdelivr.com/v1/package/gh/${spec.slug}@${encodeURIComponent(ref)}/flat`; const response = await guardedFetch(url, signal); if (!response.ok) { return { entries: null, status: response.status }; } const payload = await response.json(); if (!Array.isArray(payload.files)) { return { entries: null, status: 502 }; } const entries = payload.files.map((item) => ({ path: item.name.replace(/^\//, ""), size: sizeOf(item.size), })); return { entries, status: 200 }; } async function discoverTree(spec, signal) { const ref = spec.ref || "HEAD"; const api = await fetchApiTree(spec, ref, signal); if (api.entries) { return api.entries; } const cdn = await fetchJsdelivrTree(spec, ref, signal); if (cdn.entries) { return cdn.entries; } if (cdn.status === 404) { throw new RepoError("not_found", `${spec.slug} not found`); } if (api.rateLimited) { throw new RepoError("rate_limited", "GitHub rate limit reached"); } throw new RepoError("offline", `Tree request failed (${cdn.status})`); } function encodePath(relPath) { return relPath.split("/").map(encodeURIComponent).join("/"); } async function fetchFileText(spec, relPath, signal) { const ref = encodeURIComponent(spec.ref || "HEAD"); const path = encodePath(relPath); const rawUrl = `https://raw.githubusercontent.com/${spec.slug}/${ref}/${path}`; const primary = await guardedFetch(rawUrl, signal); if (primary.ok) { return primary.text(); } const cdnUrl = `https://cdn.jsdelivr.net/gh/${spec.slug}@${ref}/${path}`; const fallback = await guardedFetch(cdnUrl, signal); return fallback.ok ? fallback.text() : null; } export function isReadableText(text) { return Boolean(text) && text.trim().length > 0 && !text.includes("\0"); } export async function fetchRepoFiles(spec, { signal, focusPath, onProgress }) { const tree = await discoverTree(spec, signal); const targets = selectEntries(tree, spec, focusPath || spec.blobPath); if (!targets.length) { throw new RepoError("no_files", `No code files in ${spec.slug}`); } const contents = new Array(targets.length).fill(null); let cursor = 0; let fetched = 0; async function drain() { while (cursor < targets.length) { const index = cursor; cursor += 1; const text = await fetchFileText(spec, targets[index].path, signal); contents[index] = isReadableText(text) ? text : null; fetched += 1; onProgress(fetched, targets.length); } } const lanes = Math.min(FETCH_CONCURRENCY, targets.length); await Promise.all(Array.from({ length: lanes }, drain)); const files = targets .map((entry, index) => ({ path: entry.path, content: contents[index] })) .filter((file) => file.content !== null); if (!files.length) { throw new RepoError("no_files", `No readable files in ${spec.slug}`); } return { files }; }