codehygiene / web /src /github.js
brianbellx's picture
Deploy scoring-only Space: remove edit model and fix features
a1415e8 verified
Raw History Blame Contribute Delete
10 kB
const CODE_EXTENSIONS = new Set([
".py", ".pyi", ".js", ".mjs", ".cjs", ".ts", ".tsx", ".jsx", ".c", ".h", ".cc",
".cpp", ".cxx", ".hpp", ".rs", ".go", ".zig", ".java", ".cs", ".lua", ".rb",
".php", ".swift", ".kt", ".scala", ".sh",
]);
const SKIP_DIRS = new Set([
".git", ".hg", ".svn", "__pycache__", "node_modules", "target", "build", "dist",
"venv", ".venv", ".tox", ".mypy_cache", ".pytest_cache",
]);
const SECONDARY_DIRS = new Set([
"test", "tests", "__tests__", "spec", "example", "examples", "benchmark",
"benchmarks", "fixtures",
]);
const TEST_DIRS = new Set(["test", "tests", "__tests__", "spec"]);
const TEST_INFIXES = [".test.", ".spec.", "_test.", "_spec."];
export function isTestPath(relPath) {
const parts = relPath.toLowerCase().split("/");
const baseName = parts[parts.length - 1];
if (parts.slice(0, -1).some((dir) => TEST_DIRS.has(dir))) {
return true;
}
return /^tests?_/.test(baseName) || TEST_INFIXES.some((infix) => baseName.includes(infix));
}
const REPO_HOSTS =new Set(["github.com", "www.github.com", "gitingest.com", "www.gitingest.com"]);
const RAW_HOST = "raw.githubusercontent.com";
const MINIFIED_SUFFIXES = [".min.js", ".min.mjs", ".min.cjs", ".bundle.js"];
const SAFE_SEGMENT = /^[A-Za-z0-9_.-]+$/;
const MAX_FILE_BYTES = 131072;
const MAX_SCORED_FILES = 200;
const FETCH_CONCURRENCY = 12;
export class RepoError extends Error {
constructor(code, message) {
super(message);
this.code = code;
}
}
function describeRepo(owner, rawRepo, ref = "", subpath = "", blobPath = "") {
const repo = rawRepo.endsWith(".git") ? rawRepo.slice(0, -4) : rawRepo;
if (!SAFE_SEGMENT.test(owner) || !SAFE_SEGMENT.test(repo)) {
return null;
}
return { owner, repo, ref, subpath, blobPath, slug: `${owner}/${repo}` };
}
function stripScheme(text) {
return text.replace(/^https?:\/\//i, "");
}
function cleanPath(text) {
const cut = text.search(/[?#]/);
const sliced = cut >= 0 ? text.slice(0, cut) : text;
return sliced.replace(/^\/+|\/+$/g, "");
}
function describeFromSegments(segments) {
const [owner, repo, kind, ref, ...rest] = segments;
if (kind === "tree" && ref) {
return describeRepo(owner, repo, ref, rest.join("/"));
}
if (kind === "blob" && ref && rest.length > 0) {
return describeRepo(owner, repo, ref, "", rest.join("/"));
}
return describeRepo(owner, repo);
}
function describeRawSegments(segments) {
const [owner, repo, ref, ...rest] = segments;
if (!ref || rest.length === 0) {
return null;
}
return describeRepo(owner, repo, ref, "", rest.join("/"));
}
function describeRefSuffix(segments) {
const [owner, repoWithRef] = segments;
const atIndex = repoWithRef.indexOf("@");
if (atIndex <= 0) {
return describeRepo(owner, repoWithRef);
}
return describeRepo(owner, repoWithRef.slice(0, atIndex), repoWithRef.slice(atIndex + 1));
}
export function parseRepoSpec(rawInput) {
const trimmed = rawInput.trim();
if (!trimmed || /\s/.test(trimmed)) {
return null;
}
const withoutScheme = stripScheme(trimmed);
const firstSlash = withoutScheme.indexOf("/");
if (firstSlash <= 0) {
return null;
}
const host = withoutScheme.slice(0, firstSlash).toLowerCase();
const pathText = REPO_HOSTS.has(host) || host === RAW_HOST
? withoutScheme.slice(firstSlash + 1)
: withoutScheme;
const rawParts = cleanPath(pathText).split("/").filter(Boolean);
const segments = rawParts.map(decodeURIComponent);
if (segments.length < 2) {
return null;
}
if (host === RAW_HOST) {
return describeRawSegments(segments);
}
if (segments.length === 2) {
return describeRefSuffix(segments);
}
return describeFromSegments(segments);
}
export function isFullRepoUrl(rawText) {
const text = stripScheme(rawText.trim()).toLowerCase();
const host = text.slice(0, text.indexOf("/"));
return (REPO_HOSTS.has(host) || host === RAW_HOST) && parseRepoSpec(rawText) !== null;
}
export function specLabel(spec) {
return spec.ref ? `${spec.slug}@${spec.ref}` : spec.slug;
}
function fileExtension(relPath) {
const baseName = relPath.slice(relPath.lastIndexOf("/") + 1);
const dotIndex = baseName.lastIndexOf(".");
return dotIndex <= 0 ? "" : baseName.slice(dotIndex).toLowerCase();
}
function hasSkippedDirectory(segments) {
return segments.slice(0, -1).some(
(dir) => dir.startsWith(".") || dir.endsWith(".egg-info") || SKIP_DIRS.has(dir),
);
}
function isScorablePath(relPath, byteSize, subpath) {
const lowerPath = relPath.toLowerCase();
if (MINIFIED_SUFFIXES.some((suffix) => lowerPath.endsWith(suffix))) {
return false;
}
if (byteSize !== null && (byteSize <= 0 || byteSize > MAX_FILE_BYTES)) {
return false;
}
if (subpath && relPath !== subpath && !relPath.startsWith(`${subpath}/`)) {
return false;
}
const segments = relPath.split("/").filter(Boolean);
if (segments.length === 0 || hasSkippedDirectory(segments)) {
return false;
}
return CODE_EXTENSIONS.has(fileExtension(relPath));
}
function pathPriority(relPath, focusPath) {
if (focusPath && relPath === focusPath) {
return -10;
}
const parts = relPath.toLowerCase().split("/");
const inSecondary = parts.slice(0, -1).some((dir) => SECONDARY_DIRS.has(dir));
return (inSecondary ? 100 : 0) + parts.length;
}
function selectEntries(entries, spec, focusPath) {
const scorable = entries.filter((entry) => isScorablePath(entry.path, entry.size, spec.subpath));
scorable.sort((left, right) => {
const rankDiff = pathPriority(left.path, focusPath) - pathPriority(right.path, focusPath);
return rankDiff !== 0 ? rankDiff : left.path.localeCompare(right.path);
});
return scorable.slice(0, MAX_SCORED_FILES);
}
export function selectLocalEntries(entries) {
return selectEntries(entries, { subpath: "" }, "");
}
async function guardedFetch(url, signal, headers = {}) {
try {
return await fetch(url, { signal, headers });
} catch (error) {
if (error.name === "AbortError") {
throw error;
}
throw new RepoError("offline", "Network request failed");
}
}
function sizeOf(value) {
return typeof value === "number" ? value : null;
}
async function fetchApiTree(spec, ref, signal) {
const treeRef = encodeURIComponent(ref);
const url = `https://api.github.com/repos/${spec.slug}/git/trees/${treeRef}?recursive=1`;
const response = await guardedFetch(url, signal);
if (response.status === 404) {
throw new RepoError("not_found", `${spec.slug} not found`);
}
if (response.status === 403 || response.status === 429) {
return { entries: null, rateLimited: true };
}
if (!response.ok) {
return { entries: null, rateLimited: false };
}
const body = await response.json();
if (body.truncated || !Array.isArray(body.tree)) {
return { entries: null, rateLimited: false };
}
const entries = body.tree
.filter((node) => node.type === "blob" && typeof node.path === "string")
.map((node) => ({ path: node.path, size: sizeOf(node.size) }));
return { entries, rateLimited: false };
}
async function fetchJsdelivrTree(spec, ref, signal) {
const url = `https://data.jsdelivr.com/v1/package/gh/${spec.slug}@${encodeURIComponent(ref)}/flat`;
const response = await guardedFetch(url, signal);
if (!response.ok) {
return { entries: null, status: response.status };
}
const payload = await response.json();
if (!Array.isArray(payload.files)) {
return { entries: null, status: 502 };
}
const entries = payload.files.map((item) => ({
path: item.name.replace(/^\//, ""),
size: sizeOf(item.size),
}));
return { entries, status: 200 };
}
async function discoverTree(spec, signal) {
const ref = spec.ref || "HEAD";
const api = await fetchApiTree(spec, ref, signal);
if (api.entries) {
return api.entries;
}
const cdn = await fetchJsdelivrTree(spec, ref, signal);
if (cdn.entries) {
return cdn.entries;
}
if (cdn.status === 404) {
throw new RepoError("not_found", `${spec.slug} not found`);
}
if (api.rateLimited) {
throw new RepoError("rate_limited", "GitHub rate limit reached");
}
throw new RepoError("offline", `Tree request failed (${cdn.status})`);
}
function encodePath(relPath) {
return relPath.split("/").map(encodeURIComponent).join("/");
}
async function fetchFileText(spec, relPath, signal) {
const ref = encodeURIComponent(spec.ref || "HEAD");
const path = encodePath(relPath);
const rawUrl = `https://raw.githubusercontent.com/${spec.slug}/${ref}/${path}`;
const primary = await guardedFetch(rawUrl, signal);
if (primary.ok) {
return primary.text();
}
const cdnUrl = `https://cdn.jsdelivr.net/gh/${spec.slug}@${ref}/${path}`;
const fallback = await guardedFetch(cdnUrl, signal);
return fallback.ok ? fallback.text() : null;
}
export function isReadableText(text) {
return Boolean(text) && text.trim().length > 0 && !text.includes("\0");
}
export async function fetchRepoFiles(spec, { signal, focusPath, onProgress }) {
const tree = await discoverTree(spec, signal);
const targets = selectEntries(tree, spec, focusPath || spec.blobPath);
if (!targets.length) {
throw new RepoError("no_files", `No code files in ${spec.slug}`);
}
const contents = new Array(targets.length).fill(null);
let cursor = 0;
let fetched = 0;
async function drain() {
while (cursor < targets.length) {
const index = cursor;
cursor += 1;
const text = await fetchFileText(spec, targets[index].path, signal);
contents[index] = isReadableText(text) ? text : null;
fetched += 1;
onProgress(fetched, targets.length);
}
}
const lanes = Math.min(FETCH_CONCURRENCY, targets.length);
await Promise.all(Array.from({ length: lanes }, drain));
const files = targets
.map((entry, index) => ({ path: entry.path, content: contents[index] }))
.filter((file) => file.content !== null);
if (!files.length) {
throw new RepoError("no_files", `No readable files in ${spec.slug}`);
}
return { files };
}