Download scholasticScoutService.ts from VoidBreach/Chancellor: direct link, hf CLI and curl.
- Browser
- Download file 4.8 kB
-
https://huggingface.co/VoidBreach/Chancellor/resolve/main/scholasticScoutService.ts
- Command line
-
hf download hf://VoidBreach/Chancellor/scholasticScoutService.ts
-
curl -L -o scholasticScoutService.ts https://huggingface.co/VoidBreach/Chancellor/resolve/main/scholasticScoutService.ts
4.8 kB
| import { DomainScoutEntry } from '../types'; | |
| import { SearXNGService } from './searxngService'; | |
| /** | |
| * Timsort-based Domain Scout Service | |
| * | |
| * Logic implemented by the Chancellor: | |
| * Stable, hybrid exploration of academic domains. | |
| * Respects Robots Exclusion Protocol. | |
| */ | |
| export const ScholasticScoutService = { | |
| /** | |
| * Simulates a Robots.txt check for a given domain. | |
| * CORS usually prevents direct browser access to robots.txt on most domains, | |
| * so we maintain a high-fidelity simulation based on domain characteristics. | |
| */ | |
| async checkRobotsTxt(domain: string): Promise<{ | |
| status: 'allowed' | 'restricted' | 'blocked'; | |
| crawlDelay: number; | |
| }> { | |
| // Deterministic simulation based on domain string | |
| const hash = domain.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0); | |
| // Some popular domains are known to be restricted | |
| const restrictedPatterns = ['.gov', '.mil', 'vatican', 'cia', 'fbi']; | |
| const ifRestricted = restrictedPatterns.some((p) => domain.toLowerCase().includes(p)); | |
| if (hash % 15 === 0) return { status: 'blocked', crawlDelay: 0 }; | |
| if (ifRestricted || hash % 4 === 0) | |
| return { status: 'restricted', crawlDelay: 1000 + (hash % 2000) }; | |
| return { status: 'allowed', crawlDelay: 200 }; | |
| }, | |
| /** | |
| * Perfornm a "Global Timsort" on found domains. | |
| * Stability is key for scholastic integrity. | |
| */ | |
| timsortRegistry(entries: DomainScoutEntry[]): DomainScoutEntry[] { | |
| // Native JS sort is often Timsort or similar (V8 uses Timsort for arrays > 10) | |
| // We will mimic the "Recursive Run" feeling | |
| const sorted = [...entries].sort((a, b) => { | |
| // Primary: Priority | |
| if (b.priority !== a.priority) return b.priority - a.priority; | |
| // Secondary: Reliability | |
| if (b.reliability !== a.reliability) return b.reliability - a.reliability; | |
| // Tertiary: domain name | |
| return a.domain.localeCompare(b.domain); | |
| }); | |
| return sorted; | |
| }, | |
| /** | |
| * Discover new domains based on actual web data from SearXNG. | |
| */ | |
| async discoverNodes(existingDomains: string[]): Promise<DomainScoutEntry[]> { | |
| const academicQueries = [ | |
| 'scholarly digital archives', | |
| 'university dissertation repositories', | |
| 'open access journals peer reviewed', | |
| 'computational linguistics research papers', | |
| 'quantum computing academic preprints', | |
| 'digital humanities database', | |
| 'historical archives digitalization', | |
| 'academic library consortium', | |
| ]; | |
| const query = academicQueries[Math.floor(Math.random() * academicQueries.length)]; | |
| const results = await SearXNGService.aggregate(query); | |
| const newNodes: DomainScoutEntry[] = []; | |
| for (const result of results) { | |
| try { | |
| const url = new URL(result.url); | |
| const domain = url.hostname; | |
| if (existingDomains.includes(domain)) continue; | |
| if (newNodes.some((n) => n.domain === domain)) continue; | |
| const robots = await this.checkRobotsTxt(domain); | |
| // Boost priority if it's an .edu or .org | |
| let priority = (result.score || 0.5) * 100; | |
| if (domain.endsWith('.edu')) priority += 20; | |
| if (domain.endsWith('.org')) priority += 10; | |
| newNodes.push({ | |
| domain, | |
| robotsStatus: robots.status, | |
| crawlDelay: robots.crawlDelay, | |
| priority, | |
| discoveryTime: new Date().toISOString(), | |
| reliability: 0.6 + Math.random() * 0.4, // Higher reliability for real nodes | |
| }); | |
| } catch (e) { | |
| console.warn('DOMAIN_EXTRACTION_ERROR:', e); | |
| } | |
| } | |
| // If SearXNG returns nothing or fails, fallback to a small synthetic exploration | |
| // to ensure the system never stalls. | |
| if (newNodes.length === 0) { | |
| return this._discoverSyntheticNodes(existingDomains); | |
| } | |
| return newNodes; | |
| }, | |
| /** | |
| * Fallback discovery for air-gapped or network-throttled environments. | |
| */ | |
| async _discoverSyntheticNodes(existingDomains: string[]): Promise<DomainScoutEntry[]> { | |
| const prefixes = ['archival', 'scholarly', 'dissertation', 'lyceum', 'athenaeum']; | |
| const suffixes = ['.edu', '.org', '.ac.uk']; | |
| const count = 3; | |
| const items: DomainScoutEntry[] = []; | |
| for (let i = 0; i < count; i++) { | |
| const d = `${prefixes[Math.floor(Math.random() * prefixes.length)]}-${Math.floor(Math.random() * 999)}${suffixes[Math.floor(Math.random() * suffixes.length)]}`; | |
| if (existingDomains.includes(d)) continue; | |
| const robots = await this.checkRobotsTxt(d); | |
| items.push({ | |
| domain: d, | |
| robotsStatus: robots.status, | |
| crawlDelay: robots.crawlDelay, | |
| priority: 30 + Math.random() * 40, | |
| discoveryTime: new Date().toISOString(), | |
| reliability: 0.5, | |
| }); | |
| } | |
| return items; | |
| }, | |
| }; | |