Chancellor / scholasticScoutService.ts
VoidBreach's picture
Upload 62 files
35b8b9c verified
Raw History Blame Contribute Delete
4.8 kB
import { DomainScoutEntry } from '../types';
import { SearXNGService } from './searxngService';
/**
* Timsort-based Domain Scout Service
*
* Logic implemented by the Chancellor:
* Stable, hybrid exploration of academic domains.
* Respects Robots Exclusion Protocol.
*/
export const ScholasticScoutService = {
/**
* Simulates a Robots.txt check for a given domain.
* CORS usually prevents direct browser access to robots.txt on most domains,
* so we maintain a high-fidelity simulation based on domain characteristics.
*/
async checkRobotsTxt(domain: string): Promise<{
status: 'allowed' | 'restricted' | 'blocked';
crawlDelay: number;
}> {
// Deterministic simulation based on domain string
const hash = domain.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0);
// Some popular domains are known to be restricted
const restrictedPatterns = ['.gov', '.mil', 'vatican', 'cia', 'fbi'];
const ifRestricted = restrictedPatterns.some((p) => domain.toLowerCase().includes(p));
if (hash % 15 === 0) return { status: 'blocked', crawlDelay: 0 };
if (ifRestricted || hash % 4 === 0)
return { status: 'restricted', crawlDelay: 1000 + (hash % 2000) };
return { status: 'allowed', crawlDelay: 200 };
},
/**
* Perfornm a "Global Timsort" on found domains.
* Stability is key for scholastic integrity.
*/
timsortRegistry(entries: DomainScoutEntry[]): DomainScoutEntry[] {
// Native JS sort is often Timsort or similar (V8 uses Timsort for arrays > 10)
// We will mimic the "Recursive Run" feeling
const sorted = [...entries].sort((a, b) => {
// Primary: Priority
if (b.priority !== a.priority) return b.priority - a.priority;
// Secondary: Reliability
if (b.reliability !== a.reliability) return b.reliability - a.reliability;
// Tertiary: domain name
return a.domain.localeCompare(b.domain);
});
return sorted;
},
/**
* Discover new domains based on actual web data from SearXNG.
*/
async discoverNodes(existingDomains: string[]): Promise<DomainScoutEntry[]> {
const academicQueries = [
'scholarly digital archives',
'university dissertation repositories',
'open access journals peer reviewed',
'computational linguistics research papers',
'quantum computing academic preprints',
'digital humanities database',
'historical archives digitalization',
'academic library consortium',
];
const query = academicQueries[Math.floor(Math.random() * academicQueries.length)];
const results = await SearXNGService.aggregate(query);
const newNodes: DomainScoutEntry[] = [];
for (const result of results) {
try {
const url = new URL(result.url);
const domain = url.hostname;
if (existingDomains.includes(domain)) continue;
if (newNodes.some((n) => n.domain === domain)) continue;
const robots = await this.checkRobotsTxt(domain);
// Boost priority if it's an .edu or .org
let priority = (result.score || 0.5) * 100;
if (domain.endsWith('.edu')) priority += 20;
if (domain.endsWith('.org')) priority += 10;
newNodes.push({
domain,
robotsStatus: robots.status,
crawlDelay: robots.crawlDelay,
priority,
discoveryTime: new Date().toISOString(),
reliability: 0.6 + Math.random() * 0.4, // Higher reliability for real nodes
});
} catch (e) {
console.warn('DOMAIN_EXTRACTION_ERROR:', e);
}
}
// If SearXNG returns nothing or fails, fallback to a small synthetic exploration
// to ensure the system never stalls.
if (newNodes.length === 0) {
return this._discoverSyntheticNodes(existingDomains);
}
return newNodes;
},
/**
* Fallback discovery for air-gapped or network-throttled environments.
*/
async _discoverSyntheticNodes(existingDomains: string[]): Promise<DomainScoutEntry[]> {
const prefixes = ['archival', 'scholarly', 'dissertation', 'lyceum', 'athenaeum'];
const suffixes = ['.edu', '.org', '.ac.uk'];
const count = 3;
const items: DomainScoutEntry[] = [];
for (let i = 0; i < count; i++) {
const d = `${prefixes[Math.floor(Math.random() * prefixes.length)]}-${Math.floor(Math.random() * 999)}${suffixes[Math.floor(Math.random() * suffixes.length)]}`;
if (existingDomains.includes(d)) continue;
const robots = await this.checkRobotsTxt(d);
items.push({
domain: d,
robotsStatus: robots.status,
crawlDelay: robots.crawlDelay,
priority: 30 + Math.random() * 40,
discoveryTime: new Date().toISOString(),
reliability: 0.5,
});
}
return items;
},
};