File size: 3,181 Bytes
35b8b9c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
export interface MappingResult {
  url: string;
  isEthical: boolean;
  botSafety: number;
}

const calculateSafety = (url: string): number => {
  // Computational metaphor for String Theory resonance:
  // Simulate harmonic vibration across multiple dimensional frequencies
  let resonance = 0;
  for (let i = 0; i < url.length; i++) {
    const charCode = url.charCodeAt(i);
    // Apply non-linear harmonic transformations
    resonance +=
      Math.sin(charCode * 0.137) * Math.cos(charCode * 0.051) * Math.tan(charCode * 0.017);
  }
  // Normalize to [0, 1] using a sigmoid transformation function
  const normalized = 1 / (1 + Math.exp(-Math.abs(resonance) * 0.05));
  // Scale and shift to the target range [0.9, 0.999]
  return 0.9 + normalized * 0.099;
};

const ETHICAL_REGISTRY = [
  { url: 'https://human-scholars.org', isEthical: true, botSafety: 0.95 },
  { url: 'https://open-library-project.edu', isEthical: true, botSafety: 0.9 },
  { url: 'https://vulnerable-scan-tool.net', isEthical: false, botSafety: 0.1 },
  { url: 'https://bot-farm-hub.xyz', isEthical: false, botSafety: 0.05 },
  {
    url: 'https://developer.mozilla.org',
    isEthical: true,
    botSafety: calculateSafety('https://developer.mozilla.org'),
  },
  { url: 'https://react.dev', isEthical: true, botSafety: calculateSafety('https://react.dev') },
  {
    url: 'https://tailwindcss.com',
    isEthical: true,
    botSafety: calculateSafety('https://tailwindcss.com'),
  },
  {
    url: 'https://cloud.google.com',
    isEthical: true,
    botSafety: calculateSafety('https://cloud.google.com'),
  },
  {
    url: 'https://wikipedia.org',
    isEthical: true,
    botSafety: calculateSafety('https://wikipedia.org'),
  },
  { url: 'https://arxiv.org', isEthical: true, botSafety: calculateSafety('https://arxiv.org') },
  { url: 'https://github.com', isEthical: true, botSafety: calculateSafety('https://github.com') },
  { url: 'https://lucide.dev', isEthical: true, botSafety: calculateSafety('https://lucide.dev') },
  { url: 'https://web.dev', isEthical: true, botSafety: calculateSafety('https://web.dev') },
  { url: 'https://svelte.dev', isEthical: true, botSafety: calculateSafety('https://svelte.dev') },
  { url: 'https://nextjs.org', isEthical: true, botSafety: calculateSafety('https://nextjs.org') },
  { url: 'https://python.org', isEthical: true, botSafety: calculateSafety('https://python.org') },
  { url: 'https://npmjs.com', isEthical: true, botSafety: calculateSafety('https://npmjs.com') },
];

export const EthicalMappingService = {
  async performEthicalMap(): Promise<MappingResult[]> {
    // Return registry based on filtering immediately
    return ETHICAL_REGISTRY.filter((item) => item.isEthical && item.botSafety > 0.5);
  },

  async verifyRobotsTxt(url: string): Promise<boolean> {
    try {
      const response = await fetch(`${url}/robots.txt`);
      const text = await response.text();
      return !text.includes('Disallow: /bot') && !text.includes('Disallow: /scraper');
    } catch {
      // Assuming false if we can't reach the robots.txt,
      // or implement server-side proxying to bypass CORS if needed later.
      return false;
    }
  },
};