Muttered3 commited on
Commit
6cbf539
·
verified ·
1 Parent(s): 916d293

Create wordlist.py

Browse files
Files changed (1) hide show
  1. wordlist.py +38 -0
wordlist.py ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import asyncio
2
+ import urllib.request
3
+ import urllib.error
4
+ import config
5
+
6
+ async def fetch_word_list() -> list[str]:
7
+ """
8
+ Downloads config.WORDLIST_URL (tab-separated word<TAB>freq file).
9
+ Filters: length MIN_WORD_LEN..MAX_WORD_LEN, strictly alpha only.
10
+ Sorts by frequency descending.
11
+ Uses loop.run_in_executor so it does not block the event loop.
12
+ """
13
+ def _download():
14
+ req = urllib.request.Request(config.WORDLIST_URL, headers={'User-Agent': 'Mozilla/5.0'})
15
+ try:
16
+ with urllib.request.urlopen(req) as response:
17
+ return response.read().decode('utf-8').splitlines()
18
+ except urllib.error.URLError as e:
19
+ print(f"Failed to fetch word list: {e}")
20
+ return []
21
+
22
+ loop = asyncio.get_event_loop()
23
+ lines = await loop.run_in_executor(None, _download)
24
+
25
+ data = []
26
+ for line in lines:
27
+ parts = line.split('\t')
28
+ if len(parts) == 2:
29
+ word = parts[0].strip()
30
+ if config.MIN_WORD_LEN <= len(word) <= config.MAX_WORD_LEN and word.isalpha():
31
+ try:
32
+ count = int(parts[1].strip())
33
+ data.append((word, count))
34
+ except ValueError:
35
+ continue
36
+
37
+ data.sort(key=lambda x: x[1], reverse=True)
38
+ return [word for word, count in data]