Neon-AI commited on
Commit
7e80c08
·
verified ·
1 Parent(s): cddfa58

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +25 -15
app.py CHANGED
@@ -1,18 +1,28 @@
 
1
  import gzip
2
  import json
3
 
4
- with gzip.open("/data/cdx-00000.gz", "rb") as gz:
5
- found = 0
6
- for line in gz:
7
- if found >= 2:
8
- break
9
- try:
10
- parts = line.decode("utf-8", errors="ignore").strip().split(" ", 2)
11
- if len(parts) < 3:
12
- continue
13
- meta = json.loads(parts[2])
14
- if meta.get("status") == "200" and meta.get("mime") == "text/html":
15
- print(meta["url"])
16
- found += 1
17
- except Exception:
18
- continue
 
 
 
 
 
 
 
 
 
 
1
+ import httpx
2
  import gzip
3
  import json
4
 
5
+ url = "https://data.commoncrawl.org/cc-index/collections/CC-MAIN-2026-08/indexes/cdx-00000.gz"
6
+
7
+ print("Downloading...")
8
+ with httpx.stream("GET", url) as r:
9
+ with open("/data/cdx-2026.gz", "wb") as f:
10
+ for chunk in r.iter_bytes():
11
+ f.write(chunk)
12
+
13
+ print("Extracting 2 URLs...")
14
+ with gzip.open("/data/cdx-2026.gz", "rb") as gz:
15
+ found = 0
16
+ for line in gz:
17
+ if found >= 2:
18
+ break
19
+ try:
20
+ parts = line.decode("utf-8", errors="ignore").strip().split(" ", 2)
21
+ if len(parts) < 3:
22
+ continue
23
+ meta = json.loads(parts[2])
24
+ if meta.get("status") == "200" and meta.get("mime") == "text/html":
25
+ print(meta["url"])
26
+ found += 1
27
+ except Exception:
28
+ continue