Update app.py
Browse files
app.py
CHANGED
|
@@ -5,13 +5,17 @@ url = "https://data.commoncrawl.org/cc-index/collections/CC-MAIN-2008-2009/index
|
|
| 5 |
|
| 6 |
print("Downloading...")
|
| 7 |
with httpx.stream("GET", url) as r:
|
| 8 |
-
|
| 9 |
-
|
| 10 |
-
|
| 11 |
|
| 12 |
-
print("Unzipping
|
| 13 |
with gzip.open("/data/cdx-00000.gz", "rb") as gz:
|
| 14 |
-
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
|
| 6 |
print("Downloading...")
|
| 7 |
with httpx.stream("GET", url) as r:
|
| 8 |
+
with open("/data/cdx-00000.gz", "wb") as f:
|
| 9 |
+
for chunk in r.iter_bytes():
|
| 10 |
+
f.write(chunk)
|
| 11 |
|
| 12 |
+
print("Unzipping...")
|
| 13 |
with gzip.open("/data/cdx-00000.gz", "rb") as gz:
|
| 14 |
+
content = gz.read()
|
| 15 |
+
with open("/data/cdx-00000.txt", "wb") as f:
|
| 16 |
+
f.write(content)
|
| 17 |
+
|
| 18 |
+
print("Done. First 20 lines:")
|
| 19 |
+
lines = content.decode("utf-8", errors="ignore").splitlines()
|
| 20 |
+
for line in lines[:20]:
|
| 21 |
+
print(line)
|