Neon-AI commited on
Commit
6f6d092
·
verified ·
1 Parent(s): 3641ca9

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +12 -8
app.py CHANGED
@@ -5,13 +5,17 @@ url = "https://data.commoncrawl.org/cc-index/collections/CC-MAIN-2008-2009/index
5
 
6
  print("Downloading...")
7
  with httpx.stream("GET", url) as r:
8
- with open("/data/cdx-00000.gz", "wb") as f:
9
- for chunk in r.iter_bytes():
10
- f.write(chunk)
11
 
12
- print("Unzipping first 20 lines...")
13
  with gzip.open("/data/cdx-00000.gz", "rb") as gz:
14
- for i, line in enumerate(gz):
15
- if i >= 20:
16
- break
17
- print(line.decode("utf-8", errors="ignore"))
 
 
 
 
 
5
 
6
  print("Downloading...")
7
  with httpx.stream("GET", url) as r:
8
+ with open("/data/cdx-00000.gz", "wb") as f:
9
+ for chunk in r.iter_bytes():
10
+ f.write(chunk)
11
 
12
+ print("Unzipping...")
13
  with gzip.open("/data/cdx-00000.gz", "rb") as gz:
14
+ content = gz.read()
15
+ with open("/data/cdx-00000.txt", "wb") as f:
16
+ f.write(content)
17
+
18
+ print("Done. First 20 lines:")
19
+ lines = content.decode("utf-8", errors="ignore").splitlines()
20
+ for line in lines[:20]:
21
+ print(line)