Neon-tech commited on
Commit
39d275e
·
verified ·
1 Parent(s): dc27ddd

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -0
app.py CHANGED
@@ -201,6 +201,7 @@ def download_loop(state):
201
  text_col = entry["text_col"]
202
  fmt = entry.get("fmt", "parquet")
203
 
 
204
  base_name = url.split("?")[0].split("/")[-1].replace(".parquet", "").replace(".jsonl", "")
205
  shard_name = f"{source}__{base_name}_{lang}.jsonl"
206
  jsonl_path = Path(RAW_DIR) / shard_name
 
201
  text_col = entry["text_col"]
202
  fmt = entry.get("fmt", "parquet")
203
 
204
+ lang = url.split("?")[0].split("/")[-6]
205
  base_name = url.split("?")[0].split("/")[-1].replace(".parquet", "").replace(".jsonl", "")
206
  shard_name = f"{source}__{base_name}_{lang}.jsonl"
207
  jsonl_path = Path(RAW_DIR) / shard_name