Spaces:
Paused
Paused
Upload app.py with huggingface_hub
Browse files
app.py
CHANGED
|
@@ -660,7 +660,7 @@ def _hash_line(line: str) -> int:
|
|
| 660 |
|
| 661 |
def list_bucket_files():
|
| 662 |
from datetime import datetime, timezone
|
| 663 |
-
infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True)
|
| 664 |
out=[]
|
| 665 |
for i in infos:
|
| 666 |
if not isinstance(i, dict): continue
|
|
@@ -692,7 +692,7 @@ def do_extract(job_id, targets, selected=None):
|
|
| 692 |
# list shards
|
| 693 |
if selected:
|
| 694 |
sel=set(selected)
|
| 695 |
-
infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True)
|
| 696 |
all_shards=[]
|
| 697 |
for i in infos:
|
| 698 |
if not isinstance(i, dict): continue
|
|
@@ -702,12 +702,12 @@ def do_extract(job_id, targets, selected=None):
|
|
| 702 |
if base in sel:
|
| 703 |
all_shards.append(nm)
|
| 704 |
else:
|
| 705 |
-
paths = hffs.ls(f"buckets/{BUCKET_ID}", detail=False)
|
| 706 |
all_shards = [p for p in paths if p.lower().endswith(".txt")]
|
| 707 |
shards = all_shards
|
| 708 |
per_gb=dict()
|
| 709 |
try:
|
| 710 |
-
for i in hffs.ls(f"buckets/{BUCKET_ID}", detail=True):
|
| 711 |
if not isinstance(i, dict): continue
|
| 712 |
nm=str(i.get("name",""))
|
| 713 |
if not nm.lower().endswith(".txt"): continue
|
|
|
|
| 660 |
|
| 661 |
def list_bucket_files():
|
| 662 |
from datetime import datetime, timezone
|
| 663 |
+
infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True)
|
| 664 |
out=[]
|
| 665 |
for i in infos:
|
| 666 |
if not isinstance(i, dict): continue
|
|
|
|
| 692 |
# list shards
|
| 693 |
if selected:
|
| 694 |
sel=set(selected)
|
| 695 |
+
infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True)
|
| 696 |
all_shards=[]
|
| 697 |
for i in infos:
|
| 698 |
if not isinstance(i, dict): continue
|
|
|
|
| 702 |
if base in sel:
|
| 703 |
all_shards.append(nm)
|
| 704 |
else:
|
| 705 |
+
paths = hffs.ls(f"buckets/{BUCKET_ID}", detail=False, refresh=True)
|
| 706 |
all_shards = [p for p in paths if p.lower().endswith(".txt")]
|
| 707 |
shards = all_shards
|
| 708 |
per_gb=dict()
|
| 709 |
try:
|
| 710 |
+
for i in hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True):
|
| 711 |
if not isinstance(i, dict): continue
|
| 712 |
nm=str(i.get("name",""))
|
| 713 |
if not nm.lower().endswith(".txt"): continue
|