kovinape commited on
Commit
c2bcbcf
·
verified ·
1 Parent(s): bdddf73

Upload app.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. app.py +4 -4
app.py CHANGED
@@ -660,7 +660,7 @@ def _hash_line(line: str) -> int:
660
 
661
  def list_bucket_files():
662
  from datetime import datetime, timezone
663
- infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True)
664
  out=[]
665
  for i in infos:
666
  if not isinstance(i, dict): continue
@@ -692,7 +692,7 @@ def do_extract(job_id, targets, selected=None):
692
  # list shards
693
  if selected:
694
  sel=set(selected)
695
- infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True)
696
  all_shards=[]
697
  for i in infos:
698
  if not isinstance(i, dict): continue
@@ -702,12 +702,12 @@ def do_extract(job_id, targets, selected=None):
702
  if base in sel:
703
  all_shards.append(nm)
704
  else:
705
- paths = hffs.ls(f"buckets/{BUCKET_ID}", detail=False)
706
  all_shards = [p for p in paths if p.lower().endswith(".txt")]
707
  shards = all_shards
708
  per_gb=dict()
709
  try:
710
- for i in hffs.ls(f"buckets/{BUCKET_ID}", detail=True):
711
  if not isinstance(i, dict): continue
712
  nm=str(i.get("name",""))
713
  if not nm.lower().endswith(".txt"): continue
 
660
 
661
  def list_bucket_files():
662
  from datetime import datetime, timezone
663
+ infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True)
664
  out=[]
665
  for i in infos:
666
  if not isinstance(i, dict): continue
 
692
  # list shards
693
  if selected:
694
  sel=set(selected)
695
+ infos = hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True)
696
  all_shards=[]
697
  for i in infos:
698
  if not isinstance(i, dict): continue
 
702
  if base in sel:
703
  all_shards.append(nm)
704
  else:
705
+ paths = hffs.ls(f"buckets/{BUCKET_ID}", detail=False, refresh=True)
706
  all_shards = [p for p in paths if p.lower().endswith(".txt")]
707
  shards = all_shards
708
  per_gb=dict()
709
  try:
710
+ for i in hffs.ls(f"buckets/{BUCKET_ID}", detail=True, refresh=True):
711
  if not isinstance(i, dict): continue
712
  nm=str(i.get("name",""))
713
  if not nm.lower().endswith(".txt"): continue