davanstrien HF Staff commited on
Commit
afeeaf8
·
verified ·
1 Parent(s): 4a80489

Upload probe.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. probe.py +17 -18
probe.py CHANGED
@@ -2,19 +2,15 @@
2
  # requires-python = ">=3.10"
3
  # dependencies = [
4
  # "torch>=2.1",
5
- # "transformers>=4.40",
6
  # "datasets>=2.18",
7
  # "pycocotools",
8
  # "Pillow",
9
  # "numpy",
10
  # ]
11
  # ///
12
- import os, io, sys
13
- import torch
14
  from PIL import Image
15
  import datasets as hfds
16
- from pycocotools.coco import COCO
17
- from pycocotools.cocoeval import COCOeval
18
 
19
  data_dir = "/data"
20
  if not os.path.isdir(data_dir):
@@ -22,18 +18,21 @@ if not os.path.isdir(data_dir):
22
  ds = hfds.load_dataset(data_dir, split="validation")
23
  examples = list(ds)
24
  print("loaded", len(examples))
25
- print("keys:", list(examples[0].keys()))
26
  ex = examples[0]
27
- print("objects keys:", list(ex["objects"].keys()))
28
- print("cat first val:", ex["objects"]["category_id"][:3] if isinstance(ex["objects"].get("category_id"), list) else ex["objects"]["category_id"])
 
 
 
 
 
29
  img = ex["image"]
30
- print("image type:", type(img))
31
- if isinstance(img, dict):
32
- print("dict keys:", img.keys())
33
- im = Image.open(io.BytesIO(img["bytes"]))
34
- else:
35
- im = img
36
- print("img size:", im.size)
37
- # bbox sample
38
- print("bbox:", ex["objects"]["bbox"][:1])
39
- print("height/width cols:", ex.get("width"), ex.get("height"))
 
2
  # requires-python = ">=3.10"
3
  # dependencies = [
4
  # "torch>=2.1",
 
5
  # "datasets>=2.18",
6
  # "pycocotools",
7
  # "Pillow",
8
  # "numpy",
9
  # ]
10
  # ///
11
+ import os, io
 
12
  from PIL import Image
13
  import datasets as hfds
 
 
14
 
15
  data_dir = "/data"
16
  if not os.path.isdir(data_dir):
 
18
  ds = hfds.load_dataset(data_dir, split="validation")
19
  examples = list(ds)
20
  print("loaded", len(examples))
 
21
  ex = examples[0]
22
+ print("keys:", list(ex.keys()), "w/h:", ex["width"], ex["height"])
23
+ objs = ex["objects"]
24
+ print("objects type:", type(objs), "len:", len(objs))
25
+ o = objs[0]
26
+ print("obj keys:", list(o.keys()))
27
+ print("category_id:", repr(o["category_id"]), "type:", type(o["category_id"]).__name__)
28
+ print("bbox:", o["bbox"], "type:", type(o["bbox"]).__name__)
29
  img = ex["image"]
30
+ print("image type:", type(img).__name__)
31
+ im = img.size if not isinstance(img, dict) else Image.open(io.BytesIO(img["bytes"])).size
32
+ print("img size:", im)
33
+ from collections import Counter
34
+ cats = Counter()
35
+ for e in examples[:200]:
36
+ for o in e["objects"]:
37
+ cats[repr(o["category_id"])] += 1
38
+ print("category_id value distribution sample:", dict(cats))