Tonic commited on
Commit
245bf01
·
verified ·
1 Parent(s): d21b333

Upload export_job.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. export_job.py +69 -69
export_job.py CHANGED
@@ -1,69 +1,69 @@
1
- # /// script
2
- # requires-python = ">=3.11"
3
- # dependencies = ["torch", "torchvision", "pillow", "numpy", "ai-edge-torch", "huggingface_hub>=0.30", "timm"]
4
- # ///
5
- """HF-job credibility gate: convert the wave-3 winner checkpoint to int8 TFLite (224x224)
6
- and upload the artifact + report to the code repo."""
7
- import os, subprocess, sys
8
- from huggingface_hub import HfApi, hf_hub_download
9
-
10
- tok = os.environ["HF_TOKEN"]
11
- api = HfApi(token=tok)
12
- ckpt = hf_hub_download("Tonic/challenge5-acoustic-squeezenet1_1-s7", "best.pt",
13
- repo_type="model", token=tok)
14
- print("ckpt:", ckpt, os.path.getsize(ckpt) // 1024, "KB", flush=True)
15
-
16
- here = os.path.dirname(os.path.abspath(__file__))
17
- # calibration spectrograms from the challenge pack are not in the job; synthesize rep data
18
- # from the unified dataset windows instead (same rendering):
19
- from datasets import load_dataset # noqa
20
- sys.path.insert(0, here)
21
- # write a tiny driver that reuses esp32_demo/tools/export_tflite.py logic but with
22
- # synthetic calibration from unified windows
23
- import numpy as np, torch
24
- import ai_edge_torch
25
- from PIL import Image
26
- import io
27
- from datasets import load_dataset
28
-
29
- ck = torch.load(ckpt, map_location="cpu", weights_only=False)
30
- sd = ck.get("state_dict", ck) if isinstance(ck, dict) else ck
31
- arch = ck.get("arch", "squeezenet1_1") if isinstance(ck, dict) else "squeezenet1_1"
32
-
33
- import importlib.util
34
- spec = importlib.util.spec_from_file_location("job_train", os.path.join(here, "job_train.py"))
35
- jt = importlib.util.module_from_spec(spec)
36
- spec.loader.exec_module(jt)
37
- model = jt.build_model(arch, pretrained=0)
38
- model.load_state_dict(sd, strict=False)
39
- model.eval()
40
- print("arch:", arch, "params:", sum(p.numel() for p in model.parameters()), flush=True)
41
-
42
- ds = load_dataset("Tonic/unified-drone-acoustic-cuas", "windows_5s", split="train",
43
- streaming=True, token=tok)
44
- def rep_data():
45
- n = 0
46
- for r in ds:
47
- im = Image.open(io.BytesIO(r["spectrogram"]["bytes"])).convert("RGB").resize((224, 224))
48
- x = np.asarray(im, np.float32) / 255.0
49
- x = (x - np.array([0.485, 0.456, 0.406], np.float32)) / np.array([0.229, 0.224, 0.225], np.float32)
50
- yield [np.transpose(x, (2, 0, 1))[None].astype(np.float32)]
51
- n += 1
52
- if n >= 256:
53
- break
54
-
55
- dummy = torch.zeros(1, 3, 224, 224)
56
- conv = ai_edge_torch.converters.TorchModuleToTfliteConverter(model, (dummy,))
57
- try:
58
- tfl = conv.convert_with_full_integer_quantization(rep_data)
59
- mode = "full-int8"
60
- except AttributeError:
61
- tfl = conv.convert_with_dynamic_range_quantization()
62
- mode = "dynamic-range"
63
- out = f"/tmp/model_full_224_{mode.replace('-', '_')}.tflite"
64
- open(out, "wb").write(tfl)
65
- print("mode:", mode, "size:", os.path.getsize(out) // 1024, "KB", flush=True)
66
- api.upload_file(path_or_fileobj=out, path_in_repo=f"esp32/model_full_224_{mode.replace('-', '_')}.tflite",
67
- repo_id="Tonic/challenge5-acoustic-code", repo_type="model", token=tok,
68
- commit_message=f"esp32: int8 tflite export of squeezenet1_1-s7 ({mode}, {os.path.getsize(out)//1024} KB)")
69
- print("UPLOADED esp32/ model artifact", flush=True)
 
1
+ # /// script
2
+ # requires-python = ">=3.11"
3
+ # dependencies = ["torch", "torchvision", "pillow", "numpy", "ai-edge-torch", "huggingface_hub>=0.30", "timm", "datasets>=4.0"]
4
+ # ///
5
+ """HF-job credibility gate: convert the wave-3 winner checkpoint to int8 TFLite (224x224)
6
+ and upload the artifact + report to the code repo."""
7
+ import os, subprocess, sys
8
+ from huggingface_hub import HfApi, hf_hub_download
9
+
10
+ tok = os.environ["HF_TOKEN"]
11
+ api = HfApi(token=tok)
12
+ ckpt = hf_hub_download("Tonic/challenge5-acoustic-squeezenet1_1-s7", "best.pt",
13
+ repo_type="model", token=tok)
14
+ print("ckpt:", ckpt, os.path.getsize(ckpt) // 1024, "KB", flush=True)
15
+
16
+ here = os.path.dirname(os.path.abspath(__file__))
17
+ # calibration spectrograms from the challenge pack are not in the job; synthesize rep data
18
+ # from the unified dataset windows instead (same rendering):
19
+ from datasets import load_dataset # noqa
20
+ sys.path.insert(0, here)
21
+ # write a tiny driver that reuses esp32_demo/tools/export_tflite.py logic but with
22
+ # synthetic calibration from unified windows
23
+ import numpy as np, torch
24
+ import ai_edge_torch
25
+ from PIL import Image
26
+ import io
27
+ from datasets import load_dataset
28
+
29
+ ck = torch.load(ckpt, map_location="cpu", weights_only=False)
30
+ sd = ck.get("state_dict", ck) if isinstance(ck, dict) else ck
31
+ arch = ck.get("arch", "squeezenet1_1") if isinstance(ck, dict) else "squeezenet1_1"
32
+
33
+ import importlib.util
34
+ spec = importlib.util.spec_from_file_location("job_train", os.path.join(here, "job_train.py"))
35
+ jt = importlib.util.module_from_spec(spec)
36
+ spec.loader.exec_module(jt)
37
+ model = jt.build_model(arch, pretrained=0)
38
+ model.load_state_dict(sd, strict=False)
39
+ model.eval()
40
+ print("arch:", arch, "params:", sum(p.numel() for p in model.parameters()), flush=True)
41
+
42
+ ds = load_dataset("Tonic/unified-drone-acoustic-cuas", "windows_5s", split="train",
43
+ streaming=True, token=tok)
44
+ def rep_data():
45
+ n = 0
46
+ for r in ds:
47
+ im = Image.open(io.BytesIO(r["spectrogram"]["bytes"])).convert("RGB").resize((224, 224))
48
+ x = np.asarray(im, np.float32) / 255.0
49
+ x = (x - np.array([0.485, 0.456, 0.406], np.float32)) / np.array([0.229, 0.224, 0.225], np.float32)
50
+ yield [np.transpose(x, (2, 0, 1))[None].astype(np.float32)]
51
+ n += 1
52
+ if n >= 256:
53
+ break
54
+
55
+ dummy = torch.zeros(1, 3, 224, 224)
56
+ conv = ai_edge_torch.converters.TorchModuleToTfliteConverter(model, (dummy,))
57
+ try:
58
+ tfl = conv.convert_with_full_integer_quantization(rep_data)
59
+ mode = "full-int8"
60
+ except AttributeError:
61
+ tfl = conv.convert_with_dynamic_range_quantization()
62
+ mode = "dynamic-range"
63
+ out = f"/tmp/model_full_224_{mode.replace('-', '_')}.tflite"
64
+ open(out, "wb").write(tfl)
65
+ print("mode:", mode, "size:", os.path.getsize(out) // 1024, "KB", flush=True)
66
+ api.upload_file(path_or_fileobj=out, path_in_repo=f"esp32/model_full_224_{mode.replace('-', '_')}.tflite",
67
+ repo_id="Tonic/challenge5-acoustic-code", repo_type="model", token=tok,
68
+ commit_message=f"esp32: int8 tflite export of squeezenet1_1-s7 ({mode}, {os.path.getsize(out)//1024} KB)")
69
+ print("UPLOADED esp32/ model artifact", flush=True)