Upload folder using huggingface_hub

.gitattributes

@@ -45,3 +45,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 2026.10.04/17.15.34_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261004_171543-pnv1tivo/run-pnv1tivo.wandb filter=lfs diff=lfs merge=lfs -text
 2026.10.05/14.05.16_train_diffusion_unet_lowdim_FrankaKitchen-v1/logs.json.txt filter=lfs diff=lfs merge=lfs -text
 2026.10.05/14.05.16_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_140524-0iejtsht/run-0iejtsht.wandb filter=lfs diff=lfs merge=lfs -text
+2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/logs.json.txt filter=lfs diff=lfs merge=lfs -text
+2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/run-a0y1jzsf.wandb filter=lfs diff=lfs merge=lfs -text

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/.hydra/config.yaml

@@ -0,0 +1,161 @@
+name: train_diffusion_unet_lowdim
+_target_: llmbc.workspace.train_diffusion_unet_lowdim_workspace.TrainDiffusionUnetLowdimWorkspace
+obs_dim: ${task.obs_dim}
+action_dim: ${task.action_dim}
+task_name: ${task.name}
+exp_name: default
+horizon: 16
+n_obs_steps: 2
+n_action_steps: 8
+n_latency_steps: 0
+past_action_visible: false
+keypoint_visible_rate: 1.0
+obs_as_local_cond: false
+obs_as_global_cond: true
+pred_action_steps_only: false
+policy:
+  _target_: llmbc.policy.diffusion_unet_lowdim_policy.DiffusionUnetLowdimPolicy
+  model:
+    _target_: llmbc.model.diffusion.conditional_unet1d.ConditionalUnet1D
+    input_dim: '${eval: ${task.action_dim} if ${obs_as_local_cond} or ${obs_as_global_cond}
+      else ${task.obs_dim} + ${task.action_dim}}'
+    local_cond_dim: '${eval: ${task.obs_dim} if ${obs_as_local_cond} else None}'
+    global_cond_dim: '${eval: ${task.obs_dim}*${n_obs_steps} if ${obs_as_global_cond}
+      else None}'
+    diffusion_step_embed_dim: 256
+    down_dims:
+    - 256
+    - 512
+    - 1024
+    kernel_size: 5
+    upsample_kernel_size: 4
+    n_groups: 8
+    cond_predict_scale: true
+  noise_scheduler:
+    _target_: diffusers.schedulers.scheduling_ddpm.DDPMScheduler
+    num_train_timesteps: 100
+    beta_start: 0.0001
+    beta_end: 0.02
+    beta_schedule: squaredcos_cap_v2
+    variance_type: fixed_small
+    clip_sample: true
+    prediction_type: epsilon
+  horizon: ${horizon}
+  obs_dim: ${obs_dim}
+  action_dim: ${action_dim}
+  n_action_steps: ${eval:'${n_action_steps}+${n_latency_steps}'}
+  n_obs_steps: ${n_obs_steps}
+  num_inference_steps: 10
+  obs_as_local_cond: ${obs_as_local_cond}
+  obs_as_global_cond: ${obs_as_global_cond}
+  pred_action_steps_only: ${pred_action_steps_only}
+  oa_step_convention: true
+ema:
+  _target_: llmbc.model.diffusion.ema_model.EMAModel
+  update_after_step: 0
+  inv_gamma: 1.0
+  power: 0.75
+  min_value: 0.0
+  max_value: 0.9999
+dataloader:
+  batch_size: 256
+  num_workers: 0
+  shuffle: true
+  pin_memory: false
+  persistent_workers: false
+val_dataloader:
+  batch_size: 256
+  num_workers: 0
+  shuffle: false
+  pin_memory: false
+  persistent_workers: false
+optimizer:
+  _target_: torch.optim.AdamW
+  lr: 0.0001
+  betas:
+  - 0.95
+  - 0.999
+  eps: 1.0e-08
+  weight_decay: 1.0e-06
+training:
+  device: cuda:0
+  seed: 45
+  debug: false
+  resume: true
+  lr_scheduler: cosine
+  lr_warmup_steps: 500
+  num_epochs: 5001
+  gradient_accumulate_every: 1
+  use_ema: true
+  rollout_every: 50
+  checkpoint_every: 50
+  val_every: 1
+  sample_every: 50
+  max_train_steps: null
+  max_val_steps: null
+  tqdm_interval_sec: 1.0
+logging:
+  project: ${task.name}-training
+  resume: true
+  mode: online
+  name: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+  tags:
+  - ${name}
+  - ${task_name}
+  - ${exp_name}
+  id: null
+  group: null
+checkpoint:
+  topk:
+    monitor_key: test_success_rate
+    mode: max
+    k: 5
+    format_str: epoch={epoch:04d}-test_success_rate={test_success_rate:.3f}.ckpt
+  save_last_ckpt: true
+  save_last_snapshot: false
+multi_run:
+  run_dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  wandb_name_base: ${now:%Y.%m.%d-%H.%M.%S}_${name}_${task_name}
+task:
+  name: FrankaKitchen-v1
+  obs_dim: 59
+  action_dim: 9
+  lang_multistep_merger:
+    _target_: llfbench.envs.kitchen.multistep_merger.franka_kitchen_merger.KitchenMultistepMerger
+  env_runner:
+    _target_: llmbc.env_runner.kitchen_lowdim_runner.KitchenLowdimRunner
+    env_name: llf-kitchen-FrankaKitchen-v1
+    n_train: 10
+    n_test: 50
+    n_envs: 10
+    max_steps: 500
+    n_obs_steps: ${n_obs_steps}
+    n_action_steps: ${n_action_steps}
+    instruction_type: b
+    feedback_type:
+    - hp
+    - hn
+    - fp
+    tasks_to_complete:
+    - microwave
+    - kettle
+    - light switch
+    - slide cabinet
+    visual: false
+    discount: 0.99
+    inference_expert_action: false
+    lang_multistep_merger:
+      _target_: llfbench.envs.kitchen.multistep_merger.franka_kitchen_merger.KitchenMultistepMerger
+  dataset:
+    _target_: llmbc.dataset.kitchen_lowdim_dataset.KitchenLowdimDataset
+    data_path: datasets/FrankaKitchen-v1.pt
+    data_path2: datasets/FrankaKitchen-v1.pt
+    horizon: ${horizon}
+    pad_before: ${eval:'${n_obs_steps}-1'}
+    pad_after: ${eval:'${n_action_steps}-1'}
+    obs_eef_target: true
+    use_manual_normalizer: false
+    val_ratio: 0.05
+    dummy_normalizer: false
+  instructor:
+    _target_: llmbc.translator.instructor.kitchen_instructor.FrankaKitchen_v1_instructor.FrankaKitchenV1Instructor

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/.hydra/hydra.yaml

@@ -0,0 +1,155 @@
+hydra:
+  run:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+  sweep:
+    dir: data/outputs/${now:%Y.%m.%d}/${now:%H.%M.%S}_${name}_${task_name}
+    subdir: ${hydra.job.num}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - training.seed=45
+  job:
+    name: train
+    chdir: null
+    override_dirname: training.seed=45
+    id: ???
+    num: ???
+    config_name: dp_FrankaKitchen-v1.yaml
+    env_set: {}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.2.0
+    version_base: '1.2'
+    cwd: /tmp2/chyang/workspace/LLM-BC
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /tmp2/chyang/workspace/LLM-BC/config/main_table
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1
+    choices:
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/.hydra/overrides.yaml

@@ -0,0 +1 @@
+- training.seed=45

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/checkpoints/epoch=2600-test_success_rate=0.620.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e7d09f74f7bc8c9e44e90fa2ebc49e74c4e81f2ffa06a57dab6ee6cb302907f
+size 1070882738

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/checkpoints/epoch=2950-test_success_rate=0.620.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee01864556d3bb6bd4972d0f9068bb9bc2878aa766a67e188574c8a6c92d67f0
+size 1070882738

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/checkpoints/epoch=3300-test_success_rate=0.620.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a0947bdd9b9b7bb8aaadac2b97de5e1b280f3967b23348f9b9d9c8f4d0d928a
+size 1070882738

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/checkpoints/epoch=3550-test_success_rate=0.600.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59938386a3b42ad7ec9599787d983828a1f9c7167b498d8be1dd1ea7059b9e19
+size 1070882738

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/checkpoints/epoch=3650-test_success_rate=0.600.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:923324641399655346ba3234eedb67111d614c9121a9b5f111354a7a8d99744e
+size 1070882738

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/checkpoints/latest.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:632a2f7de62d98ea5d73cecf4b00991223f46becafc40b08fe8e26a9e096477f
+size 1070882738

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/logs.json.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:31f8f843a5fc1fb35478b8b06ad2bba9cd4924c22ee879c5b022dc6662d3bb2c
+size 261639513

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/train.log

@@ -0,0 +1,8 @@
+[2026-10-05 13:45:51,044][numexpr.utils][INFO] - Note: NumExpr detected 40 cores but "NUMEXPR_MAX_THREADS" not set, so enforcing safe limit of 16.
+[2026-10-05 13:45:51,045][numexpr.utils][INFO] - NumExpr defaulting to 16 threads.
+[2026-10-05 13:45:53,584][llmbc.model.diffusion.conditional_unet1d][INFO] - number of parameters: 6.691406e+07
+[2026-10-05 13:45:58,157][absl][INFO] - MUJOCO_GL=osmesa, attempting to import specified OpenGL backend.
+[2026-10-05 13:45:58,161][absl][INFO] - MuJoCo library version is: 2.3.7
+[2026-10-05 13:45:58,232][datasets][INFO] - PyTorch version 2.2.2 available.
+[2026-10-05 13:45:58,232][datasets][INFO] - TensorFlow version 2.15.1 available.
+[2026-10-05 13:45:58,233][datasets][INFO] - JAX version 0.4.30 available.

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/debug-internal.log

@@ -0,0 +1,162 @@
+{"time":"2026-10-05T13:45:59.163355915+08:00","level":"INFO","msg":"using version","core version":"0.18.6"}
+{"time":"2026-10-05T13:45:59.163364924+08:00","level":"INFO","msg":"created symlink","path":"/tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/logs/debug-core.log"}
+{"time":"2026-10-05T13:45:59.269047163+08:00","level":"INFO","msg":"created new stream","id":"a0y1jzsf"}
+{"time":"2026-10-05T13:45:59.269111886+08:00","level":"INFO","msg":"stream: started","id":"a0y1jzsf"}
+{"time":"2026-10-05T13:45:59.269178689+08:00","level":"INFO","msg":"handler: started","stream_id":{"value":"a0y1jzsf"}}
+{"time":"2026-10-05T13:45:59.269379864+08:00","level":"INFO","msg":"sender: started","stream_id":"a0y1jzsf"}
+{"time":"2026-10-05T13:45:59.269168469+08:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"a0y1jzsf"}}
+{"time":"2026-10-05T13:46:00.434351743+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2026-10-06T00:55:17.233211224+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 192.168.11.41:58154->34.8.250.101:443: read: connection reset by peer"}
+{"time":"2026-10-06T09:25:04.216906944+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:25:15.252828409+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp 34.8.250.101:443: connect: no route to host"}
+{"time":"2026-10-06T09:25:26.397707765+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:39419->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:25:36.667141225+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:25:36.756524289+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:25:54.045717582+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:60142->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:26:10.833705525+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:26:19.764617144+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:26:50.502002568+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:26:57.698292324+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:27:39.461033137+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:28:07.699959385+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:50452->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:28:43.153942633+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:29:17.703107112+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:36665->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:30:13.15448044+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:30:27.705770185+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:42721->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:31:37.708864611+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:43878->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:31:43.15597339+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:32:47.710960866+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:46731->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:33:13.157227823+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:33:57.71429012+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:38521->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:34:34.218070669+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000912319,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"8lx8hbxpiv60\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:34:35.701292599+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000212389,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:34:40.436056895+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000583931,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:40.436161283+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000713944,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:40.438335724+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000227179,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:40.439452019+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000451924,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:40.4395116+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000070491,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:43.158026918+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:35:07.716961743+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:57595->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:36:13.159016826+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:36:17.72009588+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:37109->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:37:27.722522334+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:34984->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:37:43.161342328+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:38:37.726080975+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:57156->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:39:13.162840088+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:39:47.728677821+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59728->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:40:43.164031242+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:40:57.730952264+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:42114->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:41:53.16702903+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:54374->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:42:07.73436387+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:50668->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:43:03.16922781+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:54076->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:43:17.736868037+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:55535->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:44:13.171973988+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:36098->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:44:27.739336521+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:54781->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:44:34.239471319+08:00","level":"WARN","msg":"sender: taking a long time","seconds":1200.022912721,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"8lx8hbxpiv60\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:44:35.70342383+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.002371121,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:44:40.436779533+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.001288564,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:44:40.436895391+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.001441669,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:44:40.439087859+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.00097571,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:44:40.440188055+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.000730352,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:44:40.440189578+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.001201081,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:45:23.174036006+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:32808->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:45:37.74262234+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:57118->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:46:33.175647107+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:34142->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:46:47.744827673+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:38788->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:47:43.178129417+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:51485->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:47:57.747348679+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:41227->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:48:53.180282859+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:58661->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:48:53.180492936+08:00","level":"ERROR","msg":"sender: sendStopStatus: failed to get run stopped status: api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s): Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:58661->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:48:53.182181928+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1458.965629728,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"8lx8hbxpiv60\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:48:53.182330263+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1457.481319991,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:48:53.182428528+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.746976632,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:48:53.182475062+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.744278468,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:48:53.182446719+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.742950553,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:48:53.182566064+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.743523327,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:48:53.182440969+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.746942282,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:49:03.31098711+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:52560->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:49:03.311014342+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:52560->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:49:15.51000472+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:46088->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:49:30.220917721+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:43837->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:49:49.577054247+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:51242->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:50:08.787118718+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:50:08.7871671+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:50:57.235008055+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59888->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:51:18.790095596+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:45443->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:52:07.238034763+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:60728->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:52:28.793223924+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:54344->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:53:17.241303288+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59484->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:53:38.795011432+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:50171->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:54:27.244118673+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59227->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:54:48.797972117+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:45997->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:55:37.247023164+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:34908->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:55:58.800364514+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:49683->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:56:47.249888902+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:60992->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:57:08.802311113+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:53471->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:57:57.252954476+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:40719->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:58:17.085117628+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59879->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:58:23.228954725+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:48731->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:58:53.310509872+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000650079,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"gplclwr5yrv3\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:58:54.728910487+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000849739,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:59:00.435880525+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.00045047,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:00.43601939+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000525846,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:00.438191073+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.00039204,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:00.438198711+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000120136,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:00.439335822+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000417846,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:05.085657873+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:46257->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:59:32.673067475+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:38517->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:00:15.088446059+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:42397->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:00:42.675974984+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:33976->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:01:25.090369352+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:41435->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:01:52.679257229+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:37099->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:02:35.093533054+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:43216->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:03:02.682033223+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:58548->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:03:45.095777469+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:50451->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:04:12.685307219+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:56278->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:04:55.099256898+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:48863->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:05:22.688113437+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:49585->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:06:05.101706688+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:40857->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:06:30.469216735+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T10:07:05.891669426+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1092.581635314,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"gplclwr5yrv3\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:07:05.891847858+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1091.163881748,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:07:05.89194221+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.456543256,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:07:05.892005599+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.456550335,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:07:05.892078977+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.454229769,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:07:05.892165516+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.454052586,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:07:05.892249043+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.45331293,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:08:20.896007012+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:08:52.988732973+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:09:27.388072917+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:10:07.176838085+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:10:54.373793246+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:12:02.60384915+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:13:32.605053705+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:15:02.606602306+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:16:32.608834641+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:17:50.89575474+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000208414,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"ckfgj8ye2snm\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:17:52.434840284+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000559945,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:18:00.43574907+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000133108,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:00.435792208+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000401175,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:00.438949663+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000530324,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:00.438965768+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000356572,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:00.439011938+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.00004734,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:02.610833003+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:19:32.611778582+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:21:02.61382365+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:22:32.615544993+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:23:42.963766085+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 192.168.11.41:41968->34.8.250.101:443: read: network is unreachable"}
+{"time":"2026-10-06T10:24:43.875271386+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1012.979744627,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"ckfgj8ye2snm\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:24:43.875689942+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1011.441443372,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:24:43.875721175+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.440111468,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:24:43.875814017+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.437401986,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:24:43.875746082+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.440385756,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:24:43.875852672+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.436850157,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:24:43.876036242+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.437415844,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T13:19:59.039120309+08:00","level":"INFO","msg":"stream: closing","id":"a0y1jzsf"}
+{"time":"2026-10-06T13:19:59.039154305+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2026-10-06T13:19:59.111492217+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2026-10-06T13:20:00.866016279+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-10-06T13:20:01.218067036+08:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"a0y1jzsf"}}
+{"time":"2026-10-06T13:20:01.21814782+08:00","level":"INFO","msg":"sender: closed","stream_id":"a0y1jzsf"}
+{"time":"2026-10-06T13:20:01.218137984+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"a0y1jzsf"}}
+{"time":"2026-10-06T13:20:01.218298956+08:00","level":"INFO","msg":"stream: closed","id":"a0y1jzsf"}

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/debug.log

@@ -0,0 +1,27 @@
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Current SDK version is 0.18.6
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Configure stats pid to 1342604
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Loading settings from /home/chyang/.config/wandb/settings
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Loading settings from /tmp2/chyang/workspace/LLM-BC/wandb/settings
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program_abspath': '/tmp2/chyang/workspace/LLM-BC/train.py', 'program': '/tmp2/chyang/workspace/LLM-BC/./train.py'}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Applying login settings: {}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:_log_setup():533] Logging user logs to /tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/logs/debug.log
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:_log_setup():534] Logging internal logs to /tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/logs/debug-internal.log
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:init():619] calling init triggers
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {'name': 'train_diffusion_unet_lowdim', '_target_': 'llmbc.workspace.train_diffusion_unet_lowdim_workspace.TrainDiffusionUnetLowdimWorkspace', 'obs_dim': 59, 'action_dim': 9, 'task_name': 'FrankaKitchen-v1', 'exp_name': 'default', 'horizon': 16, 'n_obs_steps': 2, 'n_action_steps': 8, 'n_latency_steps': 0, 'past_action_visible': False, 'keypoint_visible_rate': 1.0, 'obs_as_local_cond': False, 'obs_as_global_cond': True, 'pred_action_steps_only': False, 'policy': {'_target_': 'llmbc.policy.diffusion_unet_lowdim_policy.DiffusionUnetLowdimPolicy', 'model': {'_target_': 'llmbc.model.diffusion.conditional_unet1d.ConditionalUnet1D', 'input_dim': 9, 'local_cond_dim': None, 'global_cond_dim': 118, 'diffusion_step_embed_dim': 256, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'upsample_kernel_size': 4, 'n_groups': 8, 'cond_predict_scale': True}, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddpm.DDPMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'variance_type': 'fixed_small', 'clip_sample': True, 'prediction_type': 'epsilon'}, 'horizon': 16, 'obs_dim': 59, 'action_dim': 9, 'n_action_steps': 8, 'n_obs_steps': 2, 'num_inference_steps': 10, 'obs_as_local_cond': False, 'obs_as_global_cond': True, 'pred_action_steps_only': False, 'oa_step_convention': True}, 'ema': {'_target_': 'llmbc.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 256, 'num_workers': 0, 'shuffle': True, 'pin_memory': False, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 256, 'num_workers': 0, 'shuffle': False, 'pin_memory': False, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 45, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 5001, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 50, 'checkpoint_every': 50, 'val_every': 1, 'sample_every': 50, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'project': 'FrankaKitchen-v1-training', 'resume': True, 'mode': 'online', 'name': '2026.10.05-13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1', 'tags': ['train_diffusion_unet_lowdim', 'FrankaKitchen-v1', 'default'], 'id': None, 'group': None}, 'checkpoint': {'topk': {'monitor_key': 'test_success_rate', 'mode': 'max', 'k': 5, 'format_str': 'epoch={epoch:04d}-test_success_rate={test_success_rate:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1', 'wandb_name_base': '2026.10.05-13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1'}, 'task': {'name': 'FrankaKitchen-v1', 'obs_dim': 59, 'action_dim': 9, 'lang_multistep_merger': {'_target_': 'llfbench.envs.kitchen.multistep_merger.franka_kitchen_merger.KitchenMultistepMerger'}, 'env_runner': {'_target_': 'llmbc.env_runner.kitchen_lowdim_runner.KitchenLowdimRunner', 'env_name': 'llf-kitchen-FrankaKitchen-v1', 'n_train': 10, 'n_test': 50, 'n_envs': 10, 'max_steps': 500, 'n_obs_steps': 2, 'n_action_steps': 8, 'instruction_type': 'b', 'feedback_type': ['hp', 'hn', 'fp'], 'tasks_to_complete': ['microwave', 'kettle', 'light switch', 'slide cabinet'], 'visual': False, 'discount': 0.99, 'inference_expert_action': False, 'lang_multistep_merger': {'_target_': 'llfbench.envs.kitchen.multistep_merger.franka_kitchen_merger.KitchenMultistepMerger'}}, 'dataset': {'_target_': 'llmbc.dataset.kitchen_lowdim_dataset.KitchenLowdimDataset', 'data_path': 'datasets/FrankaKitchen-v1.pt', 'data_path2': 'datasets/FrankaKitchen-v1.pt', 'horizon': 16, 'pad_before': 1, 'pad_after': 7, 'obs_eef_target': True, 'use_manual_normalizer': False, 'val_ratio': 0.05, 'dummy_normalizer': False}, 'instructor': {'_target_': 'llmbc.translator.instructor.kitchen_instructor.FrankaKitchen_v1_instructor.FrankaKitchenV1Instructor'}}}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:init():669] starting backend
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:init():673] sending inform_init request
+2026-10-05 13:45:59,162 INFO    MainThread:1342604 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-10-05 13:45:59,162 INFO    MainThread:1342604 [wandb_init.py:init():686] backend started and connected
+2026-10-05 13:45:59,171 INFO    MainThread:1342604 [wandb_init.py:init():781] updated telemetry
+2026-10-05 13:45:59,287 INFO    MainThread:1342604 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2026-10-05 13:46:00,431 INFO    MainThread:1342604 [wandb_init.py:init():867] starting run threads in backend
+2026-10-05 13:46:00,735 INFO    MainThread:1342604 [wandb_run.py:_console_start():2451] atexit reg
+2026-10-05 13:46:00,735 INFO    MainThread:1342604 [wandb_run.py:_redirect():2299] redirect: wrap_raw
+2026-10-05 13:46:00,735 INFO    MainThread:1342604 [wandb_run.py:_redirect():2364] Wrapping output streams.
+2026-10-05 13:46:00,735 INFO    MainThread:1342604 [wandb_run.py:_redirect():2389] Redirects installed.
+2026-10-05 13:46:00,736 INFO    MainThread:1342604 [wandb_init.py:init():911] run started, returning control to user process
+2026-10-05 13:46:00,737 INFO    MainThread:1342604 [wandb_run.py:_config_callback():1389] config_cb None None {'output_dir': '/tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1'}
+2026-10-06 13:19:59,039 WARNING MsgRouterThr:1342604 [router.py:message_loop():75] message_loop has been closed

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/files/config.yaml

@@ -0,0 +1,244 @@
+_target_:
+    value: llmbc.workspace.train_diffusion_unet_lowdim_workspace.TrainDiffusionUnetLowdimWorkspace
+_wandb:
+    value:
+        cli_version: 0.18.6
+        m: []
+        python_version: 3.9.23
+        t:
+            "1":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 12
+                - 41
+                - 49
+                - 50
+                - 51
+                - 53
+                - 55
+                - 71
+                - 83
+                - 95
+                - 98
+                - 100
+                - 105
+            "2":
+                - 1
+                - 2
+                - 3
+                - 5
+                - 11
+                - 12
+                - 41
+                - 49
+                - 50
+                - 51
+                - 53
+                - 55
+                - 71
+                - 83
+                - 95
+                - 98
+                - 100
+                - 105
+            "3":
+                - 13
+                - 15
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.9.23
+            "5": 0.18.6
+            "6": 4.47.1
+            "8":
+                - 5
+            "12": 0.18.6
+            "13": linux-x86_64
+action_dim:
+    value: 9
+checkpoint:
+    value:
+        save_last_ckpt: true
+        save_last_snapshot: false
+        topk:
+            format_str: epoch={epoch:04d}-test_success_rate={test_success_rate:.3f}.ckpt
+            k: 5
+            mode: max
+            monitor_key: test_success_rate
+dataloader:
+    value:
+        batch_size: 256
+        num_workers: 0
+        persistent_workers: false
+        pin_memory: false
+        shuffle: true
+ema:
+    value:
+        _target_: llmbc.model.diffusion.ema_model.EMAModel
+        inv_gamma: 1
+        max_value: 0.9999
+        min_value: 0
+        power: 0.75
+        update_after_step: 0
+exp_name:
+    value: default
+horizon:
+    value: 16
+keypoint_visible_rate:
+    value: 1
+logging:
+    value:
+        group: null
+        id: null
+        mode: online
+        name: 2026.10.05-13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1
+        project: FrankaKitchen-v1-training
+        resume: true
+        tags:
+            - train_diffusion_unet_lowdim
+            - FrankaKitchen-v1
+            - default
+multi_run:
+    value:
+        run_dir: data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1
+        wandb_name_base: 2026.10.05-13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1
+n_action_steps:
+    value: 8
+n_latency_steps:
+    value: 0
+n_obs_steps:
+    value: 2
+name:
+    value: train_diffusion_unet_lowdim
+obs_as_global_cond:
+    value: true
+obs_as_local_cond:
+    value: false
+obs_dim:
+    value: 59
+optimizer:
+    value:
+        _target_: torch.optim.AdamW
+        betas:
+            - 0.95
+            - 0.999
+        eps: 1e-08
+        lr: 0.0001
+        weight_decay: 1e-06
+output_dir:
+    value: /tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1
+past_action_visible:
+    value: false
+policy:
+    value:
+        _target_: llmbc.policy.diffusion_unet_lowdim_policy.DiffusionUnetLowdimPolicy
+        action_dim: 9
+        horizon: 16
+        model:
+            _target_: llmbc.model.diffusion.conditional_unet1d.ConditionalUnet1D
+            cond_predict_scale: true
+            diffusion_step_embed_dim: 256
+            down_dims:
+                - 256
+                - 512
+                - 1024
+            global_cond_dim: 118
+            input_dim: 9
+            kernel_size: 5
+            local_cond_dim: null
+            n_groups: 8
+            upsample_kernel_size: 4
+        n_action_steps: 8
+        n_obs_steps: 2
+        noise_scheduler:
+            _target_: diffusers.schedulers.scheduling_ddpm.DDPMScheduler
+            beta_end: 0.02
+            beta_schedule: squaredcos_cap_v2
+            beta_start: 0.0001
+            clip_sample: true
+            num_train_timesteps: 100
+            prediction_type: epsilon
+            variance_type: fixed_small
+        num_inference_steps: 10
+        oa_step_convention: true
+        obs_as_global_cond: true
+        obs_as_local_cond: false
+        obs_dim: 59
+        pred_action_steps_only: false
+pred_action_steps_only:
+    value: false
+task:
+    value:
+        action_dim: 9
+        dataset:
+            _target_: llmbc.dataset.kitchen_lowdim_dataset.KitchenLowdimDataset
+            data_path: datasets/FrankaKitchen-v1.pt
+            data_path2: datasets/FrankaKitchen-v1.pt
+            dummy_normalizer: false
+            horizon: 16
+            obs_eef_target: true
+            pad_after: 7
+            pad_before: 1
+            use_manual_normalizer: false
+            val_ratio: 0.05
+        env_runner:
+            _target_: llmbc.env_runner.kitchen_lowdim_runner.KitchenLowdimRunner
+            discount: 0.99
+            env_name: llf-kitchen-FrankaKitchen-v1
+            feedback_type:
+                - hp
+                - hn
+                - fp
+            inference_expert_action: false
+            instruction_type: b
+            lang_multistep_merger:
+                _target_: llfbench.envs.kitchen.multistep_merger.franka_kitchen_merger.KitchenMultistepMerger
+            max_steps: 500
+            n_action_steps: 8
+            n_envs: 10
+            n_obs_steps: 2
+            n_test: 50
+            n_train: 10
+            tasks_to_complete:
+                - microwave
+                - kettle
+                - light switch
+                - slide cabinet
+            visual: false
+        instructor:
+            _target_: llmbc.translator.instructor.kitchen_instructor.FrankaKitchen_v1_instructor.FrankaKitchenV1Instructor
+        lang_multistep_merger:
+            _target_: llfbench.envs.kitchen.multistep_merger.franka_kitchen_merger.KitchenMultistepMerger
+        name: FrankaKitchen-v1
+        obs_dim: 59
+task_name:
+    value: FrankaKitchen-v1
+training:
+    value:
+        checkpoint_every: 50
+        debug: false
+        device: cuda:0
+        gradient_accumulate_every: 1
+        lr_scheduler: cosine
+        lr_warmup_steps: 500
+        max_train_steps: null
+        max_val_steps: null
+        num_epochs: 5001
+        resume: true
+        rollout_every: 50
+        sample_every: 50
+        seed: 45
+        tqdm_interval_sec: 1
+        use_ema: true
+        val_every: 1
+val_dataloader:
+    value:
+        batch_size: 256
+        num_workers: 0
+        persistent_workers: false
+        pin_memory: false
+        shuffle: false

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/files/output.log

@@ -0,0 +1,3 @@
+Eval KitchenLowdimRunner 1/6:   0%|                                                                                                             | 0/500 [00:00<?, ?it/s]/tmp2/chyang/workspace/LLM-BC/llmbc/common/llfbench_util.py:43: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ../torch/csrc/utils/tensor_new.cpp:275.)
+  obs = torch.tensor(obs, dtype=torch.float32).unsqueeze(dim=0).to(device)
+                                                                                                                                                                        

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/files/wandb-metadata.json

@@ -0,0 +1,56 @@
+{
+  "os": "Linux-6.8.0-139-generic-x86_64-with-glibc2.39",
+  "python": "3.9.23",
+  "startedAt": "2026-10-05T05:45:59.163218Z",
+  "args": [
+    "--config-path",
+    "config/main_table",
+    "--config-name",
+    "dp_FrankaKitchen-v1.yaml",
+    "training.seed=45"
+  ],
+  "program": "/tmp2/chyang/workspace/LLM-BC/./train.py",
+  "codePath": "train.py",
+  "git": {
+    "remote": "https://github.com/CHYang25/LLM-BC.git",
+    "commit": "0387ae9461c25cfb52eee0b8b2c4eaeca0f29642"
+  },
+  "email": "chris920325@gmail.com",
+  "root": "/tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1",
+  "host": "rllab1015",
+  "username": "chyang",
+  "executable": "/home/chyang/miniconda3/envs/llm-bc/bin/python3",
+  "codePathLocal": "train.py",
+  "cpu_count": 20,
+  "cpu_count_logical": 40,
+  "gpu": "NVIDIA GeForce RTX 4090",
+  "gpu_count": 2,
+  "disk": {
+    "/": {
+      "total": "1967317549056",
+      "used": "1392980328448"
+    }
+  },
+  "memory": {
+    "total": "66890878976"
+  },
+  "cpu": {
+    "count": 20,
+    "countLogical": 40
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    },
+    {
+      "name": "NVIDIA GeForce RTX 4090",
+      "memoryTotal": "25757220864",
+      "cudaCores": 16384,
+      "architecture": "Ada"
+    }
+  ],
+  "cudaVersion": "13.0"
+}

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/files/wandb-summary.json

@@ -0,0 +1 @@
+{"test/p3_score":0.72,"_runtime":84839.875924085,"_wandb":{"runtime":84839},"_timestamp":1.7912639913773453e+09,"train/p2_score":1,"global_step":2480495,"train_action_mse_error":0.00022003134654369205,"test/cumulative_reward":1.0155143312105646,"test/p1_score":1,"test/p4_score":0.5,"train/p4_score":0.7,"test/p2_score":0.86,"train/cumulative_reward":1.1194035906854443,"test/mean_score":0.77,"epoch":5000,"train/success_rate":0.7,"val_loss":0.052233316004276276,"train/mean_score":0.925,"lr":0,"test/success_rate":0.5,"train/p3_score":1,"train_loss":0.0001353582352426348,"_step":2480495,"train/p1_score":1}

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2026-10-05T13:45:58.520988554+08:00","level":"INFO","msg":"started logging, with flags","port-filename":"/tmp/tmpxqdfhghl/port-1342604.txt","pid":1342604,"debug":false,"disable-analytics":false}
+{"time":"2026-10-05T13:45:58.521011902+08:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2026-10-05T13:45:58.522084677+08:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":38729,"Zone":""}}
+{"time":"2026-10-05T13:45:58.522156611+08:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":1342604}
+{"time":"2026-10-05T13:45:58.719068703+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:51142"}
+{"time":"2026-10-05T13:45:59.163255832+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"a0y1jzsf","id":"127.0.0.1:51142"}
+{"time":"2026-10-05T13:45:59.269117398+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"a0y1jzsf","id":"127.0.0.1:51142"}
+{"time":"2026-10-06T13:19:59.03889803+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:51142"}
+{"time":"2026-10-06T13:19:59.039127777+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-10-06T13:19:59.039119149+08:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:51142"}
+{"time":"2026-10-06T13:19:59.039243207+08:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:51142"}
+{"time":"2026-10-06T13:20:01.218422377+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:51142"}
+{"time":"2026-10-06T13:20:01.218453721+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:51142"}
+{"time":"2026-10-06T13:20:01.218469605+08:00","level":"INFO","msg":"server is closed"}

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/logs/debug-internal.log

@@ -0,0 +1,162 @@
+{"time":"2026-10-05T13:45:59.163355915+08:00","level":"INFO","msg":"using version","core version":"0.18.6"}
+{"time":"2026-10-05T13:45:59.163364924+08:00","level":"INFO","msg":"created symlink","path":"/tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/logs/debug-core.log"}
+{"time":"2026-10-05T13:45:59.269047163+08:00","level":"INFO","msg":"created new stream","id":"a0y1jzsf"}
+{"time":"2026-10-05T13:45:59.269111886+08:00","level":"INFO","msg":"stream: started","id":"a0y1jzsf"}
+{"time":"2026-10-05T13:45:59.269178689+08:00","level":"INFO","msg":"handler: started","stream_id":{"value":"a0y1jzsf"}}
+{"time":"2026-10-05T13:45:59.269379864+08:00","level":"INFO","msg":"sender: started","stream_id":"a0y1jzsf"}
+{"time":"2026-10-05T13:45:59.269168469+08:00","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"a0y1jzsf"}}
+{"time":"2026-10-05T13:46:00.434351743+08:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2026-10-06T00:55:17.233211224+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 192.168.11.41:58154->34.8.250.101:443: read: connection reset by peer"}
+{"time":"2026-10-06T09:25:04.216906944+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:25:15.252828409+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp 34.8.250.101:443: connect: no route to host"}
+{"time":"2026-10-06T09:25:26.397707765+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:39419->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:25:36.667141225+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:25:36.756524289+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:25:54.045717582+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:60142->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:26:10.833705525+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:26:19.764617144+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:26:50.502002568+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:26:57.698292324+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:27:39.461033137+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:28:07.699959385+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:50452->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:28:43.153942633+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:29:17.703107112+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:36665->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:30:13.15448044+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:30:27.705770185+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:42721->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:31:37.708864611+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:43878->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:31:43.15597339+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:32:47.710960866+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:46731->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:33:13.157227823+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:33:57.71429012+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:38521->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:34:34.218070669+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000912319,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"8lx8hbxpiv60\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:34:35.701292599+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000212389,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:34:40.436056895+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000583931,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:40.436161283+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000713944,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:40.438335724+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000227179,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:40.439452019+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000451924,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:40.4395116+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000070491,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:34:43.158026918+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:35:07.716961743+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:57595->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:36:13.159016826+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:36:17.72009588+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:37109->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:37:27.722522334+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:34984->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:37:43.161342328+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T09:38:37.726080975+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:57156->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:39:13.162840088+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:39:47.728677821+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59728->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:40:43.164031242+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T09:40:57.730952264+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:42114->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:41:53.16702903+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:54374->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:42:07.73436387+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:50668->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:43:03.16922781+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:54076->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:43:17.736868037+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:55535->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:44:13.171973988+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:36098->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:44:27.739336521+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:54781->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:44:34.239471319+08:00","level":"WARN","msg":"sender: taking a long time","seconds":1200.022912721,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"8lx8hbxpiv60\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:44:35.70342383+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.002371121,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:44:40.436779533+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.001288564,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:44:40.436895391+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.001441669,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:44:40.439087859+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.00097571,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:44:40.440188055+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.000730352,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:44:40.440189578+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":1200.001201081,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:45:23.174036006+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:32808->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:45:37.74262234+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:57118->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:46:33.175647107+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:34142->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:46:47.744827673+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:38788->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:47:43.178129417+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:51485->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:47:57.747348679+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:41227->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:48:53.180282859+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:58661->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:48:53.180492936+08:00","level":"ERROR","msg":"sender: sendStopStatus: failed to get run stopped status: api: failed sending: POST https://api.wandb.ai/graphql giving up after 21 attempt(s): Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:58661->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:48:53.182181928+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1458.965629728,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"8lx8hbxpiv60\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:48:53.182330263+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1457.481319991,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:48:53.182428528+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.746976632,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:48:53.182475062+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.744278468,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:48:53.182446719+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.742950553,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:48:53.182566064+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.743523327,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:48:53.182440969+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1452.746942282,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:49:03.31098711+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:52560->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:49:03.311014342+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:52560->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:49:15.51000472+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:46088->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:49:30.220917721+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:43837->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:49:49.577054247+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:51242->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:50:08.787118718+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:50:08.7871671+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T09:50:57.235008055+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59888->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:51:18.790095596+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:45443->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:52:07.238034763+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:60728->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:52:28.793223924+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:54344->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:53:17.241303288+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59484->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:53:38.795011432+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:50171->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:54:27.244118673+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59227->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:54:48.797972117+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:45997->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:55:37.247023164+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:34908->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:55:58.800364514+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:49683->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:56:47.249888902+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:60992->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:57:08.802311113+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:53471->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:57:57.252954476+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:40719->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:58:17.085117628+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:59879->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:58:23.228954725+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:48731->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:58:53.310509872+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000650079,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"gplclwr5yrv3\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:58:54.728910487+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000849739,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T09:59:00.435880525+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.00045047,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:00.43601939+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000525846,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:00.438191073+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.00039204,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:00.438198711+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000120136,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:00.439335822+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000417846,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T09:59:05.085657873+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:46257->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T09:59:32.673067475+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:38517->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:00:15.088446059+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:42397->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:00:42.675974984+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:33976->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:01:25.090369352+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:41435->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:01:52.679257229+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:37099->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:02:35.093533054+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:43216->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:03:02.682033223+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:58548->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:03:45.095777469+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:50451->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:04:12.685307219+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:56278->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:04:55.099256898+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:48863->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:05:22.688113437+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:49585->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:06:05.101706688+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: read udp 127.0.0.1:40857->127.0.0.53:53: i/o timeout"}
+{"time":"2026-10-06T10:06:30.469216735+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/b10902069-national-taiwan-university/FrankaKitchen-v1-training/a0y1jzsf/file_stream\": dial tcp: lookup api.wandb.ai on 127.0.0.53:53: server misbehaving"}
+{"time":"2026-10-06T10:07:05.891669426+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1092.581635314,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"gplclwr5yrv3\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:07:05.891847858+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1091.163881748,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:07:05.89194221+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.456543256,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:07:05.892005599+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.456550335,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:07:05.892078977+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.454229769,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:07:05.892165516+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.454052586,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:07:05.892249043+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1085.45331293,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:08:20.896007012+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:08:52.988732973+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:09:27.388072917+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:10:07.176838085+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:10:54.373793246+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:12:02.60384915+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:13:32.605053705+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:15:02.606602306+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:16:32.608834641+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:17:50.89575474+08:00","level":"WARN","msg":"sender: taking a long time","seconds":600.000208414,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"ckfgj8ye2snm\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:17:52.434840284+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000559945,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:18:00.43574907+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000133108,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:00.435792208+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000401175,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:00.438949663+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000530324,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:00.438965768+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.000356572,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:00.439011938+08:00","level":"WARN","msg":"runwork: taking a long time","seconds":600.00004734,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:18:02.610833003+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:19:32.611778582+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2026-10-06T10:21:02.61382365+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:22:32.615544993+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-10-06T10:23:42.963766085+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": read tcp 192.168.11.41:41968->34.8.250.101:443: read: network is unreachable"}
+{"time":"2026-10-06T10:24:43.875271386+08:00","level":"INFO","msg":"sender: succeeded after taking longer than expected","seconds":1012.979744627,"work":"WorkRecord(*service_go_proto.Request_StopStatus); Control(local:true mailbox_slot:\"ckfgj8ye2snm\" connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:24:43.875689942+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1011.441443372,"work":"WorkRecord(*service_go_proto.Request_PartialHistory); Control(local:true connection_id:\"127.0.0.1:51142\")"}
+{"time":"2026-10-06T10:24:43.875721175+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.440111468,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:24:43.875814017+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.437401986,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:24:43.875746082+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.440385756,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:24:43.875852672+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.436850157,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T10:24:43.876036242+08:00","level":"INFO","msg":"runwork: succeeded after taking longer than expected","seconds":1003.437415844,"work":"WorkRecord(*service_go_proto.Record_Stats); Control(always_send:true)"}
+{"time":"2026-10-06T13:19:59.039120309+08:00","level":"INFO","msg":"stream: closing","id":"a0y1jzsf"}
+{"time":"2026-10-06T13:19:59.039154305+08:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2026-10-06T13:19:59.111492217+08:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2026-10-06T13:20:00.866016279+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-10-06T13:20:01.218067036+08:00","level":"INFO","msg":"handler: closed","stream_id":{"value":"a0y1jzsf"}}
+{"time":"2026-10-06T13:20:01.21814782+08:00","level":"INFO","msg":"sender: closed","stream_id":"a0y1jzsf"}
+{"time":"2026-10-06T13:20:01.218137984+08:00","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"a0y1jzsf"}}
+{"time":"2026-10-06T13:20:01.218298956+08:00","level":"INFO","msg":"stream: closed","id":"a0y1jzsf"}

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/logs/debug.log

@@ -0,0 +1,27 @@
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Current SDK version is 0.18.6
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Configure stats pid to 1342604
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Loading settings from /home/chyang/.config/wandb/settings
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Loading settings from /tmp2/chyang/workspace/LLM-BC/wandb/settings
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Loading settings from environment variables: {}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': 'train.py', 'program_abspath': '/tmp2/chyang/workspace/LLM-BC/train.py', 'program': '/tmp2/chyang/workspace/LLM-BC/./train.py'}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_setup.py:_flush():79] Applying login settings: {}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:_log_setup():533] Logging user logs to /tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/logs/debug.log
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:_log_setup():534] Logging internal logs to /tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/logs/debug-internal.log
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:init():619] calling init triggers
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {'name': 'train_diffusion_unet_lowdim', '_target_': 'llmbc.workspace.train_diffusion_unet_lowdim_workspace.TrainDiffusionUnetLowdimWorkspace', 'obs_dim': 59, 'action_dim': 9, 'task_name': 'FrankaKitchen-v1', 'exp_name': 'default', 'horizon': 16, 'n_obs_steps': 2, 'n_action_steps': 8, 'n_latency_steps': 0, 'past_action_visible': False, 'keypoint_visible_rate': 1.0, 'obs_as_local_cond': False, 'obs_as_global_cond': True, 'pred_action_steps_only': False, 'policy': {'_target_': 'llmbc.policy.diffusion_unet_lowdim_policy.DiffusionUnetLowdimPolicy', 'model': {'_target_': 'llmbc.model.diffusion.conditional_unet1d.ConditionalUnet1D', 'input_dim': 9, 'local_cond_dim': None, 'global_cond_dim': 118, 'diffusion_step_embed_dim': 256, 'down_dims': [256, 512, 1024], 'kernel_size': 5, 'upsample_kernel_size': 4, 'n_groups': 8, 'cond_predict_scale': True}, 'noise_scheduler': {'_target_': 'diffusers.schedulers.scheduling_ddpm.DDPMScheduler', 'num_train_timesteps': 100, 'beta_start': 0.0001, 'beta_end': 0.02, 'beta_schedule': 'squaredcos_cap_v2', 'variance_type': 'fixed_small', 'clip_sample': True, 'prediction_type': 'epsilon'}, 'horizon': 16, 'obs_dim': 59, 'action_dim': 9, 'n_action_steps': 8, 'n_obs_steps': 2, 'num_inference_steps': 10, 'obs_as_local_cond': False, 'obs_as_global_cond': True, 'pred_action_steps_only': False, 'oa_step_convention': True}, 'ema': {'_target_': 'llmbc.model.diffusion.ema_model.EMAModel', 'update_after_step': 0, 'inv_gamma': 1.0, 'power': 0.75, 'min_value': 0.0, 'max_value': 0.9999}, 'dataloader': {'batch_size': 256, 'num_workers': 0, 'shuffle': True, 'pin_memory': False, 'persistent_workers': False}, 'val_dataloader': {'batch_size': 256, 'num_workers': 0, 'shuffle': False, 'pin_memory': False, 'persistent_workers': False}, 'optimizer': {'_target_': 'torch.optim.AdamW', 'lr': 0.0001, 'betas': [0.95, 0.999], 'eps': 1e-08, 'weight_decay': 1e-06}, 'training': {'device': 'cuda:0', 'seed': 45, 'debug': False, 'resume': True, 'lr_scheduler': 'cosine', 'lr_warmup_steps': 500, 'num_epochs': 5001, 'gradient_accumulate_every': 1, 'use_ema': True, 'rollout_every': 50, 'checkpoint_every': 50, 'val_every': 1, 'sample_every': 50, 'max_train_steps': None, 'max_val_steps': None, 'tqdm_interval_sec': 1.0}, 'logging': {'project': 'FrankaKitchen-v1-training', 'resume': True, 'mode': 'online', 'name': '2026.10.05-13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1', 'tags': ['train_diffusion_unet_lowdim', 'FrankaKitchen-v1', 'default'], 'id': None, 'group': None}, 'checkpoint': {'topk': {'monitor_key': 'test_success_rate', 'mode': 'max', 'k': 5, 'format_str': 'epoch={epoch:04d}-test_success_rate={test_success_rate:.3f}.ckpt'}, 'save_last_ckpt': True, 'save_last_snapshot': False}, 'multi_run': {'run_dir': 'data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1', 'wandb_name_base': '2026.10.05-13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1'}, 'task': {'name': 'FrankaKitchen-v1', 'obs_dim': 59, 'action_dim': 9, 'lang_multistep_merger': {'_target_': 'llfbench.envs.kitchen.multistep_merger.franka_kitchen_merger.KitchenMultistepMerger'}, 'env_runner': {'_target_': 'llmbc.env_runner.kitchen_lowdim_runner.KitchenLowdimRunner', 'env_name': 'llf-kitchen-FrankaKitchen-v1', 'n_train': 10, 'n_test': 50, 'n_envs': 10, 'max_steps': 500, 'n_obs_steps': 2, 'n_action_steps': 8, 'instruction_type': 'b', 'feedback_type': ['hp', 'hn', 'fp'], 'tasks_to_complete': ['microwave', 'kettle', 'light switch', 'slide cabinet'], 'visual': False, 'discount': 0.99, 'inference_expert_action': False, 'lang_multistep_merger': {'_target_': 'llfbench.envs.kitchen.multistep_merger.franka_kitchen_merger.KitchenMultistepMerger'}}, 'dataset': {'_target_': 'llmbc.dataset.kitchen_lowdim_dataset.KitchenLowdimDataset', 'data_path': 'datasets/FrankaKitchen-v1.pt', 'data_path2': 'datasets/FrankaKitchen-v1.pt', 'horizon': 16, 'pad_before': 1, 'pad_after': 7, 'obs_eef_target': True, 'use_manual_normalizer': False, 'val_ratio': 0.05, 'dummy_normalizer': False}, 'instructor': {'_target_': 'llmbc.translator.instructor.kitchen_instructor.FrankaKitchen_v1_instructor.FrankaKitchenV1Instructor'}}}
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:init():669] starting backend
+2026-10-05 13:45:59,161 INFO    MainThread:1342604 [wandb_init.py:init():673] sending inform_init request
+2026-10-05 13:45:59,162 INFO    MainThread:1342604 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2026-10-05 13:45:59,162 INFO    MainThread:1342604 [wandb_init.py:init():686] backend started and connected
+2026-10-05 13:45:59,171 INFO    MainThread:1342604 [wandb_init.py:init():781] updated telemetry
+2026-10-05 13:45:59,287 INFO    MainThread:1342604 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2026-10-05 13:46:00,431 INFO    MainThread:1342604 [wandb_init.py:init():867] starting run threads in backend
+2026-10-05 13:46:00,735 INFO    MainThread:1342604 [wandb_run.py:_console_start():2451] atexit reg
+2026-10-05 13:46:00,735 INFO    MainThread:1342604 [wandb_run.py:_redirect():2299] redirect: wrap_raw
+2026-10-05 13:46:00,735 INFO    MainThread:1342604 [wandb_run.py:_redirect():2364] Wrapping output streams.
+2026-10-05 13:46:00,735 INFO    MainThread:1342604 [wandb_run.py:_redirect():2389] Redirects installed.
+2026-10-05 13:46:00,736 INFO    MainThread:1342604 [wandb_init.py:init():911] run started, returning control to user process
+2026-10-05 13:46:00,737 INFO    MainThread:1342604 [wandb_run.py:_config_callback():1389] config_cb None None {'output_dir': '/tmp2/chyang/workspace/LLM-BC/data/outputs/2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1'}
+2026-10-06 13:19:59,039 WARNING MsgRouterThr:1342604 [router.py:message_loop():75] message_loop has been closed

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/run-20261005_134559-a0y1jzsf/run-a0y1jzsf.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3daf68b047568a0ed95cf444c0af0e4b7bc7c151638f8f35a2af528cbd34acdb
+size 1220692416

2026.10.05/13.45.49_train_diffusion_unet_lowdim_FrankaKitchen-v1/wandb/wandb-resume.json

@@ -0,0 +1 @@
+{"run_id": "a0y1jzsf"}