Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+foundation_model/wandb/run-20241121_142236-scq8411d/run-scq8411d.wandb filter=lfs diff=lfs merge=lfs -text
+superconductor_generator/wandb/run-20241126_222417-csuxafea/run-csuxafea.wandb filter=lfs diff=lfs merge=lfs -text

foundation_model/.hydra/config.yaml

@@ -0,0 +1,163 @@
+data:
+  root_path: ${oc.env:PROJECT_ROOT}/data/alex_20
+  prop: eah
+  num_targets: 1
+  niggli: true
+  primitive: false
+  graph_method: crystalnn
+  lattice_scale_method: scale_length
+  preprocess_workers: 30
+  readout: mean
+  max_atoms: 20
+  otf_graph: false
+  eval_model_name: alex20
+  tolerance: 0.1
+  use_space_group: false
+  use_pos_index: false
+  train_max_epochs: 6000
+  early_stopping_patience: 60
+  teacher_forcing_max_epoch: 500
+  datamodule:
+    _target_: diffcsp.pl_data.datamodule.CrystDataModule
+    datasets:
+      train:
+        _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy train
+        path: ${data.root_path}/train.csv
+        save_path: ${data.root_path}/train_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+      val:
+      - _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy val
+        path: ${data.root_path}/val.csv
+        save_path: ${data.root_path}/val_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+      test:
+      - _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy test
+        path: ${data.root_path}/test.csv
+        save_path: ${data.root_path}/test_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+    num_workers:
+      train: 0
+      val: 0
+      test: 0
+    batch_size:
+      train: 4096
+      val: 4096
+      test: 1024
+logging:
+  val_check_interval: 1
+  wandb:
+    name: ${expname}
+    project: diffcsp
+    entity: null
+    log_model: true
+    mode: online
+    group: ${expname}
+  wandb_watch:
+    log: all
+    log_freq: 500
+  lr_monitor:
+    logging_interval: step
+    log_momentum: false
+model:
+  decoder:
+    _target_: diffcsp.pl_modules.cspnet.CSPNet
+    hidden_dim: 512
+    latent_dim: 0
+    max_atoms: 100
+    num_layers: 6
+    act_fn: silu
+    dis_emb: sin
+    num_freqs: 128
+    edge_style: fc
+    max_neighbors: ${model.max_neighbors}
+    cutoff: ${model.radius}
+    ln: true
+    ip: true
+  beta_scheduler:
+    _target_: diffcsp.pl_modules.diff_utils.BetaScheduler
+    timesteps: ${model.timesteps}
+    scheduler_mode: cosine
+  sigma_scheduler:
+    _target_: diffcsp.pl_modules.diff_utils.SigmaScheduler
+    timesteps: ${model.timesteps}
+    sigma_begin: 0.005
+    sigma_end: 0.5
+  _target_: diffcsp.pl_modules.diffusion_w_type.CSPDiffusion
+  time_dim: 256
+  latent_dim: 0
+  cost_coord: 1.0
+  cost_lattice: 1.0
+  cost_type: 20.0
+  max_neighbors: 20
+  radius: 7.0
+  timesteps: 1000
+optim:
+  optimizer:
+    _target_: torch.optim.Adam
+    lr: 0.001
+    betas:
+    - 0.9
+    - 0.999
+    eps: 1.0e-08
+    weight_decay: 0
+  use_lr_scheduler: true
+  lr_scheduler:
+    _target_: torch.optim.lr_scheduler.ReduceLROnPlateau
+    factor: 0.6
+    patience: 30
+    min_lr: 0.0001
+train:
+  deterministic: true
+  random_seed: 42
+  pl_trainer:
+    fast_dev_run: false
+    accelerator: auto
+    precision: 32
+    max_epochs: ${data.train_max_epochs}
+    accumulate_grad_batches: 1
+    num_sanity_val_steps: 2
+    gradient_clip_val: 0.5
+    gradient_clip_algorithm: value
+    profiler: simple
+  monitor_metric: val_loss
+  monitor_metric_mode: min
+  early_stopping:
+    patience: ${data.early_stopping_patience}
+    verbose: true
+  model_checkpoints:
+    save_top_k: 1
+    verbose: false
+    save_last: false
+  ckpt_path: null
+expname: alex_20
+core:
+  version: 0.0.1
+  tags:
+  - ${now:%Y-%m-%d}

foundation_model/.hydra/hydra.yaml

@@ -0,0 +1,167 @@
+hydra:
+  run:
+    dir: ${oc.env:HYDRA_JOBS}/singlerun/${now:%Y-%m-%d}/${expname}/
+  sweep:
+    dir: ${oc.env:HYDRA_JOBS}/multirun/${now:%Y-%m-%d}/${expname}/
+    subdir: ${hydra.job.num}_${hydra.job.id}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data=alex_20
+    - model=diffusion_w_type
+    - expname=alex_20
+  job:
+    name: run
+    chdir: null
+    override_dirname: data=alex_20,expname=alex_20,model=diffusion_w_type
+    id: ???
+    num: ???
+    config_name: default
+    env_set:
+      WANDB_START_METHOD: thread
+      WANDB_DIR: ${oc.env:WABDB_DIR}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20
+    choices:
+      train: default
+      optim: default
+      model: diffusion_w_type
+      model/sigma_scheduler: wrapped
+      model/beta_scheduler: cosine
+      model/decoder: cspnet
+      logging: default
+      data: alex_20
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

foundation_model/.hydra/overrides.yaml

@@ -0,0 +1,3 @@
+- data=alex_20
+- model=diffusion_w_type
+- expname=alex_20

foundation_model/epoch=656-step=74898.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:425f3ab4ec9a8dc12089ee966dd38952e50dcbe6917a740788874992432071c6
+size 148295922

foundation_model/hparams.yaml

@@ -0,0 +1,163 @@
+data:
+  root_path: ${oc.env:PROJECT_ROOT}/data/alex_20
+  prop: eah
+  num_targets: 1
+  niggli: true
+  primitive: false
+  graph_method: crystalnn
+  lattice_scale_method: scale_length
+  preprocess_workers: 30
+  readout: mean
+  max_atoms: 20
+  otf_graph: false
+  eval_model_name: alex20
+  tolerance: 0.1
+  use_space_group: false
+  use_pos_index: false
+  train_max_epochs: 6000
+  early_stopping_patience: 60
+  teacher_forcing_max_epoch: 500
+  datamodule:
+    _target_: diffcsp.pl_data.datamodule.CrystDataModule
+    datasets:
+      train:
+        _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy train
+        path: ${data.root_path}/train.csv
+        save_path: ${data.root_path}/train_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+      val:
+      - _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy val
+        path: ${data.root_path}/val.csv
+        save_path: ${data.root_path}/val_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+      test:
+      - _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy test
+        path: ${data.root_path}/test.csv
+        save_path: ${data.root_path}/test_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+    num_workers:
+      train: 0
+      val: 0
+      test: 0
+    batch_size:
+      train: 4096
+      val: 4096
+      test: 1024
+logging:
+  val_check_interval: 1
+  wandb:
+    name: ${expname}
+    project: diffcsp
+    entity: null
+    log_model: true
+    mode: online
+    group: ${expname}
+  wandb_watch:
+    log: all
+    log_freq: 500
+  lr_monitor:
+    logging_interval: step
+    log_momentum: false
+model:
+  decoder:
+    _target_: diffcsp.pl_modules.cspnet.CSPNet
+    hidden_dim: 512
+    latent_dim: 0
+    max_atoms: 100
+    num_layers: 6
+    act_fn: silu
+    dis_emb: sin
+    num_freqs: 128
+    edge_style: fc
+    max_neighbors: ${model.max_neighbors}
+    cutoff: ${model.radius}
+    ln: true
+    ip: true
+  beta_scheduler:
+    _target_: diffcsp.pl_modules.diff_utils.BetaScheduler
+    timesteps: ${model.timesteps}
+    scheduler_mode: cosine
+  sigma_scheduler:
+    _target_: diffcsp.pl_modules.diff_utils.SigmaScheduler
+    timesteps: ${model.timesteps}
+    sigma_begin: 0.005
+    sigma_end: 0.5
+  _target_: diffcsp.pl_modules.diffusion_w_type.CSPDiffusion
+  time_dim: 256
+  latent_dim: 0
+  cost_coord: 1.0
+  cost_lattice: 1.0
+  cost_type: 20.0
+  max_neighbors: 20
+  radius: 7.0
+  timesteps: 1000
+optim:
+  optimizer:
+    _target_: torch.optim.Adam
+    lr: 0.001
+    betas:
+    - 0.9
+    - 0.999
+    eps: 1.0e-08
+    weight_decay: 0
+  use_lr_scheduler: true
+  lr_scheduler:
+    _target_: torch.optim.lr_scheduler.ReduceLROnPlateau
+    factor: 0.6
+    patience: 30
+    min_lr: 0.0001
+train:
+  deterministic: true
+  random_seed: 42
+  pl_trainer:
+    fast_dev_run: false
+    accelerator: auto
+    precision: 32
+    max_epochs: ${data.train_max_epochs}
+    accumulate_grad_batches: 1
+    num_sanity_val_steps: 2
+    gradient_clip_val: 0.5
+    gradient_clip_algorithm: value
+    profiler: simple
+  monitor_metric: val_loss
+  monitor_metric_mode: min
+  early_stopping:
+    patience: ${data.early_stopping_patience}
+    verbose: true
+  model_checkpoints:
+    save_top_k: 1
+    verbose: false
+    save_last: false
+  ckpt_path: null
+expname: alex_20
+core:
+  version: 0.0.1
+  tags:
+  - ${now:%Y-%m-%d}

foundation_model/lattice_scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b37064c70812c4d2f89c56cf68896aeaf314feba3dd3bfd394f78bb5a16848ca
+size 1538

foundation_model/run.log

@@ -0,0 +1,12 @@
+[2024-11-21 14:20:16,224][hydra.utils][INFO] - Instantiating <diffcsp.pl_data.datamodule.CrystDataModule>
+[2024-11-21 14:22:24,447][hydra.utils][INFO] - Instantiating <diffcsp.pl_modules.diffusion_w_type.CSPDiffusion>
+[2024-11-21 14:22:36,218][hydra.utils][INFO] - Passing scaler from datamodule to model <StandardScalerTorch(means: 0.5455514788627625, stds: 0.6806707382202148)>
+[2024-11-21 14:22:36,330][hydra.utils][INFO] - Adding callback <LearningRateMonitor>
+[2024-11-21 14:22:36,330][hydra.utils][INFO] - Adding callback <EarlyStopping>
+[2024-11-21 14:22:36,330][hydra.utils][INFO] - Adding callback <ModelCheckpoint>
+[2024-11-21 14:22:36,332][hydra.utils][INFO] - Instantiating <WandbLogger>
+[2024-11-21 14:22:36,369][hydra.utils][INFO] - W&B is now watching <{cfg.logging.wandb_watch.log}>!
+[2024-11-21 14:22:38,029][hydra.utils][INFO] - No checkpoint found. Starting from scratch.
+[2024-11-21 14:22:38,029][hydra.utils][INFO] - Instantiating the Trainer
+[2024-11-21 14:22:38,080][hydra.utils][INFO] - Starting training!
+[2024-11-24 01:42:59,875][hydra.utils][INFO] - Starting testing!

foundation_model/wandb/debug-internal.log

@@ -0,0 +1,28 @@
+{"time":"2024-11-21T14:22:37.043045747-05:00","level":"INFO","msg":"using version","core version":"0.18.7"}
+{"time":"2024-11-21T14:22:37.066680433-05:00","level":"INFO","msg":"created symlink","path":"wandb/run-20241121_142236-scq8411d/logs/debug-core.log"}
+{"time":"2024-11-21T14:22:37.344284532-05:00","level":"INFO","msg":"created new stream","id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.344361949-05:00","level":"INFO","msg":"stream: started","id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.344614507-05:00","level":"INFO","msg":"handler: started","stream_id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.344662057-05:00","level":"INFO","msg":"writer: Do: started","stream_id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.344814565-05:00","level":"INFO","msg":"sender: started","stream_id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.708316569-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-21T15:55:28.886692431-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-21T18:10:31.807981434-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-21T22:26:49.683292877-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-22T03:53:28.60635499-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-22T18:45:11.801789802-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-23T23:42:04.677295397-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-23T23:42:22.690477214-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-23T23:42:32.589851211-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-24T04:29:48.5176947-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-24T04:29:49.683102181-05:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-24T04:29:49.687112558-05:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2024-11-24T04:29:52.034709299-05:00","level":"WARN","msg":"No job ingredients found, not creating job artifact"}
+{"time":"2024-11-24T04:29:52.039988286-05:00","level":"WARN","msg":"No source type found, not creating job artifact"}
+{"time":"2024-11-24T04:29:52.039996592-05:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-11-24T04:29:52.406602038-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-24T04:29:53.893798488-05:00","level":"INFO","msg":"stream: closing","id":"scq8411d"}
+{"time":"2024-11-24T04:29:53.893822644-05:00","level":"INFO","msg":"handler: closed","stream_id":"scq8411d"}
+{"time":"2024-11-24T04:29:53.89384157-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":"scq8411d"}
+{"time":"2024-11-24T04:29:53.908810589-05:00","level":"INFO","msg":"sender: closed","stream_id":"scq8411d"}
+{"time":"2024-11-24T04:29:53.908834234-05:00","level":"INFO","msg":"stream: closed","id":"scq8411d"}

foundation_model/wandb/debug.log

@@ -0,0 +1,35 @@
+2024-11-21 14:22:37,025 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
+2024-11-21 14:22:37,025 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Configure stats pid to 2298044
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Loading settings from /home/pawanprakash/.config/wandb/settings
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Loading settings from /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/wandb/settings
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'start_method': 'thread', 'root_dir': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/wandb'}
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2024-11-21 14:22:37,026 WARNING MainThread:2298044 [wandb_setup.py:_flush():79] Could not save program above cwd: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/alex_20/./../../MaterialsDiffusion_unguided/diffcsp/run.py
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program_abspath': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/diffcsp/run.py', 'program': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/alex_20/./../../MaterialsDiffusion_unguided/diffcsp/run.py'}
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_init.py:_log_setup():533] Logging user logs to ./wandb/run-20241121_142236-scq8411d/logs/debug.log
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:_log_setup():534] Logging internal logs to ./wandb/run-20241121_142236-scq8411d/logs/debug-internal.log
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:init():619] calling init triggers
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {}
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:init():669] starting backend
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:init():673] sending inform_init request
+2024-11-21 14:22:37,040 INFO    MainThread:2298044 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: fork
+2024-11-21 14:22:37,040 INFO    MainThread:2298044 [wandb_init.py:init():686] backend started and connected
+2024-11-21 14:22:37,055 INFO    MainThread:2298044 [wandb_init.py:init():781] updated telemetry
+2024-11-21 14:22:37,058 INFO    MainThread:2298044 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2024-11-21 14:22:37,703 INFO    MainThread:2298044 [wandb_init.py:init():867] starting run threads in backend
+2024-11-21 14:22:38,014 INFO    MainThread:2298044 [wandb_run.py:_console_start():2456] atexit reg
+2024-11-21 14:22:38,015 INFO    MainThread:2298044 [wandb_run.py:_redirect():2305] redirect: wrap_raw
+2024-11-21 14:22:38,015 INFO    MainThread:2298044 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2024-11-21 14:22:38,015 INFO    MainThread:2298044 [wandb_run.py:_redirect():2395] Redirects installed.
+2024-11-21 14:22:38,019 INFO    MainThread:2298044 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-21 14:22:38,019 INFO    MainThread:2298044 [wandb_watch.py:_watch():71] Watching
+2024-11-21 14:22:38,079 INFO    MainThread:2298044 [wandb_run.py:_config_callback():1387] config_cb None None {'data': {'root_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20', 'prop': 'eah', 'num_targets': 1, 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30, 'readout': 'mean', 'max_atoms': 20, 'otf_graph': False, 'eval_model_name': 'alex20', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'train_max_epochs': 6000, 'early_stopping_patience': 60, 'teacher_forcing_max_epoch': 500, 'datamodule': {'_target_': 'diffcsp.pl_data.datamodule.CrystDataModule', 'datasets': {'train': {'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy train', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/train.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/train_ori.pt', 'prop': 'eah', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}, 'val': [{'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy val', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/val.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/val_ori.pt', 'prop': 'eah', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}], 'test': [{'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy test', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/test.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/test_ori.pt', 'prop': 'eah', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}]}, 'num_workers': {'train': 0, 'val': 0, 'test': 0}, 'batch_size': {'train': 4096, 'val': 4096, 'test': 1024}}}, 'logging': {'val_check_interval': 1, 'wandb': {'name': 'alex_20', 'project': 'diffcsp', 'entity': None, 'log_model': True, 'mode': 'online', 'group': 'alex_20'}, 'wandb_watch': {'log': 'all', 'log_freq': 500}, 'lr_monitor': {'logging_interval': 'step', 'log_momentum': False}}, 'model': {'decoder': {'_target_': 'diffcsp.pl_modules.cspnet.CSPNet', 'hidden_dim': 512, 'latent_dim': 0, 'max_atoms': 100, 'num_layers': 6, 'act_fn': 'silu', 'dis_emb': 'sin', 'num_freqs': 128, 'edge_style': 'fc', 'max_neighbors': 20, 'cutoff': 7.0, 'ln': True, 'ip': True}, 'beta_scheduler': {'_target_': 'diffcsp.pl_modules.diff_utils.BetaScheduler', 'timesteps': 1000, 'scheduler_mode': 'cosine'}, 'sigma_scheduler': {'_target_': 'diffcsp.pl_modules.diff_utils.SigmaScheduler', 'timesteps': 1000, 'sigma_begin': 0.005, 'sigma_end': 0.5}, '_target_': 'diffcsp.pl_modules.diffusion_w_type.CSPDiffusion', 'time_dim': 256, 'latent_dim': 0, 'cost_coord': 1.0, 'cost_lattice': 1.0, 'cost_type': 20.0, 'max_neighbors': 20, 'radius': 7.0, 'timesteps': 1000}, 'optim': {'optimizer': {'_target_': 'torch.optim.Adam', 'lr': 0.001, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0}, 'use_lr_scheduler': True, 'lr_scheduler': {'_target_': 'torch.optim.lr_scheduler.ReduceLROnPlateau', 'factor': 0.6, 'patience': 30, 'min_lr': 0.0001}}, 'train': {'deterministic': True, 'random_seed': 42, 'pl_trainer': {'fast_dev_run': False, 'accelerator': 'auto', 'precision': 32, 'max_epochs': 6000, 'accumulate_grad_batches': 1, 'num_sanity_val_steps': 2, 'gradient_clip_val': 0.5, 'gradient_clip_algorithm': 'value', 'profiler': 'simple'}, 'monitor_metric': 'val_loss', 'monitor_metric_mode': 'min', 'early_stopping': {'patience': 60, 'verbose': True}, 'model_checkpoints': {'save_top_k': 1, 'verbose': False, 'save_last': False}, 'ckpt_path': None}, 'expname': 'alex_20', 'core': {'version': '0.0.1', 'tags': ['2024-11-21']}, 'stats/params_total': 12346468, 'stats/params_trainable': 12346468, 'stats/params_not_trainable': 0}
+2024-11-24 04:29:47,970 INFO    MainThread:2298044 [wandb_run.py:_finish():2155] finishing run uf_hennig/diffcsp/scq8411d
+2024-11-24 04:29:48,052 INFO    MainThread:2298044 [wandb_run.py:_atexit_cleanup():2420] got exitcode: 0
+2024-11-24 04:29:48,054 INFO    MainThread:2298044 [wandb_run.py:_restore():2402] restore
+2024-11-24 04:29:48,054 INFO    MainThread:2298044 [wandb_run.py:_restore():2408] restore done
+2024-11-24 04:29:53,834 INFO    MainThread:2298044 [wandb_run.py:_footer_history_summary_info():3960] rendering history
+2024-11-24 04:29:53,836 INFO    MainThread:2298044 [wandb_run.py:_footer_history_summary_info():3992] rendering summary
+2024-11-24 04:29:53,864 INFO    MainThread:2298044 [wandb_run.py:_footer_sync_info():3921] logging synced files

foundation_model/wandb/run-20241121_142236-scq8411d/files/config.yaml

@@ -0,0 +1,2847 @@
+_wandb:
+    value:
+        cli_version: 0.18.7
+        m:
+            - "1": epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.atom_latent_emb.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.node_embedding.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.atom_latent_emb.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": test_lattice_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.final_layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.type_out.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.node_embedding.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.lattice_out.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.final_layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": test_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": val_coord_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.final_layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.atom_latent_emb.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.lattice_out.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.final_layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.final_layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.type_out.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": val_lattice_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.final_layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": type_loss_step
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.type_out.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.type_out.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.type_out.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": test_type_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": type_loss_epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.node_embedding.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.coord_out.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": test_coord_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.final_layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.coord_out.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.atom_latent_emb.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lr-Adam
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.lattice_out.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.atom_latent_emb.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.atom_latent_emb.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": coord_loss_step
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lattice_loss_epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.lattice_out.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.node_embedding.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.final_layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.type_out.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.type_out.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.final_layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.node_embedding.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.coord_out.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.atom_latent_emb.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.atom_latent_emb.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.coord_out.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.atom_latent_emb.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.final_layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.final_layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.coord_out.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.type_out.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.type_out.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.node_embedding.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.node_embedding.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": train_loss_step
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": coord_loss_epoch
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.atom_latent_emb.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.node_embedding.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.lattice_out.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.type_out.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.type_out.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.node_embedding.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": val_type_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.node_mlp.2.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": val_loss
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.final_layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.node_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.lattice_out.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.atom_latent_emb.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.layer_norm.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.edge_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_4.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.edge_mlp.0.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_0.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.0.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_3.node_mlp.2.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.node_mlp.2.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.layer_norm.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.node_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_0.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.node_embedding.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.0.weight.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.edge_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.edge_mlp.2.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_2.layer_norm.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.type_out.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_3.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": lattice_loss_step
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.coord_out.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_5.node_mlp.2.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.edge_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.0.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.node_mlp.0.bias.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.node_embedding.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.atom_latent_emb.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_2.edge_mlp.0.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_1.node_mlp.2.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.node_embedding.weight._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": parameters/decoder.csp_layer_4.layer_norm.weight.values
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_1.layer_norm.bias.bins
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+            - "1": gradients/decoder.csp_layer_5.layer_norm.bias._type
+              "5": 2
+              "6":
+                - 1
+                - 3
+              "7": []
+        python_version: 3.12.2
+        t:
+            "1":
+                - 1
+                - 5
+                - 9
+                - 50
+                - 53
+                - 55
+                - 77
+                - 103
+            "2":
+                - 1
+                - 5
+                - 9
+                - 50
+                - 53
+                - 55
+                - 77
+                - 103
+            "3":
+                - 1
+                - 2
+                - 7
+                - 13
+                - 15
+                - 23
+                - 55
+                - 66
+            "4": 3.12.2
+            "5": 0.18.7
+            "8":
+                - 6
+            "12": 0.18.7
+            "13": linux-x86_64
+core:
+    value:
+        tags:
+            - "2024-11-21"
+        version: 0.0.1
+data:
+    value:
+        datamodule:
+            _target_: diffcsp.pl_data.datamodule.CrystDataModule
+            batch_size:
+                test: 1024
+                train: 4096
+                val: 4096
+            datasets:
+                test:
+                    - _target_: diffcsp.pl_data.dataset.CrystDataset
+                      graph_method: crystalnn
+                      lattice_scale_method: scale_length
+                      name: Formation energy test
+                      niggli: true
+                      path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/test.csv
+                      preprocess_workers: 30
+                      primitive: false
+                      prop: eah
+                      save_path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/test_ori.pt
+                      tolerance: 0.1
+                      use_pos_index: false
+                      use_space_group: false
+                train:
+                    _target_: diffcsp.pl_data.dataset.CrystDataset
+                    graph_method: crystalnn
+                    lattice_scale_method: scale_length
+                    name: Formation energy train
+                    niggli: true
+                    path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/train.csv
+                    preprocess_workers: 30
+                    primitive: false
+                    prop: eah
+                    save_path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/train_ori.pt
+                    tolerance: 0.1
+                    use_pos_index: false
+                    use_space_group: false
+                val:
+                    - _target_: diffcsp.pl_data.dataset.CrystDataset
+                      graph_method: crystalnn
+                      lattice_scale_method: scale_length
+                      name: Formation energy val
+                      niggli: true
+                      path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/val.csv
+                      preprocess_workers: 30
+                      primitive: false
+                      prop: eah
+                      save_path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/val_ori.pt
+                      tolerance: 0.1
+                      use_pos_index: false
+                      use_space_group: false
+            num_workers:
+                test: 0
+                train: 0
+                val: 0
+        early_stopping_patience: 60
+        eval_model_name: alex20
+        graph_method: crystalnn
+        lattice_scale_method: scale_length
+        max_atoms: 20
+        niggli: true
+        num_targets: 1
+        otf_graph: false
+        preprocess_workers: 30
+        primitive: false
+        prop: eah
+        readout: mean
+        root_path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20
+        teacher_forcing_max_epoch: 500
+        tolerance: 0.1
+        train_max_epochs: 6000
+        use_pos_index: false
+        use_space_group: false
+expname:
+    value: alex_20
+logging:
+    value:
+        lr_monitor:
+            log_momentum: false
+            logging_interval: step
+        val_check_interval: 1
+        wandb:
+            entity: null
+            group: alex_20
+            log_model: true
+            mode: online
+            name: alex_20
+            project: diffcsp
+        wandb_watch:
+            log: all
+            log_freq: 500
+model:
+    value:
+        _target_: diffcsp.pl_modules.diffusion_w_type.CSPDiffusion
+        beta_scheduler:
+            _target_: diffcsp.pl_modules.diff_utils.BetaScheduler
+            scheduler_mode: cosine
+            timesteps: 1000
+        cost_coord: 1
+        cost_lattice: 1
+        cost_type: 20
+        decoder:
+            _target_: diffcsp.pl_modules.cspnet.CSPNet
+            act_fn: silu
+            cutoff: 7
+            dis_emb: sin
+            edge_style: fc
+            hidden_dim: 512
+            ip: true
+            latent_dim: 0
+            ln: true
+            max_atoms: 100
+            max_neighbors: 20
+            num_freqs: 128
+            num_layers: 6
+        latent_dim: 0
+        max_neighbors: 20
+        radius: 7
+        sigma_scheduler:
+            _target_: diffcsp.pl_modules.diff_utils.SigmaScheduler
+            sigma_begin: 0.005
+            sigma_end: 0.5
+            timesteps: 1000
+        time_dim: 256
+        timesteps: 1000
+optim:
+    value:
+        lr_scheduler:
+            _target_: torch.optim.lr_scheduler.ReduceLROnPlateau
+            factor: 0.6
+            min_lr: 0.0001
+            patience: 30
+        optimizer:
+            _target_: torch.optim.Adam
+            betas:
+                - 0.9
+                - 0.999
+            eps: 1e-08
+            lr: 0.001
+            weight_decay: 0
+        use_lr_scheduler: true
+stats/params_not_trainable:
+    value: 0
+stats/params_total:
+    value: 12346468
+stats/params_trainable:
+    value: 12346468
+train:
+    value:
+        ckpt_path: null
+        deterministic: true
+        early_stopping:
+            patience: 60
+            verbose: true
+        model_checkpoints:
+            save_last: false
+            save_top_k: 1
+            verbose: false
+        monitor_metric: val_loss
+        monitor_metric_mode: min
+        pl_trainer:
+            accelerator: auto
+            accumulate_grad_batches: 1
+            fast_dev_run: false
+            gradient_clip_algorithm: value
+            gradient_clip_val: 0.5
+            max_epochs: 6000
+            num_sanity_val_steps: 2
+            precision: 32
+            profiler: simple
+        random_seed: 42

foundation_model/wandb/run-20241121_142236-scq8411d/files/requirements.txt

@@ -0,0 +1,107 @@
+matplotlib==3.9.2
+aiohttp==3.11.6
+fonttools==4.55.0
+importlib_metadata==8.5.0
+pyocse==0.1.2
+aiosignal==1.3.1
+urllib3==2.2.3
+pip==24.2
+vasprun-xml==1.0.4
+filelock==3.16.1
+xmltodict==0.14.2
+spglib==2.5.0
+palettable==3.3.3
+torchmetrics==1.6.0
+platformdirs==4.3.6
+omegaconf==2.3.0
+setuptools==75.1.0
+scikit-learn==1.5.2
+mpmath==1.3.0
+tqdm==4.67.0
+charset-normalizer==3.4.0
+certifi==2024.8.30
+multiprocess==0.70.17
+six==1.16.0
+torch==2.3.0
+aiohappyeyeballs==2.4.3
+latexcodec==3.0.0
+nvidia-cusolver-cu12==11.4.5.107
+tenacity==9.0.0
+dill==0.3.9
+uncertainties==3.2.2
+MarkupSafe==3.0.2
+wandb==0.18.7
+multidict==6.1.0
+nvidia-cublas-cu12==12.1.3.1
+ppft==1.7.6.9
+requests==2.32.3
+smmap==5.0.1
+numpy==1.26.4
+ruamel.yaml==0.18.6
+idna==3.10
+networkx==3.4.2
+einops==0.8.0
+pymatgen==2024.11.13
+torch_sparse==0.6.18+pt23cu121
+python-dotenv==1.0.1
+plotly==5.24.1
+attrs==24.2.0
+click==8.1.7
+pandas==2.2.3
+pillow==11.0.0
+docker-pycreds==0.4.0
+wheel==0.44.0
+typing_extensions==4.12.2
+lightning-utilities==0.11.9
+pathos==0.3.3
+ruamel.yaml.clib==0.2.12
+pybtex==0.24.0
+nvidia-cuda-nvrtc-cu12==12.1.105
+sentry-sdk==2.18.0
+tzdata==2024.2
+pyparsing==3.2.0
+lxml==5.3.0
+cycler==0.12.1
+tabulate==0.9.0
+toml==0.10.2
+sympy==1.13.3
+yarl==1.17.2
+threadpoolctl==3.5.0
+nvidia-cuda-runtime-cu12==12.1.105
+pytorch-lightning==2.4.0
+nvidia-cusparse-cu12==12.1.0.106
+diffcsp==0.0.1
+psutil==6.1.0
+kiwisolver==1.4.7
+torch-geometric==2.6.1
+GitPython==3.1.43
+gitdb==4.0.11
+PyYAML==6.0.2
+pyxtal==1.0.5
+hydra-core==1.3.2
+p_tqdm==1.4.2
+nvidia-curand-cu12==10.3.2.106
+nvidia-nccl-cu12==2.20.5
+contourpy==1.3.1
+torch_scatter==2.1.2+pt23cu121
+monty==2024.10.21
+setproctitle==1.3.4
+pox==0.3.5
+joblib==1.4.2
+scipy==1.14.1
+packaging==24.2
+nvidia-cufft-cu12==11.0.2.54
+antlr4-python3-runtime==4.9.3
+ase==3.23.0
+protobuf==5.28.3
+python-dateutil==2.9.0.post0
+propcache==0.2.0
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-nvjitlink-cu12==12.6.85
+Jinja2==3.1.4
+nvidia-nvtx-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+fsspec==2024.10.0
+frozenlist==1.5.0
+zipp==3.21.0
+pytz==2024.2

foundation_model/wandb/run-20241121_142236-scq8411d/files/wandb-metadata.json

@@ -0,0 +1,125 @@
+{
+  "os": "Linux-4.18.0-553.27.1.el8_10.x86_64-x86_64-with-glibc2.28",
+  "python": "3.12.2",
+  "startedAt": "2024-11-21T19:22:37.041057Z",
+  "args": [
+    "data=alex_20",
+    "model=diffusion_w_type",
+    "expname=alex_20"
+  ],
+  "program": "/blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/alex_20/./../../MaterialsDiffusion_unguided/diffcsp/run.py",
+  "email": "pawanprakash@ufl.edu",
+  "root": ".",
+  "host": "c1104a-s29.ufhpc",
+  "username": "pawanprakash",
+  "executable": "/home/pawanprakash/miniconda3/envs/diffcsp_ung/bin/python",
+  "cpu_count": 128,
+  "cpu_count_logical": 128,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "41922297856",
+      "used": "13082791936"
+    }
+  },
+  "memory": {
+    "total": "2159750873088"
+  },
+  "cpu": {
+    "count": 128,
+    "countLogical": 128
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    }
+  ],
+  "slurm": {
+    "cluster_name": "hipergator",
+    "conf": "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node": "1",
+    "cpus_per_task": "1",
+    "gpus": "a100:4",
+    "gpus_on_node": "4",
+    "gtids": "0",
+    "job_account": "hennig",
+    "job_cpus_per_node": "1",
+    "job_gid": "3424",
+    "job_gpus": "0,1,6,7",
+    "job_id": "50539382",
+    "job_name": "alex_20",
+    "job_nodelist": "c1104a-s29",
+    "job_num_nodes": "1",
+    "job_partition": "gpu",
+    "job_qos": "hennig qos",
+    "job_uid": "8611",
+    "job_user": "pawanprakash",
+    "jobid": "50539382",
+    "localid": "0",
+    "mem_per_cpu": "8192",
+    "mem_per_node": "131072",
+    "nnodes": "1",
+    "node_aliases": "(null)",
+    "nodeid": "0",
+    "nodelist": "c1104a-s29",
+    "nprocs": "1",
+    "ntasks": "1",
+    "prio_process": "0",
+    "procid": "0",
+    "submit_dir": "/blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/alex_20",
+    "submit_host": "c1010a-s23.ufhpc",
+    "task_pid": "2297983",
+    "tasks_per_node": "1",
+    "tmpdir": "/scratch/local/50539382",
+    "topology_addr": ".ait1-1.ibsw1102.c1104a-s29",
+    "topology_addr_pattern": "switch.switch.switch.node",
+    "working_cluster": "hipergator:vslurm:6817:9472:109"
+  },
+  "cudaVersion": "12.4"
+}

foundation_model/wandb/run-20241121_142236-scq8411d/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-21T14:22:36.615465181-05:00","level":"INFO","msg":"started logging, with flags","port-filename":"/scratch/local/50539382/tmp7kjxf86w/port-2298044.txt","pid":2298044,"debug":false,"disable-analytics":false}
+{"time":"2024-11-21T14:22:36.615498774-05:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-11-21T14:22:36.616276906-05:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":45597,"Zone":""}}
+{"time":"2024-11-21T14:22:36.616372026-05:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2298044}
+{"time":"2024-11-21T14:22:36.810345903-05:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:58730"}
+{"time":"2024-11-21T14:22:37.04007684-05:00","level":"INFO","msg":"handleInformInit: received","streamId":"scq8411d","id":"127.0.0.1:58730"}
+{"time":"2024-11-21T14:22:37.344372319-05:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"scq8411d","id":"127.0.0.1:58730"}
+{"time":"2024-11-24T04:29:53.867083837-05:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"scq8411d","id":"127.0.0.1:58730"}
+{"time":"2024-11-24T04:29:53.908842239-05:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"scq8411d","id":"127.0.0.1:58730"}
+{"time":"2024-11-24T04:29:55.944654108-05:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:58730"}
+{"time":"2024-11-24T04:29:55.998024178-05:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-11-24T04:29:55.998049156-05:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:58730"}
+{"time":"2024-11-24T04:29:55.964759195-05:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:58730"}
+{"time":"2024-11-24T04:29:55.998480001-05:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:58730"}
+{"time":"2024-11-24T04:29:56.023147789-05:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:58730"}
+{"time":"2024-11-24T04:29:56.023564308-05:00","level":"INFO","msg":"server is closed"}

foundation_model/wandb/run-20241121_142236-scq8411d/logs/debug-internal.log

@@ -0,0 +1,28 @@
+{"time":"2024-11-21T14:22:37.043045747-05:00","level":"INFO","msg":"using version","core version":"0.18.7"}
+{"time":"2024-11-21T14:22:37.066680433-05:00","level":"INFO","msg":"created symlink","path":"wandb/run-20241121_142236-scq8411d/logs/debug-core.log"}
+{"time":"2024-11-21T14:22:37.344284532-05:00","level":"INFO","msg":"created new stream","id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.344361949-05:00","level":"INFO","msg":"stream: started","id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.344614507-05:00","level":"INFO","msg":"handler: started","stream_id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.344662057-05:00","level":"INFO","msg":"writer: Do: started","stream_id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.344814565-05:00","level":"INFO","msg":"sender: started","stream_id":"scq8411d"}
+{"time":"2024-11-21T14:22:37.708316569-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-21T15:55:28.886692431-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2024-11-21T18:10:31.807981434-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-21T22:26:49.683292877-05:00","level":"INFO","msg":"api: retrying HTTP error","status":502,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-22T03:53:28.60635499-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-22T18:45:11.801789802-05:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded"}
+{"time":"2024-11-23T23:42:04.677295397-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-23T23:42:22.690477214-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-23T23:42:32.589851211-05:00","level":"INFO","msg":"api: retrying HTTP error","status":500,"url":"https://api.wandb.ai/files/uf_hennig/diffcsp/scq8411d/file_stream"}
+{"time":"2024-11-24T04:29:48.5176947-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-24T04:29:49.683102181-05:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-24T04:29:49.687112558-05:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2024-11-24T04:29:52.034709299-05:00","level":"WARN","msg":"No job ingredients found, not creating job artifact"}
+{"time":"2024-11-24T04:29:52.039988286-05:00","level":"WARN","msg":"No source type found, not creating job artifact"}
+{"time":"2024-11-24T04:29:52.039996592-05:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-11-24T04:29:52.406602038-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-24T04:29:53.893798488-05:00","level":"INFO","msg":"stream: closing","id":"scq8411d"}
+{"time":"2024-11-24T04:29:53.893822644-05:00","level":"INFO","msg":"handler: closed","stream_id":"scq8411d"}
+{"time":"2024-11-24T04:29:53.89384157-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":"scq8411d"}
+{"time":"2024-11-24T04:29:53.908810589-05:00","level":"INFO","msg":"sender: closed","stream_id":"scq8411d"}
+{"time":"2024-11-24T04:29:53.908834234-05:00","level":"INFO","msg":"stream: closed","id":"scq8411d"}

foundation_model/wandb/run-20241121_142236-scq8411d/logs/debug.log

@@ -0,0 +1,35 @@
+2024-11-21 14:22:37,025 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
+2024-11-21 14:22:37,025 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Configure stats pid to 2298044
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Loading settings from /home/pawanprakash/.config/wandb/settings
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Loading settings from /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/wandb/settings
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'start_method': 'thread', 'root_dir': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/wandb'}
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2024-11-21 14:22:37,026 WARNING MainThread:2298044 [wandb_setup.py:_flush():79] Could not save program above cwd: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/alex_20/./../../MaterialsDiffusion_unguided/diffcsp/run.py
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program_abspath': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/diffcsp/run.py', 'program': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/alex_20/./../../MaterialsDiffusion_unguided/diffcsp/run.py'}
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-21 14:22:37,026 INFO    MainThread:2298044 [wandb_init.py:_log_setup():533] Logging user logs to ./wandb/run-20241121_142236-scq8411d/logs/debug.log
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:_log_setup():534] Logging internal logs to ./wandb/run-20241121_142236-scq8411d/logs/debug-internal.log
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:init():619] calling init triggers
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {}
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:init():669] starting backend
+2024-11-21 14:22:37,027 INFO    MainThread:2298044 [wandb_init.py:init():673] sending inform_init request
+2024-11-21 14:22:37,040 INFO    MainThread:2298044 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: fork
+2024-11-21 14:22:37,040 INFO    MainThread:2298044 [wandb_init.py:init():686] backend started and connected
+2024-11-21 14:22:37,055 INFO    MainThread:2298044 [wandb_init.py:init():781] updated telemetry
+2024-11-21 14:22:37,058 INFO    MainThread:2298044 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2024-11-21 14:22:37,703 INFO    MainThread:2298044 [wandb_init.py:init():867] starting run threads in backend
+2024-11-21 14:22:38,014 INFO    MainThread:2298044 [wandb_run.py:_console_start():2456] atexit reg
+2024-11-21 14:22:38,015 INFO    MainThread:2298044 [wandb_run.py:_redirect():2305] redirect: wrap_raw
+2024-11-21 14:22:38,015 INFO    MainThread:2298044 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2024-11-21 14:22:38,015 INFO    MainThread:2298044 [wandb_run.py:_redirect():2395] Redirects installed.
+2024-11-21 14:22:38,019 INFO    MainThread:2298044 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-21 14:22:38,019 INFO    MainThread:2298044 [wandb_watch.py:_watch():71] Watching
+2024-11-21 14:22:38,079 INFO    MainThread:2298044 [wandb_run.py:_config_callback():1387] config_cb None None {'data': {'root_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20', 'prop': 'eah', 'num_targets': 1, 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30, 'readout': 'mean', 'max_atoms': 20, 'otf_graph': False, 'eval_model_name': 'alex20', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'train_max_epochs': 6000, 'early_stopping_patience': 60, 'teacher_forcing_max_epoch': 500, 'datamodule': {'_target_': 'diffcsp.pl_data.datamodule.CrystDataModule', 'datasets': {'train': {'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy train', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/train.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/train_ori.pt', 'prop': 'eah', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}, 'val': [{'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy val', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/val.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/val_ori.pt', 'prop': 'eah', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}], 'test': [{'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy test', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/test.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion_unguided/data/alex_20/test_ori.pt', 'prop': 'eah', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}]}, 'num_workers': {'train': 0, 'val': 0, 'test': 0}, 'batch_size': {'train': 4096, 'val': 4096, 'test': 1024}}}, 'logging': {'val_check_interval': 1, 'wandb': {'name': 'alex_20', 'project': 'diffcsp', 'entity': None, 'log_model': True, 'mode': 'online', 'group': 'alex_20'}, 'wandb_watch': {'log': 'all', 'log_freq': 500}, 'lr_monitor': {'logging_interval': 'step', 'log_momentum': False}}, 'model': {'decoder': {'_target_': 'diffcsp.pl_modules.cspnet.CSPNet', 'hidden_dim': 512, 'latent_dim': 0, 'max_atoms': 100, 'num_layers': 6, 'act_fn': 'silu', 'dis_emb': 'sin', 'num_freqs': 128, 'edge_style': 'fc', 'max_neighbors': 20, 'cutoff': 7.0, 'ln': True, 'ip': True}, 'beta_scheduler': {'_target_': 'diffcsp.pl_modules.diff_utils.BetaScheduler', 'timesteps': 1000, 'scheduler_mode': 'cosine'}, 'sigma_scheduler': {'_target_': 'diffcsp.pl_modules.diff_utils.SigmaScheduler', 'timesteps': 1000, 'sigma_begin': 0.005, 'sigma_end': 0.5}, '_target_': 'diffcsp.pl_modules.diffusion_w_type.CSPDiffusion', 'time_dim': 256, 'latent_dim': 0, 'cost_coord': 1.0, 'cost_lattice': 1.0, 'cost_type': 20.0, 'max_neighbors': 20, 'radius': 7.0, 'timesteps': 1000}, 'optim': {'optimizer': {'_target_': 'torch.optim.Adam', 'lr': 0.001, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0}, 'use_lr_scheduler': True, 'lr_scheduler': {'_target_': 'torch.optim.lr_scheduler.ReduceLROnPlateau', 'factor': 0.6, 'patience': 30, 'min_lr': 0.0001}}, 'train': {'deterministic': True, 'random_seed': 42, 'pl_trainer': {'fast_dev_run': False, 'accelerator': 'auto', 'precision': 32, 'max_epochs': 6000, 'accumulate_grad_batches': 1, 'num_sanity_val_steps': 2, 'gradient_clip_val': 0.5, 'gradient_clip_algorithm': 'value', 'profiler': 'simple'}, 'monitor_metric': 'val_loss', 'monitor_metric_mode': 'min', 'early_stopping': {'patience': 60, 'verbose': True}, 'model_checkpoints': {'save_top_k': 1, 'verbose': False, 'save_last': False}, 'ckpt_path': None}, 'expname': 'alex_20', 'core': {'version': '0.0.1', 'tags': ['2024-11-21']}, 'stats/params_total': 12346468, 'stats/params_trainable': 12346468, 'stats/params_not_trainable': 0}
+2024-11-24 04:29:47,970 INFO    MainThread:2298044 [wandb_run.py:_finish():2155] finishing run uf_hennig/diffcsp/scq8411d
+2024-11-24 04:29:48,052 INFO    MainThread:2298044 [wandb_run.py:_atexit_cleanup():2420] got exitcode: 0
+2024-11-24 04:29:48,054 INFO    MainThread:2298044 [wandb_run.py:_restore():2402] restore
+2024-11-24 04:29:48,054 INFO    MainThread:2298044 [wandb_run.py:_restore():2408] restore done
+2024-11-24 04:29:53,834 INFO    MainThread:2298044 [wandb_run.py:_footer_history_summary_info():3960] rendering history
+2024-11-24 04:29:53,836 INFO    MainThread:2298044 [wandb_run.py:_footer_history_summary_info():3992] rendering summary
+2024-11-24 04:29:53,864 INFO    MainThread:2298044 [wandb_run.py:_footer_sync_info():3921] logging synced files

foundation_model/wandb/run-20241121_142236-scq8411d/run-scq8411d.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:119c2d1791c7bc49bcd49b48cea0c38ed26351ebf452f5e3038de6a8eba0b089
+size 415616899

superconductor_generator/.hydra/config.yaml

@@ -0,0 +1,167 @@
+data:
+  root_path: ${oc.env:PROJECT_ROOT}/data/supccomb_12
+  prop: tcad
+  num_targets: 1
+  niggli: true
+  primitive: false
+  graph_method: crystalnn
+  lattice_scale_method: scale_length
+  preprocess_workers: 30
+  readout: mean
+  max_atoms: 12
+  otf_graph: false
+  eval_model_name: supccomb12
+  tolerance: 0.1
+  use_space_group: false
+  use_pos_index: false
+  train_max_epochs: 1500
+  early_stopping_patience: 200
+  teacher_forcing_max_epoch: 500
+  datamodule:
+    _target_: diffcsp.pl_data.datamodule.CrystDataModule
+    datasets:
+      train:
+        _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy train
+        path: ${data.root_path}/train.csv
+        save_path: ${data.root_path}/train_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+      val:
+      - _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy val
+        path: ${data.root_path}/val.csv
+        save_path: ${data.root_path}/val_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+      test:
+      - _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy test
+        path: ${data.root_path}/test.csv
+        save_path: ${data.root_path}/test_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+    num_workers:
+      train: 0
+      val: 0
+      test: 0
+    batch_size:
+      train: 256
+      val: 128
+      test: 128
+logging:
+  val_check_interval: 1
+  wandb:
+    name: ${expname}
+    project: diffcsp
+    entity: null
+    log_model: true
+    mode: online
+    group: ${expname}
+  wandb_watch:
+    log: all
+    log_freq: 500
+  lr_monitor:
+    logging_interval: step
+    log_momentum: false
+model:
+  decoder:
+    _target_: diffcsp.pl_modules.cspnet.CSPNet
+    hidden_dim: 512
+    latent_dim: 0
+    max_atoms: 100
+    num_layers: 6
+    act_fn: silu
+    dis_emb: sin
+    num_freqs: 128
+    edge_style: fc
+    max_neighbors: ${model.max_neighbors}
+    cutoff: ${model.radius}
+    ln: true
+    ip: true
+    prop_embed_dim: 64
+    am_hidden_dim: 128
+  beta_scheduler:
+    _target_: diffcsp.pl_modules.diff_utils.BetaScheduler
+    timesteps: ${model.timesteps}
+    scheduler_mode: cosine
+  sigma_scheduler:
+    _target_: diffcsp.pl_modules.diff_utils.SigmaScheduler
+    timesteps: ${model.timesteps}
+    sigma_begin: 0.005
+    sigma_end: 0.5
+  _target_: diffcsp.pl_modules.diffusion_w_type.CSPDiffusion
+  time_dim: 256
+  latent_dim: 0
+  cost_coord: 1.0
+  cost_lattice: 1.0
+  cost_type: 20.0
+  max_neighbors: 20
+  radius: 7.0
+  timesteps: 1000
+  p_uncond: 0.2
+  guide_w: 3.0
+optim:
+  optimizer:
+    _target_: torch.optim.Adam
+    lr: 0.001
+    betas:
+    - 0.9
+    - 0.999
+    eps: 1.0e-08
+    weight_decay: 0
+  use_lr_scheduler: true
+  lr_scheduler:
+    _target_: torch.optim.lr_scheduler.ReduceLROnPlateau
+    factor: 0.6
+    patience: 30
+    min_lr: 0.0001
+train:
+  deterministic: true
+  random_seed: 42
+  pl_trainer:
+    fast_dev_run: false
+    accelerator: auto
+    precision: 32
+    max_epochs: ${data.train_max_epochs}
+    accumulate_grad_batches: 1
+    num_sanity_val_steps: 2
+    gradient_clip_val: 0.5
+    gradient_clip_algorithm: value
+    profiler: simple
+  monitor_metric: val_loss
+  monitor_metric_mode: min
+  early_stopping:
+    patience: ${data.early_stopping_patience}
+    verbose: true
+  model_checkpoints:
+    save_top_k: 1
+    verbose: false
+    save_last: false
+  ckpt_path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch=656-step=74898.ckpt
+expname: finetune_alex20_sccomb
+core:
+  version: 0.0.1
+  tags:
+  - ${now:%Y-%m-%d}

superconductor_generator/.hydra/hydra.yaml

@@ -0,0 +1,168 @@
+hydra:
+  run:
+    dir: ${oc.env:HYDRA_JOBS}/singlerun/${now:%Y-%m-%d}/${expname}/
+  sweep:
+    dir: ${oc.env:HYDRA_JOBS}/multirun/${now:%Y-%m-%d}/${expname}/
+    subdir: ${hydra.job.num}_${hydra.job.id}
+  launcher:
+    _target_: hydra._internal.core_plugins.basic_launcher.BasicLauncher
+  sweeper:
+    _target_: hydra._internal.core_plugins.basic_sweeper.BasicSweeper
+    max_batch_size: null
+    params: null
+  help:
+    app_name: ${hydra.job.name}
+    header: '${hydra.help.app_name} is powered by Hydra.
+
+      '
+    footer: 'Powered by Hydra (https://hydra.cc)
+
+      Use --hydra-help to view Hydra specific help
+
+      '
+    template: '${hydra.help.header}
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (group=option)
+
+
+      $APP_CONFIG_GROUPS
+
+
+      == Config ==
+
+      Override anything in the config (foo.bar=value)
+
+
+      $CONFIG
+
+
+      ${hydra.help.footer}
+
+      '
+  hydra_help:
+    template: 'Hydra (${hydra.runtime.version})
+
+      See https://hydra.cc for more info.
+
+
+      == Flags ==
+
+      $FLAGS_HELP
+
+
+      == Configuration groups ==
+
+      Compose your configuration from those groups (For example, append hydra/job_logging=disabled
+      to command line)
+
+
+      $HYDRA_CONFIG_GROUPS
+
+
+      Use ''--cfg hydra'' to Show the Hydra config.
+
+      '
+    hydra_help: ???
+  hydra_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][HYDRA] %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+    root:
+      level: INFO
+      handlers:
+      - console
+    loggers:
+      logging_example:
+        level: DEBUG
+    disable_existing_loggers: false
+  job_logging:
+    version: 1
+    formatters:
+      simple:
+        format: '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
+    handlers:
+      console:
+        class: logging.StreamHandler
+        formatter: simple
+        stream: ext://sys.stdout
+      file:
+        class: logging.FileHandler
+        formatter: simple
+        filename: ${hydra.runtime.output_dir}/${hydra.job.name}.log
+    root:
+      level: INFO
+      handlers:
+      - console
+      - file
+    disable_existing_loggers: false
+  env: {}
+  mode: RUN
+  searchpath: []
+  callbacks: {}
+  output_subdir: .hydra
+  overrides:
+    hydra:
+    - hydra.mode=RUN
+    task:
+    - data=supccomb_12
+    - model=diffusion_w_type
+    - expname=finetune_alex20_sccomb
+    - train.ckpt_path=/blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch\=656-step\=74898.ckpt
+  job:
+    name: run
+    chdir: null
+    override_dirname: data=supccomb_12,expname=finetune_alex20_sccomb,model=diffusion_w_type,train.ckpt_path=/blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch\=656-step\=74898.ckpt
+    id: ???
+    num: ???
+    config_name: default
+    env_set:
+      WANDB_START_METHOD: thread
+      WANDB_DIR: ${oc.env:WABDB_DIR}
+    env_copy: []
+    config:
+      override_dirname:
+        kv_sep: '='
+        item_sep: ','
+        exclude_keys: []
+  runtime:
+    version: 1.3.2
+    version_base: '1.1'
+    cwd: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion
+    config_sources:
+    - path: hydra.conf
+      schema: pkg
+      provider: hydra
+    - path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/conf
+      schema: file
+      provider: main
+    - path: ''
+      schema: structured
+      provider: schema
+    output_dir: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-26/finetune_alex20_sccomb
+    choices:
+      train: default
+      optim: default
+      model: diffusion_w_type
+      model/sigma_scheduler: wrapped
+      model/beta_scheduler: cosine
+      model/decoder: cspnet
+      logging: default
+      data: supccomb_12
+      hydra/env: default
+      hydra/callbacks: null
+      hydra/job_logging: default
+      hydra/hydra_logging: default
+      hydra/hydra_help: default
+      hydra/help: default
+      hydra/sweeper: basic
+      hydra/launcher: basic
+      hydra/output: default
+  verbose: false

superconductor_generator/.hydra/overrides.yaml

@@ -0,0 +1,4 @@
+- data=supccomb_12
+- model=diffusion_w_type
+- expname=finetune_alex20_sccomb
+- train.ckpt_path=/blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch\=656-step\=74898.ckpt

superconductor_generator/epoch=368-step=8487.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7059fa18460776e70ea362a92e480f562a42ae3e1a97ba7510d64e0d18a5353b
+size 172561794

superconductor_generator/eval_gen.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d4d49ec7add1b9b9a9f298a1b846dad688c53b6d44ded13d76793ad1820735b2
+size 205629397

superconductor_generator/hparams.yaml

@@ -0,0 +1,167 @@
+data:
+  root_path: ${oc.env:PROJECT_ROOT}/data/supccomb_12
+  prop: tcad
+  num_targets: 1
+  niggli: true
+  primitive: false
+  graph_method: crystalnn
+  lattice_scale_method: scale_length
+  preprocess_workers: 30
+  readout: mean
+  max_atoms: 12
+  otf_graph: false
+  eval_model_name: supccomb12
+  tolerance: 0.1
+  use_space_group: false
+  use_pos_index: false
+  train_max_epochs: 1500
+  early_stopping_patience: 200
+  teacher_forcing_max_epoch: 500
+  datamodule:
+    _target_: diffcsp.pl_data.datamodule.CrystDataModule
+    datasets:
+      train:
+        _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy train
+        path: ${data.root_path}/train.csv
+        save_path: ${data.root_path}/train_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+      val:
+      - _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy val
+        path: ${data.root_path}/val.csv
+        save_path: ${data.root_path}/val_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+      test:
+      - _target_: diffcsp.pl_data.dataset.CrystDataset
+        name: Formation energy test
+        path: ${data.root_path}/test.csv
+        save_path: ${data.root_path}/test_ori.pt
+        prop: ${data.prop}
+        niggli: ${data.niggli}
+        primitive: ${data.primitive}
+        graph_method: ${data.graph_method}
+        tolerance: ${data.tolerance}
+        use_space_group: ${data.use_space_group}
+        use_pos_index: ${data.use_pos_index}
+        lattice_scale_method: ${data.lattice_scale_method}
+        preprocess_workers: ${data.preprocess_workers}
+    num_workers:
+      train: 0
+      val: 0
+      test: 0
+    batch_size:
+      train: 256
+      val: 128
+      test: 128
+logging:
+  val_check_interval: 1
+  wandb:
+    name: ${expname}
+    project: diffcsp
+    entity: null
+    log_model: true
+    mode: online
+    group: ${expname}
+  wandb_watch:
+    log: all
+    log_freq: 500
+  lr_monitor:
+    logging_interval: step
+    log_momentum: false
+model:
+  decoder:
+    _target_: diffcsp.pl_modules.cspnet.CSPNet
+    hidden_dim: 512
+    latent_dim: 0
+    max_atoms: 100
+    num_layers: 6
+    act_fn: silu
+    dis_emb: sin
+    num_freqs: 128
+    edge_style: fc
+    max_neighbors: ${model.max_neighbors}
+    cutoff: ${model.radius}
+    ln: true
+    ip: true
+    prop_embed_dim: 64
+    am_hidden_dim: 128
+  beta_scheduler:
+    _target_: diffcsp.pl_modules.diff_utils.BetaScheduler
+    timesteps: ${model.timesteps}
+    scheduler_mode: cosine
+  sigma_scheduler:
+    _target_: diffcsp.pl_modules.diff_utils.SigmaScheduler
+    timesteps: ${model.timesteps}
+    sigma_begin: 0.005
+    sigma_end: 0.5
+  _target_: diffcsp.pl_modules.diffusion_w_type.CSPDiffusion
+  time_dim: 256
+  latent_dim: 0
+  cost_coord: 1.0
+  cost_lattice: 1.0
+  cost_type: 20.0
+  max_neighbors: 20
+  radius: 7.0
+  timesteps: 1000
+  p_uncond: 0.2
+  #guide_w: 1.0
+optim:
+  optimizer:
+    _target_: torch.optim.Adam
+    lr: 0.001
+    betas:
+    - 0.9
+    - 0.999
+    eps: 1.0e-08
+    weight_decay: 0
+  use_lr_scheduler: true
+  lr_scheduler:
+    _target_: torch.optim.lr_scheduler.ReduceLROnPlateau
+    factor: 0.6
+    patience: 30
+    min_lr: 0.0001
+train:
+  deterministic: true
+  random_seed: 42
+  pl_trainer:
+    fast_dev_run: false
+    accelerator: auto
+    precision: 32
+    max_epochs: ${data.train_max_epochs}
+    accumulate_grad_batches: 1
+    num_sanity_val_steps: 2
+    gradient_clip_val: 0.5
+    gradient_clip_algorithm: value
+    profiler: simple
+  monitor_metric: val_loss
+  monitor_metric_mode: min
+  early_stopping:
+    patience: ${data.early_stopping_patience}
+    verbose: true
+  model_checkpoints:
+    save_top_k: 1
+    verbose: false
+    save_last: false
+  ckpt_path: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch=656-step=74898.ckpt
+expname: finetune_alex20_sccomb
+core:
+  version: 0.0.1
+  tags:
+  - ${now:%Y-%m-%d}

superconductor_generator/lattice_scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85c4d50211f84f3dae91c5df73af53492a5d89a0f1f17f0c6d995ee3ce9fb686
+size 1538

superconductor_generator/prop_scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7748d9a40fa26289f3539d09abc2e5c4d9c38ef1c70b88b308f76a6774dd9654
+size 1520

superconductor_generator/run.log

@@ -0,0 +1,16 @@
+[2024-11-26 22:22:59,693][hydra.utils][INFO] - Instantiating <diffcsp.pl_data.datamodule.CrystDataModule>
+[2024-11-26 22:24:10,067][hydra.utils][INFO] - Instantiating <diffcsp.pl_modules.diffusion_w_type.CSPDiffusion>
+[2024-11-26 22:24:16,487][hydra.utils][INFO] - Model initialized with parameters: ['decoder.node_embedding.weight', 'decoder.node_embedding.bias', 'decoder.atom_latent_emb.weight', 'decoder.atom_latent_emb.bias', 'decoder.csp_layer_0.edge_mlp.0.weight', 'decoder.csp_layer_0.edge_mlp.0.bias', 'decoder.csp_layer_0.edge_mlp.2.weight', 'decoder.csp_layer_0.edge_mlp.2.bias', 'decoder.csp_layer_0.node_mlp.0.weight', 'decoder.csp_layer_0.node_mlp.0.bias']
+[2024-11-26 22:24:16,488][hydra.utils][INFO] - Passing scaler from datamodule to model <StandardScalerTorch(means: 3.604318141937256, stds: 5.234546661376953)>
+[2024-11-26 22:24:16,511][hydra.utils][INFO] - Adding callback <LearningRateMonitor>
+[2024-11-26 22:24:16,511][hydra.utils][INFO] - Adding callback <EarlyStopping>
+[2024-11-26 22:24:16,512][hydra.utils][INFO] - Adding callback <ModelCheckpoint>
+[2024-11-26 22:24:16,515][hydra.utils][INFO] - Instantiating <WandbLogger>
+[2024-11-26 22:24:16,554][hydra.utils][INFO] - W&B is now watching <{cfg.logging.wandb_watch.log}>!
+[2024-11-26 22:24:18,334][hydra.utils][INFO] - Using checkpoint specified in config: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch=656-step=74898.ckpt
+[2024-11-26 22:24:18,334][hydra.utils][INFO] - Loading model state dict from checkpoint: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch=656-step=74898.ckpt
+[2024-11-26 22:24:21,126][hydra.utils][INFO] - Checkpoint keys: ['decoder.node_embedding.weight', 'decoder.node_embedding.bias', 'decoder.atom_latent_emb.weight', 'decoder.atom_latent_emb.bias', 'decoder.csp_layer_0.edge_mlp.0.weight', 'decoder.csp_layer_0.edge_mlp.0.bias', 'decoder.csp_layer_0.edge_mlp.2.weight', 'decoder.csp_layer_0.edge_mlp.2.bias', 'decoder.csp_layer_0.node_mlp.0.weight', 'decoder.csp_layer_0.node_mlp.0.bias']
+[2024-11-26 22:24:21,134][hydra.utils][WARNING] - Missing keys during checkpoint loading: ['decoder.adapters.0.adapter_fc1.weight', 'decoder.adapters.0.adapter_fc1.bias', 'decoder.adapters.0.adapter_fc2.weight', 'decoder.adapters.0.adapter_fc2.bias', 'decoder.adapters.0.mixin.weight', 'decoder.adapters.1.adapter_fc1.weight', 'decoder.adapters.1.adapter_fc1.bias', 'decoder.adapters.1.adapter_fc2.weight', 'decoder.adapters.1.adapter_fc2.bias', 'decoder.adapters.1.mixin.weight', 'decoder.adapters.2.adapter_fc1.weight', 'decoder.adapters.2.adapter_fc1.bias', 'decoder.adapters.2.adapter_fc2.weight', 'decoder.adapters.2.adapter_fc2.bias', 'decoder.adapters.2.mixin.weight', 'decoder.adapters.3.adapter_fc1.weight', 'decoder.adapters.3.adapter_fc1.bias', 'decoder.adapters.3.adapter_fc2.weight', 'decoder.adapters.3.adapter_fc2.bias', 'decoder.adapters.3.mixin.weight', 'decoder.adapters.4.adapter_fc1.weight', 'decoder.adapters.4.adapter_fc1.bias', 'decoder.adapters.4.adapter_fc2.weight', 'decoder.adapters.4.adapter_fc2.bias', 'decoder.adapters.4.mixin.weight', 'decoder.adapters.5.adapter_fc1.weight', 'decoder.adapters.5.adapter_fc1.bias', 'decoder.adapters.5.adapter_fc2.weight', 'decoder.adapters.5.adapter_fc2.bias', 'decoder.adapters.5.mixin.weight']
+[2024-11-26 22:24:21,659][hydra.utils][INFO] - Instantiating the Trainer
+[2024-11-26 22:24:21,704][hydra.utils][INFO] - Starting training!
+[2024-11-26 22:54:46,657][hydra.utils][INFO] - Starting testing!

superconductor_generator/wandb/debug-internal.log

@@ -0,0 +1,20 @@
+{"time":"2024-11-26T22:24:17.30616145-05:00","level":"INFO","msg":"using version","core version":"0.18.7"}
+{"time":"2024-11-26T22:24:17.326962487-05:00","level":"INFO","msg":"created symlink","path":"wandb/run-20241126_222417-csuxafea/logs/debug-core.log"}
+{"time":"2024-11-26T22:24:17.672833303-05:00","level":"INFO","msg":"created new stream","id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.672881304-05:00","level":"INFO","msg":"stream: started","id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.67289526-05:00","level":"INFO","msg":"handler: started","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.672918514-05:00","level":"INFO","msg":"writer: Do: started","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.672912863-05:00","level":"INFO","msg":"sender: started","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.980971761-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-26T22:54:56.1611024-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-26T22:54:56.161674148-05:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-26T22:54:56.396640998-05:00","level":"WARN","msg":"No job ingredients found, not creating job artifact"}
+{"time":"2024-11-26T22:54:56.396688849-05:00","level":"WARN","msg":"No source type found, not creating job artifact"}
+{"time":"2024-11-26T22:54:56.396696854-05:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-11-26T22:54:56.789916744-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-26T22:54:57.027559138-05:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2024-11-26T22:54:58.042408156-05:00","level":"INFO","msg":"stream: closing","id":"csuxafea"}
+{"time":"2024-11-26T22:54:58.042466435-05:00","level":"INFO","msg":"handler: closed","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:54:58.042483768-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:54:58.042494168-05:00","level":"INFO","msg":"sender: closed","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:54:58.044228708-05:00","level":"INFO","msg":"stream: closed","id":"csuxafea"}

superconductor_generator/wandb/debug.log

@@ -0,0 +1,35 @@
+2024-11-26 22:24:17,299 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Configure stats pid to 1599924
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Loading settings from /home/pawanprakash/.config/wandb/settings
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Loading settings from /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-26/finetune_alex20_sccomb/wandb/settings
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'start_method': 'thread', 'root_dir': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/wandb'}
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2024-11-26 22:24:17,300 WARNING MainThread:1599924 [wandb_setup.py:_flush():79] Could not save program above cwd: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/guide_alex20/guide_656/./../../../MaterialsDiffusion/diffcsp/run.py
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program_abspath': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/diffcsp/run.py', 'program': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/guide_alex20/guide_656/./../../../MaterialsDiffusion/diffcsp/run.py'}
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:_log_setup():533] Logging user logs to ./wandb/run-20241126_222417-csuxafea/logs/debug.log
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:_log_setup():534] Logging internal logs to ./wandb/run-20241126_222417-csuxafea/logs/debug-internal.log
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:init():619] calling init triggers
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {}
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:init():669] starting backend
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:init():673] sending inform_init request
+2024-11-26 22:24:17,302 INFO    MainThread:1599924 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: fork
+2024-11-26 22:24:17,303 INFO    MainThread:1599924 [wandb_init.py:init():686] backend started and connected
+2024-11-26 22:24:17,307 INFO    MainThread:1599924 [wandb_init.py:init():781] updated telemetry
+2024-11-26 22:24:17,308 INFO    MainThread:1599924 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2024-11-26 22:24:17,975 INFO    MainThread:1599924 [wandb_init.py:init():867] starting run threads in backend
+2024-11-26 22:24:18,314 INFO    MainThread:1599924 [wandb_run.py:_console_start():2456] atexit reg
+2024-11-26 22:24:18,314 INFO    MainThread:1599924 [wandb_run.py:_redirect():2305] redirect: wrap_raw
+2024-11-26 22:24:18,315 INFO    MainThread:1599924 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2024-11-26 22:24:18,315 INFO    MainThread:1599924 [wandb_run.py:_redirect():2395] Redirects installed.
+2024-11-26 22:24:18,325 INFO    MainThread:1599924 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-26 22:24:18,325 INFO    MainThread:1599924 [wandb_watch.py:_watch():71] Watching
+2024-11-26 22:24:21,703 INFO    MainThread:1599924 [wandb_run.py:_config_callback():1387] config_cb None None {'data': {'root_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12', 'prop': 'tcad', 'num_targets': 1, 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30, 'readout': 'mean', 'max_atoms': 12, 'otf_graph': False, 'eval_model_name': 'supccomb12', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'train_max_epochs': 1500, 'early_stopping_patience': 200, 'teacher_forcing_max_epoch': 500, 'datamodule': {'_target_': 'diffcsp.pl_data.datamodule.CrystDataModule', 'datasets': {'train': {'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy train', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/train.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/train_ori.pt', 'prop': 'tcad', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}, 'val': [{'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy val', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/val.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/val_ori.pt', 'prop': 'tcad', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}], 'test': [{'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy test', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/test.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/test_ori.pt', 'prop': 'tcad', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}]}, 'num_workers': {'train': 0, 'val': 0, 'test': 0}, 'batch_size': {'train': 256, 'val': 128, 'test': 128}}}, 'logging': {'val_check_interval': 1, 'wandb': {'name': 'finetune_alex20_sccomb', 'project': 'diffcsp', 'entity': None, 'log_model': True, 'mode': 'online', 'group': 'finetune_alex20_sccomb'}, 'wandb_watch': {'log': 'all', 'log_freq': 500}, 'lr_monitor': {'logging_interval': 'step', 'log_momentum': False}}, 'model': {'decoder': {'_target_': 'diffcsp.pl_modules.cspnet.CSPNet', 'hidden_dim': 512, 'latent_dim': 0, 'max_atoms': 100, 'num_layers': 6, 'act_fn': 'silu', 'dis_emb': 'sin', 'num_freqs': 128, 'edge_style': 'fc', 'max_neighbors': 20, 'cutoff': 7.0, 'ln': True, 'ip': True, 'prop_embed_dim': 64, 'am_hidden_dim': 128}, 'beta_scheduler': {'_target_': 'diffcsp.pl_modules.diff_utils.BetaScheduler', 'timesteps': 1000, 'scheduler_mode': 'cosine'}, 'sigma_scheduler': {'_target_': 'diffcsp.pl_modules.diff_utils.SigmaScheduler', 'timesteps': 1000, 'sigma_begin': 0.005, 'sigma_end': 0.5}, '_target_': 'diffcsp.pl_modules.diffusion_w_type.CSPDiffusion', 'time_dim': 256, 'latent_dim': 0, 'cost_coord': 1.0, 'cost_lattice': 1.0, 'cost_type': 20.0, 'max_neighbors': 20, 'radius': 7.0, 'timesteps': 1000, 'p_uncond': 0.2, 'guide_w': 3.0}, 'optim': {'optimizer': {'_target_': 'torch.optim.Adam', 'lr': 0.001, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0}, 'use_lr_scheduler': True, 'lr_scheduler': {'_target_': 'torch.optim.lr_scheduler.ReduceLROnPlateau', 'factor': 0.6, 'patience': 30, 'min_lr': 0.0001}}, 'train': {'deterministic': True, 'random_seed': 42, 'pl_trainer': {'fast_dev_run': False, 'accelerator': 'auto', 'precision': 32, 'max_epochs': 1500, 'accumulate_grad_batches': 1, 'num_sanity_val_steps': 2, 'gradient_clip_val': 0.5, 'gradient_clip_algorithm': 'value', 'profiler': 'simple'}, 'monitor_metric': 'val_loss', 'monitor_metric_mode': 'min', 'early_stopping': {'patience': 200, 'verbose': True}, 'model_checkpoints': {'save_top_k': 1, 'verbose': False, 'save_last': False}, 'ckpt_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch=656-step=74898.ckpt'}, 'expname': 'finetune_alex20_sccomb', 'core': {'version': '0.0.1', 'tags': ['2024-11-26']}, 'stats/params_total': 14365540, 'stats/params_trainable': 14365540, 'stats/params_not_trainable': 0}
+2024-11-26 22:54:56,158 INFO    MainThread:1599924 [wandb_run.py:_finish():2155] finishing run uf_hennig/diffcsp/csuxafea
+2024-11-26 22:54:56,159 INFO    MainThread:1599924 [wandb_run.py:_atexit_cleanup():2420] got exitcode: 0
+2024-11-26 22:54:56,160 INFO    MainThread:1599924 [wandb_run.py:_restore():2402] restore
+2024-11-26 22:54:56,160 INFO    MainThread:1599924 [wandb_run.py:_restore():2408] restore done
+2024-11-26 22:54:58,035 INFO    MainThread:1599924 [wandb_run.py:_footer_history_summary_info():3960] rendering history
+2024-11-26 22:54:58,036 INFO    MainThread:1599924 [wandb_run.py:_footer_history_summary_info():3992] rendering summary
+2024-11-26 22:54:58,041 INFO    MainThread:1599924 [wandb_run.py:_footer_sync_info():3921] logging synced files

superconductor_generator/wandb/run-20241126_222417-csuxafea/files/requirements.txt

@@ -0,0 +1,109 @@
+lightning-utilities==0.11.9
+pytorch-lightning==2.4.0
+kiwisolver==1.4.7
+six==1.16.0
+torch_scatter==2.1.2+pt23cu121
+torchmetrics==1.6.0
+multiprocess==0.70.17
+ruamel.yaml==0.18.6
+fonttools==4.55.0
+vasprun-xml==1.0.4
+joblib==1.4.2
+pandas==2.2.3
+packaging==24.2
+multidict==6.1.0
+smmap==5.0.1
+lxml==5.3.0
+sentry-sdk==2.18.0
+nvidia-cuda-nvrtc-cu12==12.1.105
+psutil==6.1.0
+nvidia-cuda-cupti-cu12==12.1.105
+setproctitle==1.3.4
+MarkupSafe==3.0.2
+xmltodict==0.14.2
+python-dotenv==1.0.1
+platformdirs==4.3.6
+nvidia-curand-cu12==10.3.2.106
+aiosignal==1.3.1
+pybtex==0.24.0
+aiohappyeyeballs==2.4.3
+contourpy==1.3.1
+nvidia-cusparse-cu12==12.1.0.106
+yarl==1.17.2
+charset-normalizer==3.4.0
+networkx==3.4.2
+click==8.1.7
+requests==2.32.3
+nvidia-nvjitlink-cu12==12.6.77
+dill==0.3.9
+spglib==2.5.0
+GitPython==3.1.43
+numpy==1.26.4
+typing_extensions==4.12.2
+SMACT==2.8
+cycler==0.12.1
+p_tqdm==1.4.2
+hydra-core==1.3.2
+nvidia-cuda-runtime-cu12==12.1.105
+latexcodec==3.0.0
+python-dateutil==2.9.0.post0
+tenacity==9.0.0
+pox==0.3.5
+importlib_metadata==8.5.0
+pyparsing==3.2.0
+fsspec==2024.10.0
+torch_sparse==0.6.18+pt23cu121
+omegaconf==2.3.0
+plotly==5.24.1
+einops==0.8.0
+pip==24.2
+pymatgen==2024.11.13
+ase==3.23.0
+nvidia-cudnn-cu12==8.9.2.26
+idna==3.10
+monty==2024.10.21
+protobuf==5.28.3
+uncertainties==3.2.2
+setuptools==75.1.0
+propcache==0.2.0
+torch==2.3.0
+tzdata==2024.2
+nvidia-cublas-cu12==12.1.3.1
+threadpoolctl==3.5.0
+certifi==2024.8.30
+sympy==1.13.3
+nvidia-nvtx-cu12==12.1.105
+pathos==0.3.3
+chemparse==0.3.1
+torch-geometric==2.6.1
+wheel==0.44.0
+tqdm==4.67.0
+scikit-learn==1.5.2
+toml==0.10.2
+mpmath==1.3.0
+nvidia-cufft-cu12==11.0.2.54
+pillow==11.0.0
+urllib3==2.2.3
+zipp==3.21.0
+nvidia-nccl-cu12==2.20.5
+scipy==1.14.1
+pytz==2024.2
+aiohttp==3.11.6
+ppft==1.7.6.9
+tabulate==0.9.0
+palettable==3.3.3
+pyocse==0.1.2
+wandb==0.18.7
+ruamel.yaml.clib==0.2.12
+attrs==24.2.0
+docker-pycreds==0.4.0
+frozenlist==1.5.0
+matplotlib==3.9.2
+Jinja2==3.1.4
+PyYAML==6.0.2
+nvidia-cusolver-cu12==11.4.5.107
+antlr4-python3-runtime==4.9.3
+filelock==3.16.1
+diffcsp==0.0.1
+pyxtal==1.0.5
+gitdb==4.0.11

superconductor_generator/wandb/run-20241126_222417-csuxafea/files/wandb-metadata.json

@@ -0,0 +1,126 @@
+{
+  "os": "Linux-4.18.0-553.27.1.el8_10.x86_64-x86_64-with-glibc2.28",
+  "python": "3.12.2",
+  "startedAt": "2024-11-27T03:24:17.303247Z",
+  "args": [
+    "data=supccomb_12",
+    "model=diffusion_w_type",
+    "expname=finetune_alex20_sccomb",
+    "train.ckpt_path=/blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch\\=656-step\\=74898.ckpt"
+  ],
+  "program": "/blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/guide_alex20/guide_656/./../../../MaterialsDiffusion/diffcsp/run.py",
+  "email": "pawanprakash@ufl.edu",
+  "root": ".",
+  "host": "c1106a-s11.ufhpc",
+  "username": "pawanprakash",
+  "executable": "/home/pawanprakash/miniconda3/envs/diffcsp2/bin/python",
+  "cpu_count": 128,
+  "cpu_count_logical": 128,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "41922297856",
+      "used": "13336027136"
+    }
+  },
+  "memory": {
+    "total": "2159777349632"
+  },
+  "cpu": {
+    "count": 128,
+    "countLogical": 128
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere"
+    }
+  ],
+  "slurm": {
+    "cluster_name": "hipergator",
+    "conf": "/opt/slurm/etc/slurm.conf",
+    "cpus_on_node": "8",
+    "cpus_per_task": "1",
+    "gpus": "a100:1",
+    "gpus_on_node": "1",
+    "gtids": "0",
+    "job_account": "hennig",
+    "job_cpus_per_node": "8",
+    "job_gid": "3424",
+    "job_gpus": "2",
+    "job_id": "51111921",
+    "job_name": "a20_sccomb",
+    "job_nodelist": "c1106a-s11",
+    "job_num_nodes": "1",
+    "job_partition": "gpu",
+    "job_qos": "hennig qos",
+    "job_uid": "8611",
+    "job_user": "pawanprakash",
+    "jobid": "51111921",
+    "localid": "0",
+    "mem_per_cpu": "4000",
+    "nnodes": "1",
+    "node_aliases": "(null)",
+    "nodeid": "0",
+    "nodelist": "c1106a-s11",
+    "nprocs": "8",
+    "ntasks": "8",
+    "ntasks_per_node": "8",
+    "prio_process": "0",
+    "procid": "0",
+    "submit_dir": "/blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/guide_alex20/guide_656",
+    "submit_host": "c0906a-s23.ufhpc",
+    "task_pid": "1599840",
+    "tasks_per_node": "8",
+    "tmpdir": "/scratch/local/51111921",
+    "topology_addr": ".ait1-1.ibsw1108.c1106a-s11",
+    "topology_addr_pattern": "switch.switch.switch.node",
+    "working_cluster": "hipergator:vslurm:6817:9472:109"
+  },
+  "cudaVersion": "12.4"
+}

superconductor_generator/wandb/run-20241126_222417-csuxafea/logs/debug-core.log

@@ -0,0 +1,16 @@
+{"time":"2024-11-26T22:24:16.886888646-05:00","level":"INFO","msg":"started logging, with flags","port-filename":"/scratch/local/51111921/tmpfet75owl/port-1599924.txt","pid":1599924,"debug":false,"disable-analytics":false}
+{"time":"2024-11-26T22:24:16.88692314-05:00","level":"INFO","msg":"FeatureState","shutdownOnParentExitEnabled":false}
+{"time":"2024-11-26T22:24:16.887688233-05:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":1599924}
+{"time":"2024-11-26T22:24:16.887660801-05:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":40327,"Zone":""}}
+{"time":"2024-11-26T22:24:17.080796589-05:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:24:17.304611247-05:00","level":"INFO","msg":"handleInformInit: received","streamId":"csuxafea","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:24:17.672890681-05:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"csuxafea","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:54:58.042148466-05:00","level":"INFO","msg":"handleInformFinish: finish message received","streamId":"csuxafea","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:54:58.044250178-05:00","level":"INFO","msg":"handleInformFinish: stream closed","streamId":"csuxafea","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:54:58.082820196-05:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:54:58.082845744-05:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:54:58.082850102-05:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2024-11-26T22:54:58.082850453-05:00","level":"INFO","msg":"connection: Close: initiating connection closure","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:54:58.082896269-05:00","level":"INFO","msg":"connection: Close: connection successfully closed","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:54:58.082899576-05:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:50432"}
+{"time":"2024-11-26T22:54:58.082903964-05:00","level":"INFO","msg":"server is closed"}

superconductor_generator/wandb/run-20241126_222417-csuxafea/logs/debug-internal.log

@@ -0,0 +1,20 @@
+{"time":"2024-11-26T22:24:17.30616145-05:00","level":"INFO","msg":"using version","core version":"0.18.7"}
+{"time":"2024-11-26T22:24:17.326962487-05:00","level":"INFO","msg":"created symlink","path":"wandb/run-20241126_222417-csuxafea/logs/debug-core.log"}
+{"time":"2024-11-26T22:24:17.672833303-05:00","level":"INFO","msg":"created new stream","id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.672881304-05:00","level":"INFO","msg":"stream: started","id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.67289526-05:00","level":"INFO","msg":"handler: started","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.672918514-05:00","level":"INFO","msg":"writer: Do: started","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.672912863-05:00","level":"INFO","msg":"sender: started","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:24:17.980971761-05:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-11-26T22:54:56.1611024-05:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-11-26T22:54:56.161674148-05:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-11-26T22:54:56.396640998-05:00","level":"WARN","msg":"No job ingredients found, not creating job artifact"}
+{"time":"2024-11-26T22:54:56.396688849-05:00","level":"WARN","msg":"No source type found, not creating job artifact"}
+{"time":"2024-11-26T22:54:56.396696854-05:00","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-11-26T22:54:56.789916744-05:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-11-26T22:54:57.027559138-05:00","level":"INFO","msg":"handler: operation stats","stats":{}}
+{"time":"2024-11-26T22:54:58.042408156-05:00","level":"INFO","msg":"stream: closing","id":"csuxafea"}
+{"time":"2024-11-26T22:54:58.042466435-05:00","level":"INFO","msg":"handler: closed","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:54:58.042483768-05:00","level":"INFO","msg":"writer: Close: closed","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:54:58.042494168-05:00","level":"INFO","msg":"sender: closed","stream_id":"csuxafea"}
+{"time":"2024-11-26T22:54:58.044228708-05:00","level":"INFO","msg":"stream: closed","id":"csuxafea"}

superconductor_generator/wandb/run-20241126_222417-csuxafea/logs/debug.log

@@ -0,0 +1,35 @@
+2024-11-26 22:24:17,299 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Current SDK version is 0.18.7
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Configure stats pid to 1599924
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Loading settings from /home/pawanprakash/.config/wandb/settings
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Loading settings from /blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-26/finetune_alex20_sccomb/wandb/settings
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'start_method': 'thread', 'root_dir': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/wandb'}
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': 'online', '_disable_service': None}
+2024-11-26 22:24:17,300 WARNING MainThread:1599924 [wandb_setup.py:_flush():79] Could not save program above cwd: /blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/guide_alex20/guide_656/./../../../MaterialsDiffusion/diffcsp/run.py
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program_abspath': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/diffcsp/run.py', 'program': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/runjobs/guide_alex20/guide_656/./../../../MaterialsDiffusion/diffcsp/run.py'}
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:_log_setup():533] Logging user logs to ./wandb/run-20241126_222417-csuxafea/logs/debug.log
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:_log_setup():534] Logging internal logs to ./wandb/run-20241126_222417-csuxafea/logs/debug-internal.log
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:init():619] calling init triggers
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:init():626] wandb.init called with sweep_config: {}
+config: {}
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:init():669] starting backend
+2024-11-26 22:24:17,300 INFO    MainThread:1599924 [wandb_init.py:init():673] sending inform_init request
+2024-11-26 22:24:17,302 INFO    MainThread:1599924 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: fork
+2024-11-26 22:24:17,303 INFO    MainThread:1599924 [wandb_init.py:init():686] backend started and connected
+2024-11-26 22:24:17,307 INFO    MainThread:1599924 [wandb_init.py:init():781] updated telemetry
+2024-11-26 22:24:17,308 INFO    MainThread:1599924 [wandb_init.py:init():814] communicating run to backend with 90.0 second timeout
+2024-11-26 22:24:17,975 INFO    MainThread:1599924 [wandb_init.py:init():867] starting run threads in backend
+2024-11-26 22:24:18,314 INFO    MainThread:1599924 [wandb_run.py:_console_start():2456] atexit reg
+2024-11-26 22:24:18,314 INFO    MainThread:1599924 [wandb_run.py:_redirect():2305] redirect: wrap_raw
+2024-11-26 22:24:18,315 INFO    MainThread:1599924 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2024-11-26 22:24:18,315 INFO    MainThread:1599924 [wandb_run.py:_redirect():2395] Redirects installed.
+2024-11-26 22:24:18,325 INFO    MainThread:1599924 [wandb_init.py:init():911] run started, returning control to user process
+2024-11-26 22:24:18,325 INFO    MainThread:1599924 [wandb_watch.py:_watch():71] Watching
+2024-11-26 22:24:21,703 INFO    MainThread:1599924 [wandb_run.py:_config_callback():1387] config_cb None None {'data': {'root_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12', 'prop': 'tcad', 'num_targets': 1, 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30, 'readout': 'mean', 'max_atoms': 12, 'otf_graph': False, 'eval_model_name': 'supccomb12', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'train_max_epochs': 1500, 'early_stopping_patience': 200, 'teacher_forcing_max_epoch': 500, 'datamodule': {'_target_': 'diffcsp.pl_data.datamodule.CrystDataModule', 'datasets': {'train': {'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy train', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/train.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/train_ori.pt', 'prop': 'tcad', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}, 'val': [{'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy val', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/val.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/val_ori.pt', 'prop': 'tcad', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}], 'test': [{'_target_': 'diffcsp.pl_data.dataset.CrystDataset', 'name': 'Formation energy test', 'path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/test.csv', 'save_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/MaterialsDiffusion/data/supccomb_12/test_ori.pt', 'prop': 'tcad', 'niggli': True, 'primitive': False, 'graph_method': 'crystalnn', 'tolerance': 0.1, 'use_space_group': False, 'use_pos_index': False, 'lattice_scale_method': 'scale_length', 'preprocess_workers': 30}]}, 'num_workers': {'train': 0, 'val': 0, 'test': 0}, 'batch_size': {'train': 256, 'val': 128, 'test': 128}}}, 'logging': {'val_check_interval': 1, 'wandb': {'name': 'finetune_alex20_sccomb', 'project': 'diffcsp', 'entity': None, 'log_model': True, 'mode': 'online', 'group': 'finetune_alex20_sccomb'}, 'wandb_watch': {'log': 'all', 'log_freq': 500}, 'lr_monitor': {'logging_interval': 'step', 'log_momentum': False}}, 'model': {'decoder': {'_target_': 'diffcsp.pl_modules.cspnet.CSPNet', 'hidden_dim': 512, 'latent_dim': 0, 'max_atoms': 100, 'num_layers': 6, 'act_fn': 'silu', 'dis_emb': 'sin', 'num_freqs': 128, 'edge_style': 'fc', 'max_neighbors': 20, 'cutoff': 7.0, 'ln': True, 'ip': True, 'prop_embed_dim': 64, 'am_hidden_dim': 128}, 'beta_scheduler': {'_target_': 'diffcsp.pl_modules.diff_utils.BetaScheduler', 'timesteps': 1000, 'scheduler_mode': 'cosine'}, 'sigma_scheduler': {'_target_': 'diffcsp.pl_modules.diff_utils.SigmaScheduler', 'timesteps': 1000, 'sigma_begin': 0.005, 'sigma_end': 0.5}, '_target_': 'diffcsp.pl_modules.diffusion_w_type.CSPDiffusion', 'time_dim': 256, 'latent_dim': 0, 'cost_coord': 1.0, 'cost_lattice': 1.0, 'cost_type': 20.0, 'max_neighbors': 20, 'radius': 7.0, 'timesteps': 1000, 'p_uncond': 0.2, 'guide_w': 3.0}, 'optim': {'optimizer': {'_target_': 'torch.optim.Adam', 'lr': 0.001, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0}, 'use_lr_scheduler': True, 'lr_scheduler': {'_target_': 'torch.optim.lr_scheduler.ReduceLROnPlateau', 'factor': 0.6, 'patience': 30, 'min_lr': 0.0001}}, 'train': {'deterministic': True, 'random_seed': 42, 'pl_trainer': {'fast_dev_run': False, 'accelerator': 'auto', 'precision': 32, 'max_epochs': 1500, 'accumulate_grad_batches': 1, 'num_sanity_val_steps': 2, 'gradient_clip_val': 0.5, 'gradient_clip_algorithm': 'value', 'profiler': 'simple'}, 'monitor_metric': 'val_loss', 'monitor_metric_mode': 'min', 'early_stopping': {'patience': 200, 'verbose': True}, 'model_checkpoints': {'save_top_k': 1, 'verbose': False, 'save_last': False}, 'ckpt_path': '/blue/hennig/pawanprakash/diffusion/Diffscp_guide/hydra/singlerun/2024-11-21/alex_20/epoch=656-step=74898.ckpt'}, 'expname': 'finetune_alex20_sccomb', 'core': {'version': '0.0.1', 'tags': ['2024-11-26']}, 'stats/params_total': 14365540, 'stats/params_trainable': 14365540, 'stats/params_not_trainable': 0}
+2024-11-26 22:54:56,158 INFO    MainThread:1599924 [wandb_run.py:_finish():2155] finishing run uf_hennig/diffcsp/csuxafea
+2024-11-26 22:54:56,159 INFO    MainThread:1599924 [wandb_run.py:_atexit_cleanup():2420] got exitcode: 0
+2024-11-26 22:54:56,160 INFO    MainThread:1599924 [wandb_run.py:_restore():2402] restore
+2024-11-26 22:54:56,160 INFO    MainThread:1599924 [wandb_run.py:_restore():2408] restore done
+2024-11-26 22:54:58,035 INFO    MainThread:1599924 [wandb_run.py:_footer_history_summary_info():3960] rendering history
+2024-11-26 22:54:58,036 INFO    MainThread:1599924 [wandb_run.py:_footer_history_summary_info():3992] rendering summary
+2024-11-26 22:54:58,041 INFO    MainThread:1599924 [wandb_run.py:_footer_sync_info():3921] logging synced files

superconductor_generator/wandb/run-20241126_222417-csuxafea/run-csuxafea.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f52403ea13ff7099b29cc8dde89a3675bad8af8b9fcff988c2de7ebedfba1cd9
+size 36443224