upload permesk checkpoints (ignore sqlite wal/shm)

.gitattributes

@@ -192,3 +192,4 @@ run_r34_512_20260306_084725/samples/epoch_087.jpg filter=lfs diff=lfs merge=lfs
 run_r34_512_20260306_084725/samples/epoch_088.jpg filter=lfs diff=lfs merge=lfs -text
 run_r34_512_20260306_084725/samples/epoch_089.jpg filter=lfs diff=lfs merge=lfs -text
 run_r34_512_20260306_084725/samples/epoch_090.jpg filter=lfs diff=lfs merge=lfs -text
+permesk/loss_log.db filter=lfs diff=lfs merge=lfs -text

permesk/.job_config.json

@@ -0,0 +1,127 @@
+{
+  "job": "extension",
+  "config": {
+    "name": "permesk",
+    "process": [
+      {
+        "type": "diffusion_trainer",
+        "training_folder": "/workspace/ai-toolkit/output",
+        "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
+        "device": "cuda",
+        "trigger_word": "permesk",
+        "performance_log_every": 10,
+        "network": {
+          "type": "lora",
+          "linear": 512,
+          "linear_alpha": 512,
+          "conv": 16,
+          "conv_alpha": 16,
+          "lokr_full_rank": true,
+          "lokr_factor": -1,
+          "network_kwargs": {
+            "ignore_if_contains": []
+          }
+        },
+        "save": {
+          "dtype": "bf16",
+          "save_every": 1000,
+          "max_step_saves_to_keep": 4,
+          "save_format": "diffusers",
+          "push_to_hub": false
+        },
+        "datasets": [
+          {
+            "folder_path": "/workspace/ai-toolkit/datasets/qwe",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "permesk woman",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+              1024
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 2
+          }
+        ],
+        "train": {
+          "batch_size": 1,
+          "bypass_guidance_embedding": false,
+          "steps": 3000,
+          "gradient_accumulation": 1,
+          "train_unet": true,
+          "train_text_encoder": false,
+          "gradient_checkpointing": true,
+          "noise_scheduler": "flowmatch",
+          "optimizer": "adamw8bit",
+          "timestep_type": "weighted",
+          "content_or_style": "balanced",
+          "optimizer_params": {
+            "weight_decay": 0.0001
+          },
+          "unload_text_encoder": false,
+          "cache_text_embeddings": false,
+          "lr": 0.0003,
+          "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+          },
+          "skip_first_sample": false,
+          "force_first_sample": false,
+          "disable_sampling": false,
+          "dtype": "bf16",
+          "diff_output_preservation": false,
+          "diff_output_preservation_multiplier": 1,
+          "diff_output_preservation_class": "person",
+          "switch_boundary_every": 1,
+          "loss_type": "mse",
+          "do_differential_guidance": true,
+          "differential_guidance_scale": 3
+        },
+        "logging": {
+          "log_every": 1,
+          "use_ui_logger": true
+        },
+        "model": {
+          "name_or_path": "Tongyi-MAI/Z-Image-Turbo",
+          "quantize": true,
+          "qtype": "qfloat8",
+          "quantize_te": true,
+          "qtype_te": "qfloat8",
+          "arch": "zimage:turbo",
+          "low_vram": false,
+          "model_kwargs": {},
+          "layer_offloading": false,
+          "layer_offloading_text_encoder_percent": 1,
+          "layer_offloading_transformer_percent": 1,
+          "assistant_lora_path": "ostris/zimage_turbo_training_adapter/zimage_turbo_training_adapter_v2.safetensors"
+        },
+        "sample": {
+          "sampler": "flowmatch",
+          "sample_every": 250,
+          "width": 1024,
+          "height": 1024,
+          "samples": [],
+          "neg": "",
+          "seed": 42,
+          "walk_seed": true,
+          "guidance_scale": 1,
+          "sample_steps": 8,
+          "num_frames": 1,
+          "fps": 1
+        }
+      }
+    ]
+  },
+  "meta": {
+    "name": "[name]",
+    "version": "1.0"
+  }
+}

permesk/config.yaml

@@ -0,0 +1,107 @@
+job: extension
+config:
+  name: permesk
+  process:
+  - type: diffusion_trainer
+    training_folder: /workspace/ai-toolkit/output
+    sqlite_db_path: /workspace/ai-toolkit/aitk_db.db
+    device: cuda
+    trigger_word: permesk
+    performance_log_every: 10
+    network:
+      type: lora
+      linear: 512
+      linear_alpha: 512
+      conv: 16
+      conv_alpha: 16
+      lokr_full_rank: true
+      lokr_factor: -1
+      network_kwargs:
+        ignore_if_contains: []
+    save:
+      dtype: bf16
+      save_every: 1000
+      max_step_saves_to_keep: 4
+      save_format: diffusers
+      push_to_hub: false
+    datasets:
+    - folder_path: /workspace/ai-toolkit/datasets/qwe
+      mask_path: null
+      mask_min_value: 0.1
+      default_caption: permesk woman
+      caption_ext: txt
+      caption_dropout_rate: 0.05
+      cache_latents_to_disk: false
+      is_reg: false
+      network_weight: 1
+      resolution:
+      - 1024
+      controls: []
+      shrink_video_to_frames: true
+      num_frames: 1
+      flip_x: false
+      flip_y: false
+      num_repeats: 2
+    train:
+      batch_size: 1
+      bypass_guidance_embedding: false
+      steps: 3000
+      gradient_accumulation: 1
+      train_unet: true
+      train_text_encoder: false
+      gradient_checkpointing: true
+      noise_scheduler: flowmatch
+      optimizer: adamw8bit
+      timestep_type: weighted
+      content_or_style: balanced
+      optimizer_params:
+        weight_decay: 0.0001
+      unload_text_encoder: false
+      cache_text_embeddings: false
+      lr: 0.0003
+      ema_config:
+        use_ema: false
+        ema_decay: 0.99
+      skip_first_sample: false
+      force_first_sample: false
+      disable_sampling: false
+      dtype: bf16
+      diff_output_preservation: false
+      diff_output_preservation_multiplier: 1
+      diff_output_preservation_class: person
+      switch_boundary_every: 1
+      loss_type: mse
+      do_differential_guidance: true
+      differential_guidance_scale: 3
+    logging:
+      log_every: 1
+      use_ui_logger: true
+    model:
+      name_or_path: Tongyi-MAI/Z-Image-Turbo
+      quantize: true
+      qtype: qfloat8
+      quantize_te: true
+      qtype_te: qfloat8
+      arch: zimage:turbo
+      low_vram: false
+      model_kwargs: {}
+      layer_offloading: false
+      layer_offloading_text_encoder_percent: 1
+      layer_offloading_transformer_percent: 1
+      assistant_lora_path: ostris/zimage_turbo_training_adapter/zimage_turbo_training_adapter_v2.safetensors
+    sample:
+      sampler: flowmatch
+      sample_every: 250
+      width: 1024
+      height: 1024
+      samples: []
+      neg: ''
+      seed: 42
+      walk_seed: true
+      guidance_scale: 1
+      sample_steps: 8
+      num_frames: 1
+      fps: 1
+meta:
+  name: permesk
+  version: '1.0'

permesk/logs/0_log.txt

@@ -0,0 +1,246 @@
+Running 1 job
+{
+    "type": "diffusion_trainer",
+    "training_folder": "/workspace/ai-toolkit/output",
+    "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
+    "device": "cuda",
+    "trigger_word": "permesk",
+    "performance_log_every": 10,
+    "network": {
+        "type": "lora",
+        "linear": 512,
+        "linear_alpha": 512,
+        "conv": 16,
+        "conv_alpha": 16,
+        "lokr_full_rank": true,
+        "lokr_factor": -1,
+        "network_kwargs": {
+            "ignore_if_contains": []
+        }
+    },
+    "save": {
+        "dtype": "bf16",
+        "save_every": 1000,
+        "max_step_saves_to_keep": 4,
+        "save_format": "diffusers",
+        "push_to_hub": false
+    },
+    "datasets": [
+        {
+            "folder_path": "/workspace/ai-toolkit/datasets/qwe",
+            "mask_path": null,
+            "mask_min_value": 0.1,
+            "default_caption": "permesk woman",
+            "caption_ext": "txt",
+            "caption_dropout_rate": 0.05,
+            "cache_latents_to_disk": false,
+            "is_reg": false,
+            "network_weight": 1,
+            "resolution": [
+                1024
+            ],
+            "controls": [],
+            "shrink_video_to_frames": true,
+            "num_frames": 1,
+            "flip_x": false,
+            "flip_y": false,
+            "num_repeats": 3
+        }
+    ],
+    "train": {
+        "batch_size": 1,
+        "bypass_guidance_embedding": false,
+        "steps": 3000,
+        "gradient_accumulation": 1,
+        "train_unet": true,
+        "train_text_encoder": false,
+        "gradient_checkpointing": true,
+        "noise_scheduler": "flowmatch",
+        "optimizer": "adamw8bit",
+        "timestep_type": "weighted",
+        "content_or_style": "balanced",
+        "optimizer_params": {
+            "weight_decay": 0.0001
+        },
+        "unload_text_encoder": false,
+        "cache_text_embeddings": false,
+        "lr": 0.0003,
+        "ema_config": {
+            "use_ema": false,
+            "ema_decay": 0.99
+        },
+        "skip_first_sample": false,
+        "force_first_sample": false,
+        "disable_sampling": false,
+        "dtype": "bf16",
+        "diff_output_preservation": false,
+        "diff_output_preservation_multiplier": 1,
+        "diff_output_preservation_class": "person",
+        "switch_boundary_every": 1,
+        "loss_type": "mse",
+        "do_differential_guidance": true,
+        "differential_guidance_scale": 3
+    },
+    "logging": {
+        "log_every": 1,
+        "use_ui_logger": true
+    },
+    "model": {
+        "name_or_path": "Tongyi-MAI/Z-Image-Turbo",
+        "quantize": true,
+        "qtype": "qfloat8",
+        "quantize_te": true,
+        "qtype_te": "qfloat8",
+        "arch": "zimage:turbo",
+        "low_vram": false,
+        "model_kwargs": {},
+        "layer_offloading": false,
+        "layer_offloading_text_encoder_percent": 1,
+        "layer_offloading_transformer_percent": 1,
+        "assistant_lora_path": "ostris/zimage_turbo_training_adapter/zimage_turbo_training_adapter_v2.safetensors"
+    },
+    "sample": {
+        "sampler": "flowmatch",
+        "sample_every": 250,
+        "width": 1024,
+        "height": 1024,
+        "samples": [],
+        "neg": "",
+        "seed": 42,
+        "walk_seed": true,
+        "guidance_scale": 1,
+        "sample_steps": 8,
+        "num_frames": 1,
+        "fps": 1
+    }
+}
+Using SQLite database at /workspace/ai-toolkit/aitk_db.db
+Job ID: "d857f53c-4b8d-421d-bda6-7c5176c49d09"
+
+#############################################
+# Running job: permesk
+#############################################
+
+
+Running  1 process
+Loading ZImage model
+Loading transformer
+
+
+
+
+
+
+
+
+
+
+
+
+Loading assistant LoRA
+
+
+create LoRA network. base dim (rank): 64, alpha: 64
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 276 modules.
+enable LoRA for U-Net
+Merging in assistant LoRA
+Missing keys: []
+Quantizing Transformer
+ - quantizing 30 transformer blocks
+
  0%|          | 0/30 [00:00<?, ?it/s]
  0%|          | 0/30 [00:00<?, ?it/s]
  3%|3         | 1/30 [00:00<00:09,  3.08it/s]
  3%|3         | 1/30 [00:00<00:09,  3.08it/s]
  7%|6         | 2/30 [00:00<00:05,  4.68it/s]
  7%|6         | 2/30 [00:00<00:05,  4.68it/s]
 10%|#         | 3/30 [00:00<00:04,  5.71it/s]
 10%|#         | 3/30 [00:00<00:04,  5.71it/s]
 13%|#3        | 4/30 [00:00<00:03,  6.56it/s]
 13%|#3        | 4/30 [00:00<00:03,  6.56it/s]
 17%|#6        | 5/30 [00:00<00:04,  5.49it/s]
 17%|#6        | 5/30 [00:00<00:04,  5.49it/s]
 20%|##        | 6/30 [00:01<00:03,  6.27it/s]
 20%|##        | 6/30 [00:01<00:03,  6.27it/s]
 23%|##3       | 7/30 [00:01<00:03,  6.80it/s]
 23%|##3       | 7/30 [00:01<00:03,  6.80it/s]
 27%|##6       | 8/30 [00:01<00:03,  7.10it/s]
 27%|##6       | 8/30 [00:01<00:03,  7.10it/s]
 30%|###       | 9/30 [00:01<00:02,  7.26it/s]
 30%|###       | 9/30 [00:01<00:02,  7.26it/s]
 33%|###3      | 10/30 [00:01<00:02,  7.46it/s]
 33%|###3      | 10/30 [00:01<00:02,  7.46it/s]
 37%|###6      | 11/30 [00:01<00:02,  7.84it/s]
 37%|###6      | 11/30 [00:01<00:02,  7.84it/s]
 40%|####      | 12/30 [00:01<00:02,  7.44it/s]
 40%|####      | 12/30 [00:01<00:02,  7.44it/s]
 43%|####3     | 13/30 [00:01<00:02,  7.20it/s]
 43%|####3     | 13/30 [00:01<00:02,  7.20it/s]
 47%|####6     | 14/30 [00:02<00:02,  7.56it/s]
 47%|####6     | 14/30 [00:02<00:02,  7.56it/s]
 50%|#####     | 15/30 [00:02<00:01,  7.64it/s]
 50%|#####     | 15/30 [00:02<00:01,  7.64it/s]
 53%|#####3    | 16/30 [00:02<00:01,  7.67it/s]
 53%|#####3    | 16/30 [00:02<00:01,  7.67it/s]
 57%|#####6    | 17/30 [00:02<00:01,  7.71it/s]
 57%|#####6    | 17/30 [00:02<00:01,  7.71it/s]
 60%|######    | 18/30 [00:02<00:01,  7.96it/s]
 60%|######    | 18/30 [00:02<00:01,  7.96it/s]
 63%|######3   | 19/30 [00:02<00:01,  8.12it/s]
 63%|######3   | 19/30 [00:02<00:01,  8.12it/s]
 67%|######6   | 20/30 [00:02<00:01,  7.02it/s]
 67%|######6   | 20/30 [00:02<00:01,  7.02it/s]
 70%|#######   | 21/30 [00:03<00:01,  7.49it/s]
 70%|#######   | 21/30 [00:03<00:01,  7.49it/s]
 73%|#######3  | 22/30 [00:03<00:01,  7.71it/s]
 73%|#######3  | 22/30 [00:03<00:01,  7.71it/s]
 77%|#######6  | 23/30 [00:03<00:00,  7.70it/s]
 77%|#######6  | 23/30 [00:03<00:00,  7.70it/s]
 80%|########  | 24/30 [00:03<00:00,  7.73it/s]
 80%|########  | 24/30 [00:03<00:00,  7.73it/s]
 83%|########3 | 25/30 [00:03<00:00,  7.79it/s]
 83%|########3 | 25/30 [00:03<00:00,  7.79it/s]
 87%|########6 | 26/30 [00:03<00:00,  7.97it/s]
 87%|########6 | 26/30 [00:03<00:00,  7.97it/s]
 90%|######### | 27/30 [00:03<00:00,  7.56it/s]
 90%|######### | 27/30 [00:03<00:00,  7.56it/s]
 93%|#########3| 28/30 [00:04<00:00,  5.99it/s]
 93%|#########3| 28/30 [00:04<00:00,  5.99it/s]
 97%|#########6| 29/30 [00:04<00:00,  6.61it/s]
 97%|#########6| 29/30 [00:04<00:00,  6.61it/s]
+
+ - quantizing extras
+Text Encoder
+
+
+
+
+
+
+
+
+`torch_dtype` is deprecated! Use `dtype` instead!
+`torch_dtype` is deprecated! Use `dtype` instead!
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+Quantizing Text Encoder
+Loading VAE
+
+
+
+
+Making pipe
+Preparing Model
+Model Loaded
+create LoRA network. base dim (rank): 512, alpha: 512
+neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
+apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
+create LoRA for Text Encoder: 0 modules.
+create LoRA for U-Net: 240 modules.
+enable LoRA for U-Net
+Dataset: /workspace/ai-toolkit/datasets/qwe
+  -  Preprocessing image dimensions
+
  0%|          | 0/102 [00:00<?, ?it/s]
  0%|          | 0/102 [00:00<?, ?it/s]
  1%|          | 1/102 [00:00<00:39,  2.53it/s]
  1%|          | 1/102 [00:00<00:39,  2.53it/s]
  5%|4         | 5/102 [00:00<00:08, 11.65it/s]
  5%|4         | 5/102 [00:00<00:08, 11.65it/s]
  7%|6         | 7/102 [00:00<00:07, 12.38it/s]
  7%|6         | 7/102 [00:00<00:07, 12.38it/s]
  9%|8         | 9/102 [00:00<00:08, 10.48it/s]
  9%|8         | 9/102 [00:00<00:08, 10.48it/s]
 11%|#         | 11/102 [00:01<00:09,  9.61it/s]
 11%|#         | 11/102 [00:01<00:09,  9.61it/s]
 13%|#2        | 13/102 [00:01<00:09,  9.11it/s]
 13%|#2        | 13/102 [00:01<00:09,  9.11it/s]
 15%|#4        | 15/102 [00:01<00:09,  8.84it/s]
 15%|#4        | 15/102 [00:01<00:09,  8.84it/s]
 16%|#5        | 16/102 [00:01<00:09,  8.81it/s]
 16%|#5        | 16/102 [00:01<00:09,  8.81it/s]
 17%|#6        | 17/102 [00:01<00:09,  8.80it/s]
 17%|#6        | 17/102 [00:01<00:09,  8.80it/s]
 18%|#7        | 18/102 [00:01<00:09,  8.78it/s]
 18%|#7        | 18/102 [00:01<00:09,  8.78it/s]
 19%|#8        | 19/102 [00:02<00:09,  8.67it/s]
 19%|#8        | 19/102 [00:02<00:09,  8.67it/s]
 20%|#9        | 20/102 [00:02<00:14,  5.79it/s]
 20%|#9        | 20/102 [00:02<00:14,  5.79it/s]
 21%|##        | 21/102 [00:02<00:13,  5.88it/s]
 21%|##        | 21/102 [00:02<00:13,  5.88it/s]
 22%|##1       | 22/102 [00:03<00:19,  4.01it/s]
 22%|##1       | 22/102 [00:03<00:19,  4.01it/s]
 23%|##2       | 23/102 [00:03<00:23,  3.42it/s]
 23%|##2       | 23/102 [00:03<00:23,  3.42it/s]
 24%|##3       | 24/102 [00:03<00:24,  3.15it/s]
 24%|##3       | 24/102 [00:03<00:24,  3.15it/s]
 25%|##4       | 25/102 [00:04<00:25,  3.00it/s]
 25%|##4       | 25/102 [00:04<00:25,  3.00it/s]
 25%|##5       | 26/102 [00:04<00:20,  3.72it/s]
 25%|##5       | 26/102 [00:04<00:20,  3.72it/s]
 27%|##7       | 28/102 [00:04<00:13,  5.59it/s]
 27%|##7       | 28/102 [00:04<00:13,  5.59it/s]
 28%|##8       | 29/102 [00:04<00:11,  6.17it/s]
 28%|##8       | 29/102 [00:04<00:11,  6.17it/s]
 29%|##9       | 30/102 [00:04<00:10,  6.62it/s]
 29%|##9       | 30/102 [00:04<00:10,  6.62it/s]
 30%|###       | 31/102 [00:05<00:15,  4.46it/s]
 30%|###       | 31/102 [00:05<00:15,  4.46it/s]
 31%|###1      | 32/102 [00:05<00:19,  3.51it/s]
 31%|###1      | 32/102 [00:05<00:19,  3.51it/s]
 32%|###2      | 33/102 [00:05<00:16,  4.23it/s]
 32%|###2      | 33/102 [00:05<00:16,  4.23it/s]
 33%|###3      | 34/102 [00:05<00:13,  4.92it/s]
 33%|###3      | 34/102 [00:05<00:13,  4.92it/s]
+
+  -  Found 102 images
+Bucket sizes for /workspace/ai-toolkit/datasets/qwe:
+1024x1024: 102 files
+1 buckets made
+Generating baseline samples before training
+Unloading assistant lora
+
                                     
                                     
+
+
+========================================
+Result:
+ - 0 completed jobs
+ - 1 failure
+========================================
+Traceback (most recent call last):
+Traceback (most recent call last):
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+  File "/workspace/ai-toolkit/run.py", line 120, in <module>
+        main()main()
+
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+  File "/workspace/ai-toolkit/run.py", line 108, in main
+        raise eraise e
+
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+  File "/workspace/ai-toolkit/run.py", line 96, in main
+        job.run()job.run()
+
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+  File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
+        process.run()process.run()
+
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2367, in run
+  File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2367, in run
+        self.end_step_hook()self.end_step_hook()
+
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 264, in end_step_hook
+        self.maybe_stop()self.maybe_stop()
+
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+  File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 147, in maybe_stop
+        raise Exception("Job stopped")raise Exception("Job stopped")
+
+ExceptionException: : Job stoppedJob stopped
+
+
+

permesk/loss_log.db

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c0337d988642e97a7526192f7103e81afd8e72c18dba8477a55274367fc66fb6
+size 565248

permesk/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2328395fd1c6cdc914e4c5243e6912d8ab915f9dda266bdaf7d1ca578f9b9b8a
+size 2764142821

permesk/permesk.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d0b710c3c5b07cdb0c4f906ed9e6d17787f68e34d1b0207715a4ae408d35046
+size 2721118744

permesk/permesk_000001000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3704464a431b381654db512017981f3d08e42e0702d7f5a90b0c2a48b84c7c33
+size 2721118744

permesk/permesk_000002000.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6d0b4ac99bb3db260e0b58fdb335781bebca65e4f493f9ec8d166a1499b13669
+size 2721118744

permesk/pid.txt

@@ -0,0 +1 @@
+4793