arhiteector commited on
Commit
6b5bd80
·
verified ·
1 Parent(s): f303b87

upload permesk checkpoints (ignore sqlite wal/shm)

Browse files
Files changed (40) hide show
  1. .gitattributes +1 -0
  2. yora/.job_config.json +128 -0
  3. yora/config.yaml +113 -0
  4. yora/log.txt +0 -0
  5. yora/logs/0_log.txt +241 -0
  6. yora/loss_log.db +3 -0
  7. yora/optimizer.pt +3 -0
  8. yora/pid.txt +1 -0
  9. yora/samples/1777745264328__000000000_0.jpg +3 -0
  10. yora/samples/1777745641029__000000000_0.jpg +0 -0
  11. yora/samples/1777746035410__000000200_0.jpg +0 -0
  12. yora/samples/1777746430207__000000400_0.jpg +0 -0
  13. yora/samples/1777746824302__000000600_0.jpg +0 -0
  14. yora/samples/1777747218100__000000800_0.jpg +0 -0
  15. yora/samples/1777747612089__000001000_0.jpg +0 -0
  16. yora/samples/1777748006627__000001200_0.jpg +0 -0
  17. yora/samples/1777748400347__000001400_0.jpg +0 -0
  18. yora/samples/1777748794502__000001600_0.jpg +0 -0
  19. yora/samples/1777749189004__000001800_0.jpg +0 -0
  20. yora/samples/1777749582461__000002000_0.jpg +0 -0
  21. yora/samples/1777749975475__000002200_0.jpg +0 -0
  22. yora/samples/1777750368742__000002400_0.jpg +0 -0
  23. yora/samples/1777750763820__000002600_0.jpg +0 -0
  24. yora/samples/1777751157407__000002800_0.jpg +0 -0
  25. yora/samples/1777751544369__000003000_0.jpg +0 -0
  26. yora/yora.safetensors +3 -0
  27. yora/yora_000000200.safetensors +3 -0
  28. yora/yora_000000400.safetensors +3 -0
  29. yora/yora_000000600.safetensors +3 -0
  30. yora/yora_000000800.safetensors +3 -0
  31. yora/yora_000001000.safetensors +3 -0
  32. yora/yora_000001200.safetensors +3 -0
  33. yora/yora_000001400.safetensors +3 -0
  34. yora/yora_000001600.safetensors +3 -0
  35. yora/yora_000001800.safetensors +3 -0
  36. yora/yora_000002000.safetensors +3 -0
  37. yora/yora_000002200.safetensors +3 -0
  38. yora/yora_000002400.safetensors +3 -0
  39. yora/yora_000002600.safetensors +3 -0
  40. yora/yora_000002800.safetensors +3 -0
.gitattributes CHANGED
@@ -193,3 +193,4 @@ run_r34_512_20260306_084725/samples/epoch_088.jpg filter=lfs diff=lfs merge=lfs
193
  run_r34_512_20260306_084725/samples/epoch_089.jpg filter=lfs diff=lfs merge=lfs -text
194
  run_r34_512_20260306_084725/samples/epoch_090.jpg filter=lfs diff=lfs merge=lfs -text
195
  permesk/loss_log.db filter=lfs diff=lfs merge=lfs -text
 
 
193
  run_r34_512_20260306_084725/samples/epoch_089.jpg filter=lfs diff=lfs merge=lfs -text
194
  run_r34_512_20260306_084725/samples/epoch_090.jpg filter=lfs diff=lfs merge=lfs -text
195
  permesk/loss_log.db filter=lfs diff=lfs merge=lfs -text
196
+ yora/samples/1777745264328__000000000_0.jpg filter=lfs diff=lfs merge=lfs -text
yora/.job_config.json ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "job": "extension",
3
+ "config": {
4
+ "name": "yora",
5
+ "process": [
6
+ {
7
+ "type": "diffusion_trainer",
8
+ "training_folder": "/workspace/ai-toolkit/output",
9
+ "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
10
+ "device": "cuda",
11
+ "trigger_word": "vmvm",
12
+ "performance_log_every": 10,
13
+ "network": {
14
+ "type": "lora",
15
+ "linear": 32,
16
+ "linear_alpha": 32,
17
+ "conv": 16,
18
+ "conv_alpha": 16,
19
+ "lokr_full_rank": true,
20
+ "lokr_factor": -1,
21
+ "network_kwargs": {
22
+ "ignore_if_contains": []
23
+ }
24
+ },
25
+ "save": {
26
+ "dtype": "bf16",
27
+ "save_every": 200,
28
+ "max_step_saves_to_keep": 444444,
29
+ "save_format": "diffusers",
30
+ "push_to_hub": false
31
+ },
32
+ "datasets": [
33
+ {
34
+ "folder_path": "/workspace/ai-toolkit/datasets/qqqq",
35
+ "mask_path": null,
36
+ "mask_min_value": 0.1,
37
+ "default_caption": "",
38
+ "caption_ext": "txt",
39
+ "caption_dropout_rate": 0.05,
40
+ "cache_latents_to_disk": true,
41
+ "is_reg": false,
42
+ "network_weight": 1,
43
+ "resolution": [
44
+ 1024
45
+ ],
46
+ "controls": [],
47
+ "shrink_video_to_frames": true,
48
+ "num_frames": 1,
49
+ "flip_x": false,
50
+ "flip_y": false,
51
+ "num_repeats": 1
52
+ }
53
+ ],
54
+ "train": {
55
+ "batch_size": 1,
56
+ "bypass_guidance_embedding": false,
57
+ "steps": 3000,
58
+ "gradient_accumulation": 1,
59
+ "train_unet": true,
60
+ "train_text_encoder": false,
61
+ "gradient_checkpointing": true,
62
+ "noise_scheduler": "flowmatch",
63
+ "optimizer": "prodigyopt",
64
+ "timestep_type": "weighted",
65
+ "content_or_style": "balanced",
66
+ "optimizer_params": {
67
+ "weight_decay": 0.0001
68
+ },
69
+ "unload_text_encoder": false,
70
+ "cache_text_embeddings": true,
71
+ "lr": 1,
72
+ "ema_config": {
73
+ "use_ema": false,
74
+ "ema_decay": 0.99
75
+ },
76
+ "skip_first_sample": false,
77
+ "force_first_sample": false,
78
+ "disable_sampling": false,
79
+ "dtype": "bf16",
80
+ "diff_output_preservation": false,
81
+ "diff_output_preservation_multiplier": 1,
82
+ "diff_output_preservation_class": "person",
83
+ "switch_boundary_every": 1,
84
+ "loss_type": "mse"
85
+ },
86
+ "logging": {
87
+ "log_every": 1,
88
+ "use_ui_logger": true
89
+ },
90
+ "model": {
91
+ "name_or_path": "Tongyi-MAI/Z-Image",
92
+ "quantize": true,
93
+ "qtype": "qfloat8",
94
+ "quantize_te": true,
95
+ "qtype_te": "qfloat8",
96
+ "arch": "zimage",
97
+ "low_vram": true,
98
+ "model_kwargs": {},
99
+ "layer_offloading": false,
100
+ "layer_offloading_text_encoder_percent": 1,
101
+ "layer_offloading_transformer_percent": 1
102
+ },
103
+ "sample": {
104
+ "sampler": "flowmatch",
105
+ "sample_every": 200,
106
+ "width": 1024,
107
+ "height": 1024,
108
+ "samples": [
109
+ {
110
+ "prompt": "A person with messy, shoulder-length black hair stands facing the camera in a relaxed, neutral pose with arms resting naturally at their sides. They are wearing a fitted, short-sleeve black outfit with a minimalist, sleek design. The setting is a cozy indoor living room with warm ambient lighting. Behind them is a beige couch with soft cushions, a wooden floor, and a small side table holding a closed book and a charging cable. A floor lamp casts a soft, warm glow, creating a calm and intimate atmosphere. The composition is centered and symmetrical, with the subject framed from head to mid-thigh, giving a balanced and slightly cinematic look."
111
+ }
112
+ ],
113
+ "neg": "",
114
+ "seed": 42,
115
+ "walk_seed": true,
116
+ "guidance_scale": 4,
117
+ "sample_steps": 30,
118
+ "num_frames": 1,
119
+ "fps": 1
120
+ }
121
+ }
122
+ ]
123
+ },
124
+ "meta": {
125
+ "name": "[name]",
126
+ "version": "1.0"
127
+ }
128
+ }
yora/config.yaml ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ job: extension
2
+ config:
3
+ name: yora
4
+ process:
5
+ - type: diffusion_trainer
6
+ training_folder: /workspace/ai-toolkit/output
7
+ sqlite_db_path: /workspace/ai-toolkit/aitk_db.db
8
+ device: cuda
9
+ trigger_word: vmvm
10
+ performance_log_every: 10
11
+ network:
12
+ type: lora
13
+ linear: 32
14
+ linear_alpha: 32
15
+ conv: 16
16
+ conv_alpha: 16
17
+ lokr_full_rank: true
18
+ lokr_factor: -1
19
+ network_kwargs:
20
+ ignore_if_contains: []
21
+ save:
22
+ dtype: bf16
23
+ save_every: 200
24
+ max_step_saves_to_keep: 444444
25
+ save_format: diffusers
26
+ push_to_hub: false
27
+ datasets:
28
+ - folder_path: /workspace/ai-toolkit/datasets/qqqq
29
+ mask_path: null
30
+ mask_min_value: 0.1
31
+ default_caption: ''
32
+ caption_ext: txt
33
+ caption_dropout_rate: 0.05
34
+ cache_latents_to_disk: true
35
+ is_reg: false
36
+ network_weight: 1
37
+ resolution:
38
+ - 1024
39
+ controls: []
40
+ shrink_video_to_frames: true
41
+ num_frames: 1
42
+ flip_x: false
43
+ flip_y: false
44
+ num_repeats: 1
45
+ train:
46
+ batch_size: 1
47
+ bypass_guidance_embedding: false
48
+ steps: 3000
49
+ gradient_accumulation: 1
50
+ train_unet: true
51
+ train_text_encoder: false
52
+ gradient_checkpointing: true
53
+ noise_scheduler: flowmatch
54
+ optimizer: prodigyopt
55
+ timestep_type: weighted
56
+ content_or_style: balanced
57
+ optimizer_params:
58
+ weight_decay: 0.0001
59
+ unload_text_encoder: false
60
+ cache_text_embeddings: true
61
+ lr: 1
62
+ ema_config:
63
+ use_ema: false
64
+ ema_decay: 0.99
65
+ skip_first_sample: false
66
+ force_first_sample: false
67
+ disable_sampling: false
68
+ dtype: bf16
69
+ diff_output_preservation: false
70
+ diff_output_preservation_multiplier: 1
71
+ diff_output_preservation_class: person
72
+ switch_boundary_every: 1
73
+ loss_type: mse
74
+ logging:
75
+ log_every: 1
76
+ use_ui_logger: true
77
+ model:
78
+ name_or_path: Tongyi-MAI/Z-Image
79
+ quantize: true
80
+ qtype: qfloat8
81
+ quantize_te: true
82
+ qtype_te: qfloat8
83
+ arch: zimage
84
+ low_vram: true
85
+ model_kwargs: {}
86
+ layer_offloading: false
87
+ layer_offloading_text_encoder_percent: 1
88
+ layer_offloading_transformer_percent: 1
89
+ sample:
90
+ sampler: flowmatch
91
+ sample_every: 200
92
+ width: 1024
93
+ height: 1024
94
+ samples:
95
+ - prompt: A person with messy, shoulder-length black hair stands facing the
96
+ camera in a relaxed, neutral pose with arms resting naturally at their sides.
97
+ They are wearing a fitted, short-sleeve black outfit with a minimalist,
98
+ sleek design. The setting is a cozy indoor living room with warm ambient
99
+ lighting. Behind them is a beige couch with soft cushions, a wooden floor,
100
+ and a small side table holding a closed book and a charging cable. A floor
101
+ lamp casts a soft, warm glow, creating a calm and intimate atmosphere. The
102
+ composition is centered and symmetrical, with the subject framed from head
103
+ to mid-thigh, giving a balanced and slightly cinematic look.
104
+ neg: ''
105
+ seed: 42
106
+ walk_seed: true
107
+ guidance_scale: 4
108
+ sample_steps: 30
109
+ num_frames: 1
110
+ fps: 1
111
+ meta:
112
+ name: yora
113
+ version: '1.0'
yora/log.txt ADDED
The diff for this file is too large to render. See raw diff
 
yora/logs/0_log.txt ADDED
@@ -0,0 +1,241 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
0
  0%| | 0/30 [00:00<?, ?it/s]
1
  0%| | 0/30 [00:00<?, ?it/s]
2
  3%|3 | 1/30 [00:00<00:09, 3.11it/s]
3
  3%|3 | 1/30 [00:00<00:09, 3.11it/s]
4
  7%|6 | 2/30 [00:00<00:06, 4.35it/s]
5
  7%|6 | 2/30 [00:00<00:06, 4.35it/s]
6
  10%|# | 3/30 [00:00<00:05, 5.07it/s]
7
  10%|# | 3/30 [00:00<00:05, 5.07it/s]
8
  13%|#3 | 4/30 [00:00<00:04, 5.49it/s]
9
  13%|#3 | 4/30 [00:00<00:04, 5.49it/s]
10
  17%|#6 | 5/30 [00:00<00:04, 5.75it/s]
11
  17%|#6 | 5/30 [00:00<00:04, 5.75it/s]
12
  20%|## | 6/30 [00:01<00:04, 5.86it/s]
13
  20%|## | 6/30 [00:01<00:04, 5.86it/s]
14
  23%|##3 | 7/30 [00:01<00:03, 5.98it/s]
15
  23%|##3 | 7/30 [00:01<00:03, 5.98it/s]
16
  27%|##6 | 8/30 [00:01<00:03, 6.04it/s]
17
  27%|##6 | 8/30 [00:01<00:03, 6.04it/s]
18
  30%|### | 9/30 [00:01<00:03, 6.01it/s]
19
  30%|### | 9/30 [00:01<00:03, 6.01it/s]
20
  33%|###3 | 10/30 [00:01<00:03, 6.06it/s]
21
  33%|###3 | 10/30 [00:01<00:03, 6.06it/s]
22
  37%|###6 | 11/30 [00:01<00:03, 6.13it/s]
23
  37%|###6 | 11/30 [00:01<00:03, 6.13it/s]
24
  40%|#### | 12/30 [00:02<00:02, 6.15it/s]
25
  40%|#### | 12/30 [00:02<00:02, 6.15it/s]
26
  43%|####3 | 13/30 [00:02<00:02, 6.19it/s]
27
  43%|####3 | 13/30 [00:02<00:02, 6.19it/s]
28
  47%|####6 | 14/30 [00:02<00:02, 6.16it/s]
29
  47%|####6 | 14/30 [00:02<00:02, 6.16it/s]
30
  50%|##### | 15/30 [00:02<00:02, 6.09it/s]
31
  50%|##### | 15/30 [00:02<00:02, 6.09it/s]
32
  53%|#####3 | 16/30 [00:02<00:02, 6.15it/s]
33
  53%|#####3 | 16/30 [00:02<00:02, 6.15it/s]
34
  57%|#####6 | 17/30 [00:02<00:02, 6.18it/s]
35
  57%|#####6 | 17/30 [00:02<00:02, 6.18it/s]
36
  60%|###### | 18/30 [00:03<00:01, 6.20it/s]
37
  60%|###### | 18/30 [00:03<00:01, 6.20it/s]
38
  63%|######3 | 19/30 [00:03<00:01, 6.21it/s]
39
  63%|######3 | 19/30 [00:03<00:01, 6.21it/s]
40
  67%|######6 | 20/30 [00:03<00:01, 6.22it/s]
41
  67%|######6 | 20/30 [00:03<00:01, 6.22it/s]
42
  70%|####### | 21/30 [00:03<00:01, 6.22it/s]
43
  70%|####### | 21/30 [00:03<00:01, 6.22it/s]
44
  73%|#######3 | 22/30 [00:03<00:01, 5.95it/s]
45
  73%|#######3 | 22/30 [00:03<00:01, 5.95it/s]
46
  77%|#######6 | 23/30 [00:03<00:01, 5.95it/s]
47
  77%|#######6 | 23/30 [00:03<00:01, 5.95it/s]
48
  80%|######## | 24/30 [00:04<00:01, 5.97it/s]
49
  80%|######## | 24/30 [00:04<00:01, 5.97it/s]
50
  83%|########3 | 25/30 [00:04<00:00, 6.06it/s]
51
  83%|########3 | 25/30 [00:04<00:00, 6.06it/s]
52
  87%|########6 | 26/30 [00:04<00:00, 6.12it/s]
53
  87%|########6 | 26/30 [00:04<00:00, 6.12it/s]
54
  90%|######### | 27/30 [00:04<00:00, 6.16it/s]
55
  90%|######### | 27/30 [00:04<00:00, 6.16it/s]
56
  93%|#########3| 28/30 [00:04<00:00, 6.19it/s]
57
  93%|#########3| 28/30 [00:04<00:00, 6.19it/s]
58
  97%|#########6| 29/30 [00:04<00:00, 6.21it/s]
59
  97%|#########6| 29/30 [00:04<00:00, 6.21it/s]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
60
  0%| | 0/60 [00:00<?, ?it/s]
61
  0%| | 0/60 [00:00<?, ?it/s]
62
  8%|8 | 5/60 [00:00<00:01, 48.56it/s]
63
  8%|8 | 5/60 [00:00<00:01, 48.56it/s]
64
  23%|##3 | 14/60 [00:00<00:00, 71.48it/s]
65
  23%|##3 | 14/60 [00:00<00:00, 71.48it/s]
66
  40%|#### | 24/60 [00:00<00:00, 81.62it/s]
67
  40%|#### | 24/60 [00:00<00:00, 81.62it/s]
68
  55%|#####5 | 33/60 [00:00<00:00, 84.47it/s]
69
  55%|#####5 | 33/60 [00:00<00:00, 84.47it/s]
70
  70%|####### | 42/60 [00:00<00:00, 86.04it/s]
71
  70%|####### | 42/60 [00:00<00:00, 86.04it/s]
72
  87%|########6 | 52/60 [00:00<00:00, 88.68it/s]
73
  87%|########6 | 52/60 [00:00<00:00, 88.68it/s]
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
74
 
75
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ Running 1 job
2
+ {
3
+ "type": "diffusion_trainer",
4
+ "training_folder": "/workspace/ai-toolkit/output",
5
+ "sqlite_db_path": "/workspace/ai-toolkit/aitk_db.db",
6
+ "device": "cuda",
7
+ "trigger_word": "vmvm",
8
+ "performance_log_every": 10,
9
+ "network": {
10
+ "type": "lora",
11
+ "linear": 32,
12
+ "linear_alpha": 32,
13
+ "conv": 16,
14
+ "conv_alpha": 16,
15
+ "lokr_full_rank": true,
16
+ "lokr_factor": -1,
17
+ "network_kwargs": {
18
+ "ignore_if_contains": []
19
+ }
20
+ },
21
+ "save": {
22
+ "dtype": "bf16",
23
+ "save_every": 200,
24
+ "max_step_saves_to_keep": 444444,
25
+ "save_format": "diffusers",
26
+ "push_to_hub": false
27
+ },
28
+ "datasets": [
29
+ {
30
+ "folder_path": "/workspace/ai-toolkit/datasets/qqqq",
31
+ "mask_path": null,
32
+ "mask_min_value": 0.1,
33
+ "default_caption": "",
34
+ "caption_ext": "txt",
35
+ "caption_dropout_rate": 0.05,
36
+ "cache_latents_to_disk": true,
37
+ "is_reg": false,
38
+ "network_weight": 1,
39
+ "resolution": [
40
+ 1024
41
+ ],
42
+ "controls": [],
43
+ "shrink_video_to_frames": true,
44
+ "num_frames": 1,
45
+ "flip_x": false,
46
+ "flip_y": false,
47
+ "num_repeats": 1
48
+ }
49
+ ],
50
+ "train": {
51
+ "batch_size": 1,
52
+ "bypass_guidance_embedding": false,
53
+ "steps": 3000,
54
+ "gradient_accumulation": 1,
55
+ "train_unet": true,
56
+ "train_text_encoder": false,
57
+ "gradient_checkpointing": true,
58
+ "noise_scheduler": "flowmatch",
59
+ "optimizer": "prodigyopt",
60
+ "timestep_type": "weighted",
61
+ "content_or_style": "balanced",
62
+ "optimizer_params": {
63
+ "weight_decay": 0.0001
64
+ },
65
+ "unload_text_encoder": false,
66
+ "cache_text_embeddings": true,
67
+ "lr": 1,
68
+ "ema_config": {
69
+ "use_ema": false,
70
+ "ema_decay": 0.99
71
+ },
72
+ "skip_first_sample": false,
73
+ "force_first_sample": false,
74
+ "disable_sampling": false,
75
+ "dtype": "bf16",
76
+ "diff_output_preservation": false,
77
+ "diff_output_preservation_multiplier": 1,
78
+ "diff_output_preservation_class": "person",
79
+ "switch_boundary_every": 1,
80
+ "loss_type": "mse"
81
+ },
82
+ "logging": {
83
+ "log_every": 1,
84
+ "use_ui_logger": true
85
+ },
86
+ "model": {
87
+ "name_or_path": "Tongyi-MAI/Z-Image",
88
+ "quantize": true,
89
+ "qtype": "qfloat8",
90
+ "quantize_te": true,
91
+ "qtype_te": "qfloat8",
92
+ "arch": "zimage",
93
+ "low_vram": true,
94
+ "model_kwargs": {},
95
+ "layer_offloading": false,
96
+ "layer_offloading_text_encoder_percent": 1,
97
+ "layer_offloading_transformer_percent": 1
98
+ },
99
+ "sample": {
100
+ "sampler": "flowmatch",
101
+ "sample_every": 200,
102
+ "width": 1024,
103
+ "height": 1024,
104
+ "samples": [
105
+ {
106
+ "prompt": "woman with black hair taking a selfie in her bedroom"
107
+ }
108
+ ],
109
+ "neg": "",
110
+ "seed": 42,
111
+ "walk_seed": true,
112
+ "guidance_scale": 4,
113
+ "sample_steps": 30,
114
+ "num_frames": 1,
115
+ "fps": 1
116
+ }
117
+ }
118
+ Using SQLite database at /workspace/ai-toolkit/aitk_db.db
119
+ Job ID: "43ca7731-71a5-45fa-914f-727d11aef207"
120
+
121
+ #############################################
122
+ # Running job: yora
123
+ #############################################
124
+
125
+
126
+ Running 1 process
127
+ Loading ZImage model
128
+ Loading transformer
129
+
130
+
131
+
132
+
133
+ Quantizing Transformer
134
+ - quantizing 30 transformer blocks
135
+
136
  0%| | 0/30 [00:00<?, ?it/s]
137
  0%| | 0/30 [00:00<?, ?it/s]
138
  3%|3 | 1/30 [00:00<00:09, 3.11it/s]
139
  3%|3 | 1/30 [00:00<00:09, 3.11it/s]
140
  7%|6 | 2/30 [00:00<00:06, 4.35it/s]
141
  7%|6 | 2/30 [00:00<00:06, 4.35it/s]
142
  10%|# | 3/30 [00:00<00:05, 5.07it/s]
143
  10%|# | 3/30 [00:00<00:05, 5.07it/s]
144
  13%|#3 | 4/30 [00:00<00:04, 5.49it/s]
145
  13%|#3 | 4/30 [00:00<00:04, 5.49it/s]
146
  17%|#6 | 5/30 [00:00<00:04, 5.75it/s]
147
  17%|#6 | 5/30 [00:00<00:04, 5.75it/s]
148
  20%|## | 6/30 [00:01<00:04, 5.86it/s]
149
  20%|## | 6/30 [00:01<00:04, 5.86it/s]
150
  23%|##3 | 7/30 [00:01<00:03, 5.98it/s]
151
  23%|##3 | 7/30 [00:01<00:03, 5.98it/s]
152
  27%|##6 | 8/30 [00:01<00:03, 6.04it/s]
153
  27%|##6 | 8/30 [00:01<00:03, 6.04it/s]
154
  30%|### | 9/30 [00:01<00:03, 6.01it/s]
155
  30%|### | 9/30 [00:01<00:03, 6.01it/s]
156
  33%|###3 | 10/30 [00:01<00:03, 6.06it/s]
157
  33%|###3 | 10/30 [00:01<00:03, 6.06it/s]
158
  37%|###6 | 11/30 [00:01<00:03, 6.13it/s]
159
  37%|###6 | 11/30 [00:01<00:03, 6.13it/s]
160
  40%|#### | 12/30 [00:02<00:02, 6.15it/s]
161
  40%|#### | 12/30 [00:02<00:02, 6.15it/s]
162
  43%|####3 | 13/30 [00:02<00:02, 6.19it/s]
163
  43%|####3 | 13/30 [00:02<00:02, 6.19it/s]
164
  47%|####6 | 14/30 [00:02<00:02, 6.16it/s]
165
  47%|####6 | 14/30 [00:02<00:02, 6.16it/s]
166
  50%|##### | 15/30 [00:02<00:02, 6.09it/s]
167
  50%|##### | 15/30 [00:02<00:02, 6.09it/s]
168
  53%|#####3 | 16/30 [00:02<00:02, 6.15it/s]
169
  53%|#####3 | 16/30 [00:02<00:02, 6.15it/s]
170
  57%|#####6 | 17/30 [00:02<00:02, 6.18it/s]
171
  57%|#####6 | 17/30 [00:02<00:02, 6.18it/s]
172
  60%|###### | 18/30 [00:03<00:01, 6.20it/s]
173
  60%|###### | 18/30 [00:03<00:01, 6.20it/s]
174
  63%|######3 | 19/30 [00:03<00:01, 6.21it/s]
175
  63%|######3 | 19/30 [00:03<00:01, 6.21it/s]
176
  67%|######6 | 20/30 [00:03<00:01, 6.22it/s]
177
  67%|######6 | 20/30 [00:03<00:01, 6.22it/s]
178
  70%|####### | 21/30 [00:03<00:01, 6.22it/s]
179
  70%|####### | 21/30 [00:03<00:01, 6.22it/s]
180
  73%|#######3 | 22/30 [00:03<00:01, 5.95it/s]
181
  73%|#######3 | 22/30 [00:03<00:01, 5.95it/s]
182
  77%|#######6 | 23/30 [00:03<00:01, 5.95it/s]
183
  77%|#######6 | 23/30 [00:03<00:01, 5.95it/s]
184
  80%|######## | 24/30 [00:04<00:01, 5.97it/s]
185
  80%|######## | 24/30 [00:04<00:01, 5.97it/s]
186
  83%|########3 | 25/30 [00:04<00:00, 6.06it/s]
187
  83%|########3 | 25/30 [00:04<00:00, 6.06it/s]
188
  87%|########6 | 26/30 [00:04<00:00, 6.12it/s]
189
  87%|########6 | 26/30 [00:04<00:00, 6.12it/s]
190
  90%|######### | 27/30 [00:04<00:00, 6.16it/s]
191
  90%|######### | 27/30 [00:04<00:00, 6.16it/s]
192
  93%|#########3| 28/30 [00:04<00:00, 6.19it/s]
193
  93%|#########3| 28/30 [00:04<00:00, 6.19it/s]
194
  97%|#########6| 29/30 [00:04<00:00, 6.21it/s]
195
  97%|#########6| 29/30 [00:04<00:00, 6.21it/s]
196
+
197
+ - quantizing extras
198
+ Moving transformer to CPU
199
+ Text Encoder
200
+
201
+
202
+
203
+
204
+ Quantizing Text Encoder
205
+ Loading VAE
206
+ Making pipe
207
+ Preparing Model
208
+ Model Loaded
209
+ create LoRA network. base dim (rank): 32, alpha: 32
210
+ neuron dropout: p=None, rank dropout: p=None, module dropout: p=None
211
+ apply LoRA to Conv2d with kernel size (3,3). dim (rank): 16, alpha: 16
212
+ create LoRA for Text Encoder: 0 modules.
213
+ create LoRA for U-Net: 240 modules.
214
+ enable LoRA for U-Net
215
+ Using Prodigy optimizer
216
+ Using lr 1
217
+ Using decoupled weight decay
218
+ Dataset: /workspace/ai-toolkit/datasets/qqqq
219
+ - Preprocessing image dimensions
220
+
221
  0%| | 0/60 [00:00<?, ?it/s]
222
  0%| | 0/60 [00:00<?, ?it/s]
223
  8%|8 | 5/60 [00:00<00:01, 48.56it/s]
224
  8%|8 | 5/60 [00:00<00:01, 48.56it/s]
225
  23%|##3 | 14/60 [00:00<00:00, 71.48it/s]
226
  23%|##3 | 14/60 [00:00<00:00, 71.48it/s]
227
  40%|#### | 24/60 [00:00<00:00, 81.62it/s]
228
  40%|#### | 24/60 [00:00<00:00, 81.62it/s]
229
  55%|#####5 | 33/60 [00:00<00:00, 84.47it/s]
230
  55%|#####5 | 33/60 [00:00<00:00, 84.47it/s]
231
  70%|####### | 42/60 [00:00<00:00, 86.04it/s]
232
  70%|####### | 42/60 [00:00<00:00, 86.04it/s]
233
  87%|########6 | 52/60 [00:00<00:00, 88.68it/s]
234
  87%|########6 | 52/60 [00:00<00:00, 88.68it/s]
235
+
236
+ - Found 60 images
237
+ Bucket sizes for /workspace/ai-toolkit/datasets/qqqq:
238
+ 832x1152: 60 files
239
+ 1 buckets made
240
+ Caching latents for /workspace/ai-toolkit/datasets/qqqq
241
+ - Saving latents to disk
242
+
243
+
244
+ Caching text_embeddings for /workspace/ai-toolkit/datasets/qqqq
245
+ - Saving text embeddings to disk
246
+
247
+
248
+ Caching embeddings and unloading text encoder
249
+
250
+ ***** UNLOADING TEXT ENCODER *****
251
+ Embeddings cached to disk. We dont need the text encoder anymore
252
+ ***********************************
253
+
254
+ Generating baseline samples before training
255
+
256
 
257
 
258
+ ========================================
259
+ Result:
260
+ - 0 completed jobs
261
+ ========================================
262
+ Traceback (most recent call last):
263
+ Traceback (most recent call last):
264
+ File "/workspace/ai-toolkit/run.py", line 135, in <module>
265
+ File "/workspace/ai-toolkit/run.py", line 135, in <module>
266
+ main()main()
267
+
268
+ File "/workspace/ai-toolkit/run.py", line 131, in main
269
+ File "/workspace/ai-toolkit/run.py", line 131, in main
270
+ raise eraise e
271
+
272
+ File "/workspace/ai-toolkit/run.py", line 111, in main
273
+ File "/workspace/ai-toolkit/run.py", line 111, in main
274
+ job.run()job.run()
275
+
276
+ File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
277
+ File "/workspace/ai-toolkit/jobs/ExtensionJob.py", line 22, in run
278
+ process.run()process.run()
279
+
280
+ File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2224, in run
281
+ File "/workspace/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2224, in run
282
+ loss_dict = self.hook_train_loop(batch_list)loss_dict = self.hook_train_loop(batch_list)
283
+
284
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
285
+
286
+ File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2098, in hook_train_loop
287
+ File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2098, in hook_train_loop
288
+ loss = self.train_single_accumulation(batch)loss = self.train_single_accumulation(batch)
289
+
290
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
291
+
292
+ File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2073, in train_single_accumulation
293
+ File "/workspace/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2073, in train_single_accumulation
294
+ self.accelerator.backward(loss)self.accelerator.backward(loss)
295
+
296
+ File "/venv/main/lib/python3.12/site-packages/accelerate/accelerator.py", line 2838, in backward
297
+ File "/venv/main/lib/python3.12/site-packages/accelerate/accelerator.py", line 2838, in backward
298
+ loss.backward(**kwargs)loss.backward(**kwargs)
299
+
300
+ File "/venv/main/lib/python3.12/site-packages/torch/_tensor.py", line 648, in backward
301
+ File "/venv/main/lib/python3.12/site-packages/torch/_tensor.py", line 648, in backward
302
+ torch.autograd.backward(torch.autograd.backward(
303
+
304
+ File "/venv/main/lib/python3.12/site-packages/torch/autograd/__init__.py", line 353, in backward
305
+ File "/venv/main/lib/python3.12/site-packages/torch/autograd/__init__.py", line 353, in backward
306
+ _engine_run_backward(_engine_run_backward(
307
+
308
+ File "/venv/main/lib/python3.12/site-packages/torch/autograd/graph.py", line 824, in _engine_run_backward
309
+ File "/venv/main/lib/python3.12/site-packages/torch/autograd/graph.py", line 824, in _engine_run_backward
310
+ return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward passreturn Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
311
+
312
+ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
313
+
314
+ KeyboardInterruptKeyboardInterrupt
315
+
316
+
317
+
yora/loss_log.db ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cfe836cfd837826f6bdd4e9c6abe5f214a9f82775e9bebc60b9604c8e14153a9
3
+ size 565248
yora/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c5072050e28689f9302de57a0b17d29f89ff4ad903f1b6634d591c99f6508732
3
+ size 1149731555
yora/pid.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 4356
yora/samples/1777745264328__000000000_0.jpg ADDED

Git LFS Details

  • SHA256: dbfa1526f6c5963b907cb6ad2052a5c52fb7b1a24582e023a360595b456e76cb
  • Pointer size: 131 Bytes
  • Size of remote file: 117 kB
yora/samples/1777745641029__000000000_0.jpg ADDED
yora/samples/1777746035410__000000200_0.jpg ADDED
yora/samples/1777746430207__000000400_0.jpg ADDED
yora/samples/1777746824302__000000600_0.jpg ADDED
yora/samples/1777747218100__000000800_0.jpg ADDED
yora/samples/1777747612089__000001000_0.jpg ADDED
yora/samples/1777748006627__000001200_0.jpg ADDED
yora/samples/1777748400347__000001400_0.jpg ADDED
yora/samples/1777748794502__000001600_0.jpg ADDED
yora/samples/1777749189004__000001800_0.jpg ADDED
yora/samples/1777749582461__000002000_0.jpg ADDED
yora/samples/1777749975475__000002200_0.jpg ADDED
yora/samples/1777750368742__000002400_0.jpg ADDED
yora/samples/1777750763820__000002600_0.jpg ADDED
yora/samples/1777751157407__000002800_0.jpg ADDED
yora/samples/1777751544369__000003000_0.jpg ADDED
yora/yora.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:791ee07aefe803262ba862b1028ee447fa0d0cd94ad37ab99bb8cf4ae59ad6a6
3
+ size 170128256
yora/yora_000000200.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aa29660034cf953b68ebf77825b12973eee47d09ea42d703a42b923c730b0a97
3
+ size 170128256
yora/yora_000000400.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0f43fa830fe9f6025c019787a18b5b9a45b71ab2567509038025cb544aa1edde
3
+ size 170128256
yora/yora_000000600.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e7bb9415e13e932fe7d11b59932035453d9f9f485edebae4496dfd473b8fe297
3
+ size 170128256
yora/yora_000000800.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:665777d7ac6d3b9d57348dca964ec34aa35964d40b73814c55b06894a8ec09da
3
+ size 170128256
yora/yora_000001000.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:636618406a107d2dd6fdbdd5ba82291c872952b8effddd674be70317d92b58fa
3
+ size 170128256
yora/yora_000001200.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:81f383afb9922a5d9be5e23572b19c17534c285dedabf439fa2ea09a48381b7d
3
+ size 170128256
yora/yora_000001400.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:cd372fe8cc853b438b5dcec60507c2e64e82789411b32c53468efcf9a3310d8b
3
+ size 170128256
yora/yora_000001600.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9b52a49dfcc4bb6f487635f46eb03b2d4759de57a3d89ec92f7f87c9bff2db3
3
+ size 170128256
yora/yora_000001800.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4b9c4d237ea8f22366a88d2ba9951ff7424653ea0dafaca399346ab9a0489a23
3
+ size 170128256
yora/yora_000002000.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a21c40e4af8dff12f7bcaa5b0a88cb9c06e55815329d4b50e4c676a812ae7965
3
+ size 170128256
yora/yora_000002200.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3161df883d3d0322fcec18b08ee555e7909e84034827825f53e416f0aa99286c
3
+ size 170128256
yora/yora_000002400.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a60b2041bd8745cd37b70f43ff85e6f6597a4e1ccc037ce4af65ad58952eae3
3
+ size 170128256
yora/yora_000002600.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0b3a099c13ea3045225ecdb4135f6ccb1233585243c5c0116f2bbcec6dcec660
3
+ size 170128256
yora/yora_000002800.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e425d96e4ac2d397294b190cdd63dda87d9a7e3e2d6af4b1e6b734ce86a896a1
3
+ size 170128256