Edapinenut commited on
Commit
e017adb
·
verified ·
1 Parent(s): 1beca34

Add files using upload-large-folder tool

Browse files
Files changed (37) hide show
  1. SHA256SUMS +19 -0
  2. aux_vel/mix/aux_v_transformer/config.json +31 -0
  3. aux_vel/mix/aux_v_transformer/diffusion_pytorch_model.safetensors +3 -0
  4. aux_vel/mix/config.yaml +193 -0
  5. aux_vel/ocr/aux_v_transformer/config.json +31 -0
  6. aux_vel/ocr/aux_v_transformer/diffusion_pytorch_model.safetensors +3 -0
  7. aux_vel/ocr/config.yaml +216 -0
  8. awm_mix/checkpoint-400/generator_ema.pt +3 -0
  9. awm_mix/config.yaml +227 -0
  10. dmdawm_mix/checkpoint-400/generator_ema.pt +3 -0
  11. dmdawm_mix/config.yaml +229 -0
  12. dmdawm_ocr/checkpoint-400/generator_ema.pt +3 -0
  13. dmdawm_ocr/config.yaml +223 -0
  14. dmdnft_mix/checkpoint-400/generator_ema.pt +3 -0
  15. dmdnft_mix/config.yaml +202 -0
  16. dmdnft_ocr/checkpoint-200/generator_ema.pt +3 -0
  17. dmdnft_ocr/config.yaml +196 -0
  18. fewstepgrpo_mix/checkpoint-300/generator_ema.pt +3 -0
  19. fewstepgrpo_mix/config.yaml +206 -0
  20. fewstepgrpo_ocr/checkpoint-200/generator_ema.pt +3 -0
  21. fewstepgrpo_ocr/config.yaml +200 -0
  22. flowgrpo_mix/checkpoint-400/generator_ema.pt +3 -0
  23. flowgrpo_mix/config.yaml +229 -0
  24. meanflowawm_mix/checkpoint-400/generator_ema.pt +3 -0
  25. meanflowawm_mix/config.yaml +218 -0
  26. meanflowawm_ocr/checkpoint-400/generator_ema.pt +3 -0
  27. meanflowawm_ocr/config.yaml +198 -0
  28. meanflownft_mix/checkpoint-400/generator_ema.pt +3 -0
  29. meanflownft_mix/config.yaml +186 -0
  30. meanflownft_ocr/checkpoint-200/generator_ema.pt +3 -0
  31. meanflownft_ocr/config.yaml +220 -0
  32. meanflowopsd_mix/checkpoint-400/generator_ema.pt +3 -0
  33. meanflowopsd_mix/config.yaml +226 -0
  34. nft_mix/checkpoint-400/generator_ema.pt +3 -0
  35. nft_mix/config.yaml +227 -0
  36. reflmx_mix/checkpoint-400/generator_ema.pt +3 -0
  37. reflmx_mix/config.yaml +228 -0
SHA256SUMS ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ e53d431683868bdcb78a41c6baa99def22e7a9365119cd2c144f65df9b44af5f ./aux_vel/mix/aux_v_transformer/config.json
2
+ e469599b2d996896e79a50b57c35d3a4c88f76724ea98b7e4cfd62f6e15b31a2 ./aux_vel/mix/aux_v_transformer/diffusion_pytorch_model.safetensors
3
+ e53d431683868bdcb78a41c6baa99def22e7a9365119cd2c144f65df9b44af5f ./aux_vel/ocr/aux_v_transformer/config.json
4
+ 1daab3d85296af852db3fa336c24894dad5d8aea2488902e53ab2ca4ece60e4e ./aux_vel/ocr/aux_v_transformer/diffusion_pytorch_model.safetensors
5
+ 101e8de5d74ce4f5dde4ef14fa31155e20556461ea49746792942d6857b3d33d ./awm_mix/checkpoint-400/generator_ema.pt
6
+ 308bf0e588143cad86900008d16ae21e7a64b63b53390e7fec0fcb674e48f841 ./dmdawm_mix/checkpoint-400/generator_ema.pt
7
+ 19bd328be85c84389c7170aa076f31a3390fa70204d4df3f757b42c65181dd6b ./dmdawm_ocr/checkpoint-400/generator_ema.pt
8
+ eaf13570b1ddb93b107a1939334fbe7bac4d1752a89e8ecdde0b01c282f8dab5 ./dmdnft_mix/checkpoint-400/generator_ema.pt
9
+ 34f69486ef7294ddb74ac68999bf68b3b6e16fc4e500e9e2eb4168c59772a876 ./dmdnft_ocr/checkpoint-200/generator_ema.pt
10
+ d80228738e50513c307e1ebfeef05eb5a36dfd20656338447df6ecf20171b5fe ./fewstepgrpo_mix/checkpoint-300/generator_ema.pt
11
+ 55221ce0596ef9447b89e8234404f8b13cd32c7eb815b31f5c21f5c665abfb44 ./fewstepgrpo_ocr/checkpoint-200/generator_ema.pt
12
+ 3b9745b3d478b25280bc4d5fb3056900b8d0a17a71b5d6a5d68e8704b3c2f155 ./flowgrpo_mix/checkpoint-400/generator_ema.pt
13
+ e1dfe852ff3e154de07facac5f64ec2a671311df2c032e3949745f641da5e6e2 ./meanflowawm_mix/checkpoint-400/generator_ema.pt
14
+ c4c3b1b904c85c680671e6736070cf64b7a4ba408985dd2055f80703766fe9c8 ./meanflowawm_ocr/checkpoint-400/generator_ema.pt
15
+ 596d25e0e6a7b8fe705a5c69c755f041d196d4a8327e781af43dc1136c8b5426 ./meanflownft_mix/checkpoint-400/generator_ema.pt
16
+ 2a3252faaa568aa84318098be22ba846bab7e69da219ad7645f9c7322a78466e ./meanflownft_ocr/checkpoint-200/generator_ema.pt
17
+ ccb3092eac0581ec9c48b9cb7b7f05b51e383611bc66f6742cbc90dc9e1793bb ./meanflowopsd_mix/checkpoint-400/generator_ema.pt
18
+ 6faf97432f041db6fcf03e0e688b753fe7ccfb2bb1cd5cdb77307b76f6f13419 ./nft_mix/checkpoint-400/generator_ema.pt
19
+ b25ffea29fef57ee39611bfa7d2987fd6308a628b57b34d7c35013452e05e489 ./reflmx_mix/checkpoint-400/generator_ema.pt
aux_vel/mix/aux_v_transformer/config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "SD3Transformer2DModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "attention_head_dim": 64,
5
+ "caption_projection_dim": 1536,
6
+ "dual_attention_layers": [
7
+ 0,
8
+ 1,
9
+ 2,
10
+ 3,
11
+ 4,
12
+ 5,
13
+ 6,
14
+ 7,
15
+ 8,
16
+ 9,
17
+ 10,
18
+ 11,
19
+ 12
20
+ ],
21
+ "in_channels": 16,
22
+ "joint_attention_dim": 4096,
23
+ "num_attention_heads": 24,
24
+ "num_layers": 24,
25
+ "out_channels": 16,
26
+ "patch_size": 2,
27
+ "pooled_projection_dim": 2048,
28
+ "pos_embed_max_size": 384,
29
+ "qk_norm": "rms_norm",
30
+ "sample_size": 128
31
+ }
aux_vel/mix/aux_v_transformer/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e469599b2d996896e79a50b57c35d3a4c88f76724ea98b7e4cfd62f6e15b31a2
3
+ size 9878760864
aux_vel/mix/config.yaml ADDED
@@ -0,0 +1,193 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 100
82
+ num_batches_per_epoch: 1
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 8
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0
97
+ decay_type: 1
98
+ rl_decay_interval: 1
99
+ reward_fn: {}
100
+ reward_ckpt_path: ''
101
+ log_sample_images: 0
102
+ central_diff_epsilon: 5.0
103
+ cd_velocity_source: noise_minus_x0
104
+ share_cd_with_old: true
105
+ num_training_timesteps_per_sample: 4
106
+ rl_min_timestep: 0
107
+ rl_max_timestep: 1000
108
+ diffusion_ratio: 0.25
109
+ consistency_ratio: 0.25
110
+ use_ref_v_model: false
111
+ guidance_scale: 1.0
112
+ aux_v_enabled: true
113
+ max_aux_sample: 80
114
+ renoise_per_epoch: 10
115
+ aux_v_gradient_accum: 1
116
+ aux_v_batch_size: 8
117
+ solver:
118
+ generator:
119
+ lr: 3.0e-06
120
+ warmup_steps: 0
121
+ beta1: 0.9
122
+ beta2: 0.999
123
+ eps: 1.0e-08
124
+ weight_decay: 0.0001
125
+ max_grad_norm: 1.0
126
+ fake_score:
127
+ lr: 1.0e-06
128
+ warmup_steps: 500
129
+ beta1: 0.9
130
+ beta2: 0.999
131
+ eps: 1.0e-08
132
+ weight_decay: 0.0
133
+ max_grad_norm: 1.0
134
+ aux_v:
135
+ lr: 1.0e-05
136
+ warmup_steps: 100
137
+ beta1: 0.9
138
+ beta2: 0.999
139
+ eps: 1.0e-08
140
+ weight_decay: 0.0
141
+ max_grad_norm: 1.0
142
+ train:
143
+ total_steps: 0
144
+ save_interval: 20
145
+ log_interval: 1
146
+ seed: 42
147
+ batch_size: 8
148
+ gradient_accumulation_steps: 1
149
+ autocast_dtype: bf16
150
+ resume_from: ''
151
+ use_ema: false
152
+ ema_decay: 0.9
153
+ ema_every: 1
154
+ distributed:
155
+ strategy: fsdp
156
+ fsdp_sharding: full_shard
157
+ ddp_find_unused_parameters: null
158
+ logging:
159
+ project: fewsteprl
160
+ run_name: aux_vel_mix
161
+ entity: ''
162
+ tags:
163
+ - sd35m
164
+ - aux-vel
165
+ - flow-matching
166
+ - distill
167
+ - '512'
168
+ enabled: true
169
+ eval:
170
+ enabled: true
171
+ eval_interval: 10
172
+ eval_prompt_source: dataset
173
+ eval_prompt_path: prompts.json
174
+ dataset: drawbench
175
+ dataset_path: dataset
176
+ reward_fn:
177
+ pickscore: 1.0
178
+ hpsv2: 1.0
179
+ clipscore: 1.0
180
+ aesthetic: 1.0
181
+ reward_dataset_map:
182
+ pickscore: drawbench
183
+ hpsv2: drawbench
184
+ clipscore: drawbench
185
+ aesthetic: drawbench
186
+ reward_ckpt_path: ./models/reward_ckpts
187
+ eval_num_steps: 32
188
+ eval_guidance_scale: 1.0
189
+ eval_batch_size: 16
190
+ num_media_images: 64
191
+ eval_seed: 12345
192
+ prompt_path: dataset/pickscore
193
+ output_dir: ./outputs/aux_vel_mix
aux_vel/ocr/aux_v_transformer/config.json ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_class_name": "SD3Transformer2DModel",
3
+ "_diffusers_version": "0.31.0",
4
+ "attention_head_dim": 64,
5
+ "caption_projection_dim": 1536,
6
+ "dual_attention_layers": [
7
+ 0,
8
+ 1,
9
+ 2,
10
+ 3,
11
+ 4,
12
+ 5,
13
+ 6,
14
+ 7,
15
+ 8,
16
+ 9,
17
+ 10,
18
+ 11,
19
+ 12
20
+ ],
21
+ "in_channels": 16,
22
+ "joint_attention_dim": 4096,
23
+ "num_attention_heads": 24,
24
+ "num_layers": 24,
25
+ "out_channels": 16,
26
+ "patch_size": 2,
27
+ "pooled_projection_dim": 2048,
28
+ "pos_embed_max_size": 384,
29
+ "qk_norm": "rms_norm",
30
+ "sample_size": 128
31
+ }
aux_vel/ocr/aux_v_transformer/diffusion_pytorch_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1daab3d85296af852db3fa336c24894dad5d8aea2488902e53ab2ca4ece60e4e
3
+ size 9878760864
aux_vel/ocr/config.yaml ADDED
@@ -0,0 +1,216 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 100
82
+ num_batches_per_epoch: 1
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 8
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0
97
+ clip_range: 0.001
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn: {}
102
+ reward_ckpt_path: ''
103
+ log_sample_images: 0
104
+ central_diff_epsilon: 5.0
105
+ cd_velocity_source: noise_minus_x0
106
+ share_cd_with_old: true
107
+ num_training_timesteps_per_sample: 4
108
+ rl_min_timestep: 0
109
+ rl_max_timestep: 1000
110
+ diffusion_ratio: 0.25
111
+ consistency_ratio: 0.25
112
+ use_ref_v_model: false
113
+ guidance_scale: 1.0
114
+ aux_v_enabled: true
115
+ max_aux_sample: 80
116
+ renoise_per_epoch: 10
117
+ aux_v_gradient_accum: 1
118
+ aux_v_batch_size: 8
119
+ rollout_type: sde
120
+ reuse_x_t: false
121
+ opsd_enabled: false
122
+ opsd_rho: 0.1
123
+ opsd_query_sigma: 0.278
124
+ opsd_n_ascent: 2
125
+ opsd_eta_tgt: 1.0
126
+ opsd_reward_kind: hpsv2
127
+ opsd_target_chunk: 6
128
+ opsd_eps_grad: 1.0e-12
129
+ opsd_eps_gamma: 1.0e-05
130
+ opsd_dual_neg: true
131
+ opsd_state_mode: forward
132
+ opsd_dir_mode: grad
133
+ opsd_sigma_mode: fixed
134
+ opsd_logit_mean: -2.053
135
+ opsd_logit_std: 0.6
136
+ renoise_distribution: uniform
137
+ renoise_logit_mean: -2.053
138
+ renoise_logit_std: 0.6
139
+ opsd_r_mode: zero
140
+ opsd_adv_gain: 1.0
141
+ solver:
142
+ generator:
143
+ lr: 3.0e-06
144
+ warmup_steps: 0
145
+ beta1: 0.9
146
+ beta2: 0.999
147
+ eps: 1.0e-08
148
+ weight_decay: 0.0001
149
+ max_grad_norm: 1.0
150
+ fake_score:
151
+ lr: 1.0e-06
152
+ warmup_steps: 500
153
+ beta1: 0.9
154
+ beta2: 0.999
155
+ eps: 1.0e-08
156
+ weight_decay: 0.0
157
+ max_grad_norm: 1.0
158
+ aux_v:
159
+ lr: 1.0e-05
160
+ warmup_steps: 100
161
+ beta1: 0.9
162
+ beta2: 0.999
163
+ eps: 1.0e-08
164
+ weight_decay: 0.0
165
+ max_grad_norm: 1.0
166
+ train:
167
+ total_steps: 0
168
+ save_interval: 20
169
+ log_interval: 1
170
+ seed: 42
171
+ batch_size: 8
172
+ gradient_accumulation_steps: 1
173
+ autocast_dtype: bf16
174
+ resume_from: ''
175
+ use_ema: false
176
+ ema_decay: 0.9
177
+ ema_every: 1
178
+ distributed:
179
+ strategy: fsdp
180
+ fsdp_sharding: full_shard
181
+ ddp_find_unused_parameters: null
182
+ logging:
183
+ project: fewsteprl
184
+ run_name: aux_vel_ocr
185
+ entity: ''
186
+ tags:
187
+ - sd35m
188
+ - aux-vel
189
+ - flow-matching
190
+ - distill
191
+ - '512'
192
+ - ocr
193
+ enabled: true
194
+ eval:
195
+ enabled: true
196
+ eval_interval: 10
197
+ eval_prompt_source: dataset
198
+ eval_prompt_path: prompts.json
199
+ dataset: ocr
200
+ dataset_path: dataset
201
+ reward_fn:
202
+ aesthetic: 1.0
203
+ clipscore: 1.0
204
+ ocr: 1.0
205
+ reward_dataset_map:
206
+ aesthetic: ocr
207
+ clipscore: ocr
208
+ ocr: ocr
209
+ reward_ckpt_path: ./models/reward_ckpts
210
+ eval_num_steps: 32
211
+ eval_guidance_scale: 1.0
212
+ eval_batch_size: 16
213
+ num_media_images: 64
214
+ eval_seed: 12345
215
+ prompt_path: dataset/ocr
216
+ output_dir: ./outputs/aux_vel_ocr
awm_mix/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:101e8de5d74ce4f5dde4ef14fa31155e20556461ea49746792942d6857b3d33d
3
+ size 43203307
awm_mix/config.yaml ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 401
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.002
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ clipscore: 1.0
103
+ hpsv2: 1.0
104
+ pickscore: 1.0
105
+ reward_ckpt_path: ./models/reward_ckpts
106
+ log_sample_images: 8
107
+ central_diff_epsilon: 5.0
108
+ cd_velocity_source: noise_minus_x0
109
+ share_cd_with_old: true
110
+ disable_cd: false
111
+ num_training_timesteps_per_sample: 4
112
+ rl_min_timestep: 0
113
+ rl_max_timestep: 1000
114
+ diffusion_ratio: 0.5
115
+ consistency_ratio: 0.25
116
+ use_ref_v_model: false
117
+ guidance_scale: 1.0
118
+ aux_v_enabled: false
119
+ max_aux_sample: 0
120
+ renoise_per_epoch: 10
121
+ aux_v_gradient_accum: 1
122
+ aux_v_batch_size: 0
123
+ rollout_type: ode
124
+ reuse_x_t: false
125
+ opsd_enabled: false
126
+ opsd_rho: 0.1
127
+ opsd_query_sigma: 0.278
128
+ opsd_n_ascent: 2
129
+ opsd_eta_tgt: 1.0
130
+ opsd_reward_kind: hpsv2
131
+ opsd_target_chunk: 6
132
+ opsd_eps_grad: 1.0e-12
133
+ opsd_eps_gamma: 1.0e-05
134
+ opsd_dual_neg: true
135
+ opsd_state_mode: forward
136
+ opsd_dir_mode: grad
137
+ opsd_sigma_mode: fixed
138
+ opsd_logit_mean: -2.053
139
+ opsd_logit_std: 0.6
140
+ renoise_distribution: uniform
141
+ renoise_logit_mean: -2.053
142
+ renoise_logit_std: 0.6
143
+ opsd_r_mode: zero
144
+ opsd_adv_gain: 1.0
145
+ refl_use_advantage: false
146
+ solver:
147
+ generator:
148
+ lr: 3.0e-06
149
+ warmup_steps: 0
150
+ beta1: 0.9
151
+ beta2: 0.999
152
+ eps: 1.0e-08
153
+ weight_decay: 0.0001
154
+ max_grad_norm: 1.0
155
+ fake_score:
156
+ lr: 1.0e-06
157
+ warmup_steps: 500
158
+ beta1: 0.9
159
+ beta2: 0.999
160
+ eps: 1.0e-08
161
+ weight_decay: 0.0
162
+ max_grad_norm: 1.0
163
+ aux_v:
164
+ lr: 1.0e-05
165
+ warmup_steps: 0
166
+ beta1: 0.9
167
+ beta2: 0.999
168
+ eps: 1.0e-08
169
+ weight_decay: 0.0
170
+ max_grad_norm: 1.0
171
+ train:
172
+ total_steps: 0
173
+ save_interval: 100
174
+ log_interval: 1
175
+ seed: 42
176
+ batch_size: 9
177
+ gradient_accumulation_steps: 1
178
+ autocast_dtype: bf16
179
+ resume_from: ''
180
+ use_ema: true
181
+ ema_decay: 0.9
182
+ ema_every: 1
183
+ distributed:
184
+ strategy: fsdp
185
+ fsdp_sharding: full_shard
186
+ ddp_find_unused_parameters: null
187
+ logging:
188
+ project: fewsteprl
189
+ run_name: awm_mix
190
+ entity: ''
191
+ tags:
192
+ - no-s-term
193
+ - sd35m
194
+ - fewstep-rl
195
+ - kl-u
196
+ - mixed
197
+ - awm
198
+ - klw1e-4
199
+ - '512'
200
+ enabled: true
201
+ eval:
202
+ enabled: true
203
+ eval_interval: 50
204
+ eval_prompt_source: dataset
205
+ eval_prompt_path: prompts.json
206
+ dataset: drawbench
207
+ dataset_path: dataset
208
+ reward_fn:
209
+ aesthetic: 1.0
210
+ clipscore: 1.0
211
+ hpsv2: 1.0
212
+ ocr: 1.0
213
+ pickscore: 1.0
214
+ reward_dataset_map:
215
+ aesthetic: drawbench
216
+ clipscore: drawbench
217
+ hpsv2: drawbench
218
+ ocr: ocr
219
+ pickscore: drawbench
220
+ reward_ckpt_path: ./models/reward_ckpts
221
+ eval_num_steps: 4
222
+ eval_guidance_scale: 1.0
223
+ eval_batch_size: 16
224
+ num_media_images: 64
225
+ eval_seed: 12345
226
+ prompt_path: dataset/pickscore
227
+ output_dir: ./outputs/awm_mix
dmdawm_mix/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:308bf0e588143cad86900008d16ae21e7a64b63b53390e7fec0fcb674e48f841
3
+ size 43203307
dmdawm_mix/config.yaml ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ./checkpoints/aux_vel/mix
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 400
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.002
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ clipscore: 1.0
103
+ hpsv2: 1.0
104
+ pickscore: 1.0
105
+ reward_ckpt_path: ./models/reward_ckpts
106
+ log_sample_images: 8
107
+ central_diff_epsilon: 5.0
108
+ cd_velocity_source: noise_minus_x0
109
+ share_cd_with_old: true
110
+ disable_cd: false
111
+ num_training_timesteps_per_sample: 4
112
+ rl_min_timestep: 0
113
+ rl_max_timestep: 1000
114
+ diffusion_ratio: 0.5
115
+ consistency_ratio: 0.25
116
+ use_ref_v_model: false
117
+ guidance_scale: 1.0
118
+ aux_v_enabled: true
119
+ max_aux_sample: 40
120
+ renoise_per_epoch: 5
121
+ aux_v_gradient_accum: 1
122
+ aux_v_batch_size: 8
123
+ rollout_type: ode
124
+ reuse_x_t: false
125
+ opsd_enabled: false
126
+ opsd_rho: 0.1
127
+ opsd_query_sigma: 0.278
128
+ opsd_n_ascent: 2
129
+ opsd_eta_tgt: 1.0
130
+ opsd_reward_kind: hpsv2
131
+ opsd_target_chunk: 6
132
+ opsd_eps_grad: 1.0e-12
133
+ opsd_eps_gamma: 1.0e-05
134
+ opsd_dual_neg: true
135
+ opsd_state_mode: forward
136
+ opsd_dir_mode: grad
137
+ opsd_sigma_mode: fixed
138
+ opsd_logit_mean: -2.053
139
+ opsd_logit_std: 0.6
140
+ renoise_distribution: uniform
141
+ renoise_logit_mean: -2.053
142
+ renoise_logit_std: 0.6
143
+ opsd_r_mode: zero
144
+ opsd_adv_gain: 1.0
145
+ refl_use_advantage: false
146
+ solver:
147
+ generator:
148
+ lr: 3.0e-06
149
+ warmup_steps: 0
150
+ beta1: 0.9
151
+ beta2: 0.999
152
+ eps: 1.0e-08
153
+ weight_decay: 0.0001
154
+ max_grad_norm: 1.0
155
+ fake_score:
156
+ lr: 1.0e-06
157
+ warmup_steps: 500
158
+ beta1: 0.9
159
+ beta2: 0.999
160
+ eps: 1.0e-08
161
+ weight_decay: 0.0
162
+ max_grad_norm: 1.0
163
+ aux_v:
164
+ lr: 1.0e-05
165
+ warmup_steps: 100
166
+ beta1: 0.9
167
+ beta2: 0.999
168
+ eps: 1.0e-08
169
+ weight_decay: 0.0
170
+ max_grad_norm: 1.0
171
+ train:
172
+ total_steps: 0
173
+ save_interval: 100
174
+ log_interval: 1
175
+ seed: 42
176
+ batch_size: 9
177
+ gradient_accumulation_steps: 1
178
+ autocast_dtype: bf16
179
+ resume_from: ''
180
+ use_ema: true
181
+ ema_decay: 0.9
182
+ ema_every: 1
183
+ distributed:
184
+ strategy: fsdp
185
+ fsdp_sharding: full_shard
186
+ ddp_find_unused_parameters: null
187
+ logging:
188
+ project: fewsteprl
189
+ run_name: dmdawm_mix
190
+ entity: ''
191
+ tags:
192
+ - '512'
193
+ - anchor-u-old
194
+ - aux-rwd-center
195
+ - dmdawm
196
+ - fewstep-rl
197
+ - kl-u
198
+ - klw1e-4
199
+ - mixed
200
+ - nocd
201
+ - sd35m
202
+ enabled: true
203
+ eval:
204
+ enabled: true
205
+ eval_interval: 50
206
+ eval_prompt_source: dataset
207
+ eval_prompt_path: prompts.json
208
+ dataset: drawbench
209
+ dataset_path: dataset
210
+ reward_fn:
211
+ aesthetic: 1.0
212
+ clipscore: 1.0
213
+ hpsv2: 1.0
214
+ ocr: 1.0
215
+ pickscore: 1.0
216
+ reward_dataset_map:
217
+ aesthetic: drawbench
218
+ clipscore: drawbench
219
+ hpsv2: drawbench
220
+ ocr: ocr
221
+ pickscore: drawbench
222
+ reward_ckpt_path: ./models/reward_ckpts
223
+ eval_num_steps: 4
224
+ eval_guidance_scale: 1.0
225
+ eval_batch_size: 16
226
+ num_media_images: 64
227
+ eval_seed: 12345
228
+ prompt_path: dataset/pickscore
229
+ output_dir: ./outputs/dmdawm_mix
dmdawm_ocr/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19bd328be85c84389c7170aa076f31a3390fa70204d4df3f757b42c65181dd6b
3
+ size 43203307
dmdawm_ocr/config.yaml ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ./checkpoints/aux_vel/ocr
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 10000
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.01
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ ocr: 1.0
103
+ reward_ckpt_path: ./models/reward_ckpts
104
+ log_sample_images: 8
105
+ central_diff_epsilon: 5.0
106
+ cd_velocity_source: noise_minus_x0
107
+ share_cd_with_old: true
108
+ disable_cd: false
109
+ num_training_timesteps_per_sample: 4
110
+ rl_min_timestep: 0
111
+ rl_max_timestep: 1000
112
+ diffusion_ratio: 0.5
113
+ consistency_ratio: 0.25
114
+ use_ref_v_model: false
115
+ guidance_scale: 1.0
116
+ aux_v_enabled: true
117
+ max_aux_sample: 40
118
+ renoise_per_epoch: 5
119
+ aux_v_gradient_accum: 1
120
+ aux_v_batch_size: 8
121
+ rollout_type: ode
122
+ reuse_x_t: false
123
+ opsd_enabled: false
124
+ opsd_rho: 0.1
125
+ opsd_query_sigma: 0.278
126
+ opsd_n_ascent: 2
127
+ opsd_eta_tgt: 1.0
128
+ opsd_reward_kind: hpsv2
129
+ opsd_target_chunk: 6
130
+ opsd_eps_grad: 1.0e-12
131
+ opsd_eps_gamma: 1.0e-05
132
+ opsd_dual_neg: true
133
+ opsd_state_mode: forward
134
+ opsd_dir_mode: grad
135
+ opsd_sigma_mode: fixed
136
+ opsd_logit_mean: -2.053
137
+ opsd_logit_std: 0.6
138
+ renoise_distribution: uniform
139
+ renoise_logit_mean: -2.053
140
+ renoise_logit_std: 0.6
141
+ opsd_r_mode: zero
142
+ opsd_adv_gain: 1.0
143
+ refl_use_advantage: false
144
+ solver:
145
+ generator:
146
+ lr: 3.0e-06
147
+ warmup_steps: 0
148
+ beta1: 0.9
149
+ beta2: 0.999
150
+ eps: 1.0e-08
151
+ weight_decay: 0.0001
152
+ max_grad_norm: 1.0
153
+ fake_score:
154
+ lr: 1.0e-06
155
+ warmup_steps: 500
156
+ beta1: 0.9
157
+ beta2: 0.999
158
+ eps: 1.0e-08
159
+ weight_decay: 0.0
160
+ max_grad_norm: 1.0
161
+ aux_v:
162
+ lr: 1.0e-05
163
+ warmup_steps: 100
164
+ beta1: 0.9
165
+ beta2: 0.999
166
+ eps: 1.0e-08
167
+ weight_decay: 0.0
168
+ max_grad_norm: 1.0
169
+ train:
170
+ total_steps: 0
171
+ save_interval: 100
172
+ log_interval: 1
173
+ seed: 42
174
+ batch_size: 9
175
+ gradient_accumulation_steps: 1
176
+ autocast_dtype: bf16
177
+ resume_from: ''
178
+ use_ema: true
179
+ ema_decay: 0.9
180
+ ema_every: 1
181
+ distributed:
182
+ strategy: fsdp
183
+ fsdp_sharding: full_shard
184
+ ddp_find_unused_parameters: null
185
+ logging:
186
+ project: fewsteprl
187
+ run_name: dmdawm_ocr
188
+ entity: ''
189
+ tags:
190
+ - sd35m
191
+ - fewstep-rl
192
+ - kl-u
193
+ - ocr
194
+ - dmdawm
195
+ - nocd
196
+ - anchor-u-old
197
+ - aux-rwd-center
198
+ - klw1e-4
199
+ - '512'
200
+ enabled: true
201
+ eval:
202
+ enabled: true
203
+ eval_interval: 50
204
+ eval_prompt_source: dataset
205
+ eval_prompt_path: prompts.json
206
+ dataset: ocr
207
+ dataset_path: dataset
208
+ reward_fn:
209
+ aesthetic: 1.0
210
+ clipscore: 1.0
211
+ ocr: 1.0
212
+ reward_dataset_map:
213
+ aesthetic: ocr
214
+ clipscore: ocr
215
+ ocr: ocr
216
+ reward_ckpt_path: ./models/reward_ckpts
217
+ eval_num_steps: 4
218
+ eval_guidance_scale: 1.0
219
+ eval_batch_size: 16
220
+ num_media_images: 64
221
+ eval_seed: 12345
222
+ prompt_path: dataset/ocr
223
+ output_dir: ./outputs/dmdawm_ocr
dmdnft_mix/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:eaf13570b1ddb93b107a1939334fbe7bac4d1752a89e8ecdde0b01c282f8dab5
3
+ size 43203307
dmdnft_mix/config.yaml ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ./checkpoints/aux_vel/mix
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 10000
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ decay_type: 1
98
+ rl_decay_interval: 1
99
+ reward_fn:
100
+ clipscore: 1.0
101
+ hpsv2: 1.0
102
+ pickscore: 1.0
103
+ reward_ckpt_path: ./models/reward_ckpts
104
+ log_sample_images: 8
105
+ central_diff_epsilon: 5.0
106
+ cd_velocity_source: noise_minus_x0
107
+ share_cd_with_old: true
108
+ num_training_timesteps_per_sample: 4
109
+ rl_min_timestep: 0
110
+ rl_max_timestep: 1000
111
+ diffusion_ratio: 0.5
112
+ consistency_ratio: 0.25
113
+ use_ref_v_model: false
114
+ guidance_scale: 1.0
115
+ aux_v_enabled: true
116
+ max_aux_sample: 80
117
+ renoise_per_epoch: 10
118
+ aux_v_gradient_accum: 1
119
+ aux_v_batch_size: 8
120
+ solver:
121
+ generator:
122
+ lr: 3.0e-06
123
+ warmup_steps: 0
124
+ beta1: 0.9
125
+ beta2: 0.999
126
+ eps: 1.0e-08
127
+ weight_decay: 0.0001
128
+ max_grad_norm: 1.0
129
+ fake_score:
130
+ lr: 1.0e-06
131
+ warmup_steps: 500
132
+ beta1: 0.9
133
+ beta2: 0.999
134
+ eps: 1.0e-08
135
+ weight_decay: 0.0
136
+ max_grad_norm: 1.0
137
+ aux_v:
138
+ lr: 1.0e-05
139
+ warmup_steps: 100
140
+ beta1: 0.9
141
+ beta2: 0.999
142
+ eps: 1.0e-08
143
+ weight_decay: 0.0
144
+ max_grad_norm: 1.0
145
+ train:
146
+ total_steps: 0
147
+ save_interval: 100
148
+ log_interval: 1
149
+ seed: 42
150
+ batch_size: 9
151
+ gradient_accumulation_steps: 1
152
+ autocast_dtype: bf16
153
+ resume_from: ''
154
+ use_ema: true
155
+ ema_decay: 0.9
156
+ ema_every: 1
157
+ distributed:
158
+ strategy: fsdp
159
+ fsdp_sharding: full_shard
160
+ ddp_find_unused_parameters: null
161
+ logging:
162
+ project: fewsteprl
163
+ run_name: dmdnft_mix
164
+ entity: ''
165
+ tags:
166
+ - sd35m
167
+ - dmdnft
168
+ - variant-c
169
+ - aux-v
170
+ - kl-u
171
+ - mixed
172
+ - klw1e-4
173
+ - align-old-stage3
174
+ - '512'
175
+ enabled: true
176
+ eval:
177
+ enabled: true
178
+ eval_interval: 50
179
+ eval_prompt_source: dataset
180
+ eval_prompt_path: prompts.json
181
+ dataset: drawbench
182
+ dataset_path: dataset
183
+ reward_fn:
184
+ aesthetic: 1.0
185
+ clipscore: 1.0
186
+ hpsv2: 1.0
187
+ ocr: 1.0
188
+ pickscore: 1.0
189
+ reward_dataset_map:
190
+ aesthetic: drawbench
191
+ clipscore: drawbench
192
+ hpsv2: drawbench
193
+ ocr: ocr
194
+ pickscore: drawbench
195
+ reward_ckpt_path: ./models/reward_ckpts
196
+ eval_num_steps: 4
197
+ eval_guidance_scale: 1.0
198
+ eval_batch_size: 16
199
+ num_media_images: 64
200
+ eval_seed: 12345
201
+ prompt_path: dataset/pickscore
202
+ output_dir: ./outputs/dmdnft_mix
dmdnft_ocr/checkpoint-200/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:34f69486ef7294ddb74ac68999bf68b3b6e16fc4e500e9e2eb4168c59772a876
3
+ size 43203307
dmdnft_ocr/config.yaml ADDED
@@ -0,0 +1,196 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ./checkpoints/aux_vel/mix
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 10000
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ decay_type: 1
98
+ rl_decay_interval: 1
99
+ reward_fn:
100
+ ocr: 1.0
101
+ reward_ckpt_path: ./models/reward_ckpts
102
+ log_sample_images: 8
103
+ central_diff_epsilon: 5.0
104
+ cd_velocity_source: noise_minus_x0
105
+ share_cd_with_old: true
106
+ num_training_timesteps_per_sample: 4
107
+ rl_min_timestep: 0
108
+ rl_max_timestep: 1000
109
+ diffusion_ratio: 0.5
110
+ consistency_ratio: 0.25
111
+ use_ref_v_model: false
112
+ guidance_scale: 1.0
113
+ aux_v_enabled: true
114
+ max_aux_sample: 80
115
+ renoise_per_epoch: 10
116
+ aux_v_gradient_accum: 1
117
+ aux_v_batch_size: 8
118
+ solver:
119
+ generator:
120
+ lr: 3.0e-06
121
+ warmup_steps: 0
122
+ beta1: 0.9
123
+ beta2: 0.999
124
+ eps: 1.0e-08
125
+ weight_decay: 0.0001
126
+ max_grad_norm: 1.0
127
+ fake_score:
128
+ lr: 1.0e-06
129
+ warmup_steps: 500
130
+ beta1: 0.9
131
+ beta2: 0.999
132
+ eps: 1.0e-08
133
+ weight_decay: 0.0
134
+ max_grad_norm: 1.0
135
+ aux_v:
136
+ lr: 1.0e-05
137
+ warmup_steps: 100
138
+ beta1: 0.9
139
+ beta2: 0.999
140
+ eps: 1.0e-08
141
+ weight_decay: 0.0
142
+ max_grad_norm: 1.0
143
+ train:
144
+ total_steps: 0
145
+ save_interval: 100
146
+ log_interval: 1
147
+ seed: 42
148
+ batch_size: 9
149
+ gradient_accumulation_steps: 1
150
+ autocast_dtype: bf16
151
+ resume_from: ''
152
+ use_ema: true
153
+ ema_decay: 0.9
154
+ ema_every: 1
155
+ distributed:
156
+ strategy: fsdp
157
+ fsdp_sharding: full_shard
158
+ ddp_find_unused_parameters: null
159
+ logging:
160
+ project: fewsteprl
161
+ run_name: dmdnft_ocr
162
+ entity: ''
163
+ tags:
164
+ - sd35m
165
+ - dmdnft
166
+ - variant-c
167
+ - aux-v
168
+ - kl-u
169
+ - ocr
170
+ - klw1e-4
171
+ - align-old-stage3
172
+ - '512'
173
+ enabled: true
174
+ eval:
175
+ enabled: true
176
+ eval_interval: 50
177
+ eval_prompt_source: dataset
178
+ eval_prompt_path: prompts.json
179
+ dataset: ocr
180
+ dataset_path: dataset
181
+ reward_fn:
182
+ aesthetic: 1.0
183
+ clipscore: 1.0
184
+ ocr: 1.0
185
+ reward_dataset_map:
186
+ aesthetic: ocr
187
+ clipscore: ocr
188
+ ocr: ocr
189
+ reward_ckpt_path: ./models/reward_ckpts
190
+ eval_num_steps: 4
191
+ eval_guidance_scale: 1.0
192
+ eval_batch_size: 16
193
+ num_media_images: 64
194
+ eval_seed: 12345
195
+ prompt_path: dataset/ocr
196
+ output_dir: ./outputs/dmdnft_ocr
fewstepgrpo_mix/checkpoint-300/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d80228738e50513c307e1ebfeef05eb5a36dfd20656338447df6ecf20171b5fe
3
+ size 43203307
fewstepgrpo_mix/config.yaml ADDED
@@ -0,0 +1,206 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 400
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.003
98
+ grpo_noise_level: 0.25
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ clipscore: 1.0
103
+ hpsv2: 1.0
104
+ pickscore: 1.0
105
+ reward_ckpt_path: ./models/reward_ckpts
106
+ log_sample_images: 8
107
+ central_diff_epsilon: 5.0
108
+ cd_velocity_source: noise_minus_x0
109
+ share_cd_with_old: true
110
+ num_training_timesteps_per_sample: 4
111
+ rl_min_timestep: 0
112
+ rl_max_timestep: 1000
113
+ diffusion_ratio: 0.5
114
+ consistency_ratio: 0.25
115
+ use_ref_v_model: false
116
+ guidance_scale: 1.0
117
+ aux_v_enabled: false
118
+ max_aux_sample: 0
119
+ renoise_per_epoch: 10
120
+ aux_v_gradient_accum: 1
121
+ aux_v_batch_size: 0
122
+ rollout_type: sde
123
+ reuse_x_t: true
124
+ solver:
125
+ generator:
126
+ lr: 3.0e-06
127
+ warmup_steps: 0
128
+ beta1: 0.9
129
+ beta2: 0.999
130
+ eps: 1.0e-08
131
+ weight_decay: 0.0001
132
+ max_grad_norm: 1.0
133
+ fake_score:
134
+ lr: 1.0e-06
135
+ warmup_steps: 500
136
+ beta1: 0.9
137
+ beta2: 0.999
138
+ eps: 1.0e-08
139
+ weight_decay: 0.0
140
+ max_grad_norm: 1.0
141
+ aux_v:
142
+ lr: 1.0e-05
143
+ warmup_steps: 0
144
+ beta1: 0.9
145
+ beta2: 0.999
146
+ eps: 1.0e-08
147
+ weight_decay: 0.0
148
+ max_grad_norm: 1.0
149
+ train:
150
+ total_steps: 0
151
+ save_interval: 100
152
+ log_interval: 1
153
+ seed: 42
154
+ batch_size: 9
155
+ gradient_accumulation_steps: 1
156
+ autocast_dtype: bf16
157
+ resume_from: ''
158
+ use_ema: true
159
+ ema_decay: 0.9
160
+ ema_every: 1
161
+ distributed:
162
+ strategy: fsdp
163
+ fsdp_sharding: full_shard
164
+ ddp_find_unused_parameters: null
165
+ logging:
166
+ project: fewsteprl
167
+ run_name: fewstepgrpo_mix
168
+ entity: ''
169
+ tags:
170
+ - sd35m
171
+ - fewstep-rl
172
+ - kl-u
173
+ - mixed
174
+ - grpo
175
+ - vform
176
+ - eta025_clip3e3
177
+ - klw1e-4
178
+ - '512'
179
+ enabled: true
180
+ eval:
181
+ enabled: true
182
+ eval_interval: 50
183
+ eval_prompt_source: dataset
184
+ eval_prompt_path: prompts.json
185
+ dataset: drawbench
186
+ dataset_path: dataset
187
+ reward_fn:
188
+ aesthetic: 1.0
189
+ clipscore: 1.0
190
+ hpsv2: 1.0
191
+ ocr: 1.0
192
+ pickscore: 1.0
193
+ reward_dataset_map:
194
+ aesthetic: drawbench
195
+ clipscore: drawbench
196
+ hpsv2: drawbench
197
+ ocr: ocr
198
+ pickscore: drawbench
199
+ reward_ckpt_path: ./models/reward_ckpts
200
+ eval_num_steps: 4
201
+ eval_guidance_scale: 1.0
202
+ eval_batch_size: 16
203
+ num_media_images: 64
204
+ eval_seed: 12345
205
+ prompt_path: dataset/pickscore
206
+ output_dir: ./outputs/fewstepgrpo_mix
fewstepgrpo_ocr/checkpoint-200/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:55221ce0596ef9447b89e8234404f8b13cd32c7eb815b31f5c21f5c665abfb44
3
+ size 43203307
fewstepgrpo_ocr/config.yaml ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 400
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.004
98
+ grpo_noise_level: 0.25
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ ocr: 1.0
103
+ reward_ckpt_path: ./models/reward_ckpts
104
+ log_sample_images: 8
105
+ central_diff_epsilon: 5.0
106
+ cd_velocity_source: noise_minus_x0
107
+ share_cd_with_old: true
108
+ num_training_timesteps_per_sample: 4
109
+ rl_min_timestep: 0
110
+ rl_max_timestep: 1000
111
+ diffusion_ratio: 0.5
112
+ consistency_ratio: 0.25
113
+ use_ref_v_model: false
114
+ guidance_scale: 1.0
115
+ aux_v_enabled: false
116
+ max_aux_sample: 0
117
+ renoise_per_epoch: 10
118
+ aux_v_gradient_accum: 1
119
+ aux_v_batch_size: 0
120
+ rollout_type: sde
121
+ reuse_x_t: true
122
+ solver:
123
+ generator:
124
+ lr: 3.0e-06
125
+ warmup_steps: 0
126
+ beta1: 0.9
127
+ beta2: 0.999
128
+ eps: 1.0e-08
129
+ weight_decay: 0.0001
130
+ max_grad_norm: 1.0
131
+ fake_score:
132
+ lr: 1.0e-06
133
+ warmup_steps: 500
134
+ beta1: 0.9
135
+ beta2: 0.999
136
+ eps: 1.0e-08
137
+ weight_decay: 0.0
138
+ max_grad_norm: 1.0
139
+ aux_v:
140
+ lr: 1.0e-05
141
+ warmup_steps: 0
142
+ beta1: 0.9
143
+ beta2: 0.999
144
+ eps: 1.0e-08
145
+ weight_decay: 0.0
146
+ max_grad_norm: 1.0
147
+ train:
148
+ total_steps: 0
149
+ save_interval: 100
150
+ log_interval: 1
151
+ seed: 42
152
+ batch_size: 9
153
+ gradient_accumulation_steps: 1
154
+ autocast_dtype: bf16
155
+ resume_from: ''
156
+ use_ema: true
157
+ ema_decay: 0.9
158
+ ema_every: 1
159
+ distributed:
160
+ strategy: fsdp
161
+ fsdp_sharding: full_shard
162
+ ddp_find_unused_parameters: null
163
+ logging:
164
+ project: fewsteprl
165
+ run_name: fewstepgrpo_ocr
166
+ entity: ''
167
+ tags:
168
+ - sd35m
169
+ - fewstep-rl
170
+ - kl-u
171
+ - ocr
172
+ - grpo
173
+ - vform
174
+ - eta025_clip004
175
+ - klw1e-4
176
+ - '512'
177
+ enabled: true
178
+ eval:
179
+ enabled: true
180
+ eval_interval: 50
181
+ eval_prompt_source: dataset
182
+ eval_prompt_path: prompts.json
183
+ dataset: ocr
184
+ dataset_path: dataset
185
+ reward_fn:
186
+ aesthetic: 1.0
187
+ clipscore: 1.0
188
+ ocr: 1.0
189
+ reward_dataset_map:
190
+ aesthetic: ocr
191
+ clipscore: ocr
192
+ ocr: ocr
193
+ reward_ckpt_path: ./models/reward_ckpts
194
+ eval_num_steps: 4
195
+ eval_guidance_scale: 1.0
196
+ eval_batch_size: 16
197
+ num_media_images: 64
198
+ eval_seed: 12345
199
+ prompt_path: dataset/ocr
200
+ output_dir: ./outputs/fewstepgrpo_ocr
flowgrpo_mix/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3b9745b3d478b25280bc4d5fb3056900b8d0a17a71b5d6a5d68e8704b3c2f155
3
+ size 43203307
flowgrpo_mix/config.yaml ADDED
@@ -0,0 +1,229 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 401
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.003
98
+ grpo_noise_level: 0.25
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ clipscore: 1.0
103
+ hpsv2: 1.0
104
+ pickscore: 1.0
105
+ reward_ckpt_path: ./models/reward_ckpts
106
+ log_sample_images: 8
107
+ central_diff_epsilon: 5.0
108
+ cd_velocity_source: noise_minus_x0
109
+ share_cd_with_old: true
110
+ disable_cd: false
111
+ num_training_timesteps_per_sample: 4
112
+ rl_min_timestep: 0
113
+ rl_max_timestep: 1000
114
+ diffusion_ratio: 0.5
115
+ consistency_ratio: 0.25
116
+ use_ref_v_model: false
117
+ guidance_scale: 1.0
118
+ aux_v_enabled: false
119
+ max_aux_sample: 0
120
+ renoise_per_epoch: 10
121
+ aux_v_gradient_accum: 1
122
+ aux_v_batch_size: 0
123
+ rollout_type: sde
124
+ reuse_x_t: true
125
+ opsd_enabled: false
126
+ opsd_rho: 0.1
127
+ opsd_query_sigma: 0.278
128
+ opsd_n_ascent: 2
129
+ opsd_eta_tgt: 1.0
130
+ opsd_reward_kind: hpsv2
131
+ opsd_target_chunk: 6
132
+ opsd_eps_grad: 1.0e-12
133
+ opsd_eps_gamma: 1.0e-05
134
+ opsd_dual_neg: true
135
+ opsd_state_mode: forward
136
+ opsd_dir_mode: grad
137
+ opsd_sigma_mode: fixed
138
+ opsd_logit_mean: -2.053
139
+ opsd_logit_std: 0.6
140
+ renoise_distribution: uniform
141
+ renoise_logit_mean: -2.053
142
+ renoise_logit_std: 0.6
143
+ opsd_r_mode: zero
144
+ opsd_adv_gain: 1.0
145
+ refl_use_advantage: false
146
+ solver:
147
+ generator:
148
+ lr: 3.0e-06
149
+ warmup_steps: 0
150
+ beta1: 0.9
151
+ beta2: 0.999
152
+ eps: 1.0e-08
153
+ weight_decay: 0.0001
154
+ max_grad_norm: 1.0
155
+ fake_score:
156
+ lr: 1.0e-06
157
+ warmup_steps: 500
158
+ beta1: 0.9
159
+ beta2: 0.999
160
+ eps: 1.0e-08
161
+ weight_decay: 0.0
162
+ max_grad_norm: 1.0
163
+ aux_v:
164
+ lr: 1.0e-05
165
+ warmup_steps: 0
166
+ beta1: 0.9
167
+ beta2: 0.999
168
+ eps: 1.0e-08
169
+ weight_decay: 0.0
170
+ max_grad_norm: 1.0
171
+ train:
172
+ total_steps: 0
173
+ save_interval: 100
174
+ log_interval: 1
175
+ seed: 42
176
+ batch_size: 9
177
+ gradient_accumulation_steps: 1
178
+ autocast_dtype: bf16
179
+ resume_from: ''
180
+ use_ema: true
181
+ ema_decay: 0.9
182
+ ema_every: 1
183
+ distributed:
184
+ strategy: fsdp
185
+ fsdp_sharding: full_shard
186
+ ddp_find_unused_parameters: null
187
+ logging:
188
+ project: fewsteprl
189
+ run_name: flowgrpo_mix
190
+ entity: ''
191
+ tags:
192
+ - no-s-term
193
+ - sd35m
194
+ - fewstep-rl
195
+ - kl-u
196
+ - mixed
197
+ - grpo
198
+ - vform
199
+ - eta025_clip3e3
200
+ - klw1e-4
201
+ - '512'
202
+ enabled: true
203
+ eval:
204
+ enabled: true
205
+ eval_interval: 50
206
+ eval_prompt_source: dataset
207
+ eval_prompt_path: prompts.json
208
+ dataset: drawbench
209
+ dataset_path: dataset
210
+ reward_fn:
211
+ aesthetic: 1.0
212
+ clipscore: 1.0
213
+ hpsv2: 1.0
214
+ ocr: 1.0
215
+ pickscore: 1.0
216
+ reward_dataset_map:
217
+ aesthetic: drawbench
218
+ clipscore: drawbench
219
+ hpsv2: drawbench
220
+ ocr: ocr
221
+ pickscore: drawbench
222
+ reward_ckpt_path: ./models/reward_ckpts
223
+ eval_num_steps: 4
224
+ eval_guidance_scale: 1.0
225
+ eval_batch_size: 16
226
+ num_media_images: 64
227
+ eval_seed: 12345
228
+ prompt_path: dataset/pickscore
229
+ output_dir: ./outputs/flowgrpo_mix
meanflowawm_mix/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e1dfe852ff3e154de07facac5f64ec2a671311df2c032e3949745f641da5e6e2
3
+ size 43203307
meanflowawm_mix/config.yaml ADDED
@@ -0,0 +1,218 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 400
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.002
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ clipscore: 1.0
103
+ hpsv2: 1.0
104
+ pickscore: 1.0
105
+ reward_ckpt_path: ./models/reward_ckpts
106
+ log_sample_images: 8
107
+ central_diff_epsilon: 5.0
108
+ cd_velocity_source: noise_minus_x0
109
+ share_cd_with_old: true
110
+ num_training_timesteps_per_sample: 4
111
+ rl_min_timestep: 0
112
+ rl_max_timestep: 1000
113
+ diffusion_ratio: 0.5
114
+ consistency_ratio: 0.25
115
+ use_ref_v_model: false
116
+ guidance_scale: 1.0
117
+ aux_v_enabled: false
118
+ max_aux_sample: 0
119
+ renoise_per_epoch: 10
120
+ aux_v_gradient_accum: 1
121
+ aux_v_batch_size: 0
122
+ rollout_type: ode
123
+ reuse_x_t: false
124
+ opsd_enabled: false
125
+ opsd_rho: 0.1
126
+ opsd_query_sigma: 0.278
127
+ opsd_n_ascent: 2
128
+ opsd_eta_tgt: 1.0
129
+ opsd_reward_kind: hpsv2
130
+ opsd_target_chunk: 6
131
+ opsd_eps_grad: 1.0e-12
132
+ opsd_eps_gamma: 1.0e-05
133
+ opsd_dual_neg: true
134
+ opsd_state_mode: forward
135
+ opsd_dir_mode: grad
136
+ opsd_sigma_mode: fixed
137
+ opsd_r_mode: zero
138
+ solver:
139
+ generator:
140
+ lr: 3.0e-06
141
+ warmup_steps: 0
142
+ beta1: 0.9
143
+ beta2: 0.999
144
+ eps: 1.0e-08
145
+ weight_decay: 0.0001
146
+ max_grad_norm: 1.0
147
+ fake_score:
148
+ lr: 1.0e-06
149
+ warmup_steps: 500
150
+ beta1: 0.9
151
+ beta2: 0.999
152
+ eps: 1.0e-08
153
+ weight_decay: 0.0
154
+ max_grad_norm: 1.0
155
+ aux_v:
156
+ lr: 1.0e-05
157
+ warmup_steps: 0
158
+ beta1: 0.9
159
+ beta2: 0.999
160
+ eps: 1.0e-08
161
+ weight_decay: 0.0
162
+ max_grad_norm: 1.0
163
+ train:
164
+ total_steps: 0
165
+ save_interval: 100
166
+ log_interval: 1
167
+ seed: 42
168
+ batch_size: 9
169
+ gradient_accumulation_steps: 1
170
+ autocast_dtype: bf16
171
+ resume_from: ''
172
+ use_ema: true
173
+ ema_decay: 0.9
174
+ ema_every: 1
175
+ distributed:
176
+ strategy: fsdp
177
+ fsdp_sharding: full_shard
178
+ ddp_find_unused_parameters: null
179
+ logging:
180
+ project: fewsteprl
181
+ run_name: meanflowawm_mix
182
+ entity: ''
183
+ tags:
184
+ - sd35m
185
+ - fewstep-rl
186
+ - kl-u
187
+ - mixed
188
+ - awm
189
+ - klw1e-4
190
+ - '512'
191
+ enabled: true
192
+ eval:
193
+ enabled: true
194
+ eval_interval: 50
195
+ eval_prompt_source: dataset
196
+ eval_prompt_path: prompts.json
197
+ dataset: drawbench
198
+ dataset_path: dataset
199
+ reward_fn:
200
+ aesthetic: 1.0
201
+ clipscore: 1.0
202
+ hpsv2: 1.0
203
+ ocr: 1.0
204
+ pickscore: 1.0
205
+ reward_dataset_map:
206
+ aesthetic: drawbench
207
+ clipscore: drawbench
208
+ hpsv2: drawbench
209
+ ocr: ocr
210
+ pickscore: drawbench
211
+ reward_ckpt_path: ./models/reward_ckpts
212
+ eval_num_steps: 4
213
+ eval_guidance_scale: 1.0
214
+ eval_batch_size: 16
215
+ num_media_images: 64
216
+ eval_seed: 12345
217
+ prompt_path: dataset/pickscore
218
+ output_dir: ./outputs/meanflowawm_mix
meanflowawm_ocr/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4c3b1b904c85c680671e6736070cf64b7a4ba408985dd2055f80703766fe9c8
3
+ size 43203307
meanflowawm_ocr/config.yaml ADDED
@@ -0,0 +1,198 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 400
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.01
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ ocr: 1.0
103
+ reward_ckpt_path: ./models/reward_ckpts
104
+ log_sample_images: 8
105
+ central_diff_epsilon: 5.0
106
+ cd_velocity_source: noise_minus_x0
107
+ share_cd_with_old: true
108
+ num_training_timesteps_per_sample: 4
109
+ rl_min_timestep: 0
110
+ rl_max_timestep: 1000
111
+ diffusion_ratio: 0.5
112
+ consistency_ratio: 0.25
113
+ use_ref_v_model: false
114
+ guidance_scale: 1.0
115
+ aux_v_enabled: false
116
+ max_aux_sample: 0
117
+ renoise_per_epoch: 10
118
+ aux_v_gradient_accum: 1
119
+ aux_v_batch_size: 0
120
+ rollout_type: ode
121
+ reuse_x_t: false
122
+ solver:
123
+ generator:
124
+ lr: 3.0e-06
125
+ warmup_steps: 0
126
+ beta1: 0.9
127
+ beta2: 0.999
128
+ eps: 1.0e-08
129
+ weight_decay: 0.0001
130
+ max_grad_norm: 1.0
131
+ fake_score:
132
+ lr: 1.0e-06
133
+ warmup_steps: 500
134
+ beta1: 0.9
135
+ beta2: 0.999
136
+ eps: 1.0e-08
137
+ weight_decay: 0.0
138
+ max_grad_norm: 1.0
139
+ aux_v:
140
+ lr: 1.0e-05
141
+ warmup_steps: 0
142
+ beta1: 0.9
143
+ beta2: 0.999
144
+ eps: 1.0e-08
145
+ weight_decay: 0.0
146
+ max_grad_norm: 1.0
147
+ train:
148
+ total_steps: 0
149
+ save_interval: 100
150
+ log_interval: 1
151
+ seed: 42
152
+ batch_size: 9
153
+ gradient_accumulation_steps: 1
154
+ autocast_dtype: bf16
155
+ resume_from: ''
156
+ use_ema: true
157
+ ema_decay: 0.9
158
+ ema_every: 1
159
+ distributed:
160
+ strategy: fsdp
161
+ fsdp_sharding: full_shard
162
+ ddp_find_unused_parameters: null
163
+ logging:
164
+ project: fewsteprl
165
+ run_name: meanflowawm_ocr
166
+ entity: ''
167
+ tags:
168
+ - sd35m
169
+ - fewstep-rl
170
+ - kl-u
171
+ - ocr
172
+ - awm
173
+ - klw1e-4
174
+ - '512'
175
+ enabled: true
176
+ eval:
177
+ enabled: true
178
+ eval_interval: 50
179
+ eval_prompt_source: dataset
180
+ eval_prompt_path: prompts.json
181
+ dataset: ocr
182
+ dataset_path: dataset
183
+ reward_fn:
184
+ aesthetic: 1.0
185
+ clipscore: 1.0
186
+ ocr: 1.0
187
+ reward_dataset_map:
188
+ aesthetic: ocr
189
+ clipscore: ocr
190
+ ocr: ocr
191
+ reward_ckpt_path: ./models/reward_ckpts
192
+ eval_num_steps: 4
193
+ eval_guidance_scale: 1.0
194
+ eval_batch_size: 16
195
+ num_media_images: 64
196
+ eval_seed: 12345
197
+ prompt_path: dataset/ocr
198
+ output_dir: ./outputs/meanflowawm_ocr
meanflownft_mix/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:596d25e0e6a7b8fe705a5c69c755f041d196d4a8327e781af43dc1136c8b5426
3
+ size 43203307
meanflownft_mix/config.yaml ADDED
@@ -0,0 +1,186 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ model_type: sd35
7
+ gradient_checkpointing: true
8
+ dtype: bf16
9
+ image_resolution: 512
10
+ generator_lora:
11
+ enabled: true
12
+ rank: 32
13
+ lora_alpha: 64
14
+ target_modules:
15
+ - attn.to_q
16
+ - attn.to_k
17
+ - attn.to_v
18
+ - attn.to_out.0
19
+ - attn.add_k_proj
20
+ - attn.add_v_proj
21
+ - attn.add_q_proj
22
+ - attn.to_add_out
23
+ init_lora_weights: gaussian
24
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
25
+ pre_merge_paths:
26
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
27
+ merge_before_training: true
28
+ fake_score_lora:
29
+ enabled: false
30
+ rank: 32
31
+ lora_alpha: 8
32
+ target_modules:
33
+ - attn.to_q
34
+ - attn.to_k
35
+ - attn.to_v
36
+ - attn.to_out.0
37
+ init_lora_weights: gaussian
38
+ load_path: ''
39
+ pre_merge_paths: []
40
+ merge_before_training: false
41
+ flowmap_model:
42
+ enabled: true
43
+ gate_value: 0.25
44
+ deltatime_type: r
45
+ anyflow_pretrain:
46
+ diffusion_ratio: 0.25
47
+ consistency_ratio: 0.25
48
+ epsilon: 0.5
49
+ cd_velocity_source: noise_minus_x0
50
+ v_target_source: noise_minus_x0
51
+ forward_guidance_scale: 1.0
52
+ weight_type: gaussian
53
+ negative_embedding_path: ''
54
+ drop_text_ratio: 0.1
55
+ ema_warmup_steps: 0
56
+ anyflow_onpolicy:
57
+ num_inference_steps_list:
58
+ - 4
59
+ - 8
60
+ - 16
61
+ - 32
62
+ dmd_weight: 1.0
63
+ dmd_batch_size: 0
64
+ dmd_min_timestep: 0
65
+ dmd_max_timestep: 1000
66
+ real_guidance_scale: 0.0
67
+ gradient_normalization: true
68
+ cotrain_forward_kl: true
69
+ discriminator_update_ratio: 1
70
+ rollout_detach_between_jumps: false
71
+ rollout_full_steps: false
72
+ anyflow_dmd:
73
+ rollout_batch_size: 0
74
+ fm_inner_batchsize: 0
75
+ num_inner_epochs_fake: 1
76
+ freq_generator_update: 1
77
+ dmd_batch_size: 0
78
+ generator_rollout_source: fresh_with_grad
79
+ fewstep_rl:
80
+ num_epochs: 10000
81
+ num_batches_per_epoch: 16
82
+ num_inner_epochs: 1
83
+ rl_inner_batch_size: 0
84
+ rl_start_epoch: 0
85
+ num_image_per_prompt: 24
86
+ rl_sample_use_ema: false
87
+ sampling_num_steps: 4
88
+ sampling_guidance_scale: 1.0
89
+ text_max_sequence_length: 128
90
+ per_prompt_stat_tracking: true
91
+ per_prompt_global_std: true
92
+ weight_advantages: false
93
+ adv_clip_max: 5.0
94
+ beta: 0.1
95
+ kl_weight: 0.0001
96
+ decay_type: 1
97
+ rl_decay_interval: 1
98
+ reward_fn:
99
+ clipscore: 1.0
100
+ hpsv2: 1.0
101
+ pickscore: 1.0
102
+ reward_ckpt_path: ./models/reward_ckpts
103
+ log_sample_images: 8
104
+ central_diff_epsilon: 5.0
105
+ cd_velocity_source: noise_minus_x0
106
+ share_cd_with_old: true
107
+ num_training_timesteps_per_sample: 4
108
+ rl_min_timestep: 0
109
+ rl_max_timestep: 1000
110
+ diffusion_ratio: 0.5
111
+ consistency_ratio: 0.25
112
+ use_ref_v_model: false
113
+ guidance_scale: 1.0
114
+ solver:
115
+ generator:
116
+ lr: 3.0e-06
117
+ warmup_steps: 0
118
+ beta1: 0.9
119
+ beta2: 0.999
120
+ eps: 1.0e-08
121
+ weight_decay: 0.0001
122
+ max_grad_norm: 1.0
123
+ fake_score:
124
+ lr: 1.0e-06
125
+ warmup_steps: 500
126
+ beta1: 0.9
127
+ beta2: 0.999
128
+ eps: 1.0e-08
129
+ weight_decay: 0.0
130
+ max_grad_norm: 1.0
131
+ train:
132
+ total_steps: 0
133
+ save_interval: 100
134
+ log_interval: 1
135
+ seed: 42
136
+ batch_size: 9
137
+ gradient_accumulation_steps: 1
138
+ autocast_dtype: bf16
139
+ resume_from: ''
140
+ use_ema: true
141
+ ema_decay: 0.9
142
+ ema_every: 1
143
+ distributed:
144
+ strategy: fsdp
145
+ fsdp_sharding: full_shard
146
+ ddp_find_unused_parameters: null
147
+ logging:
148
+ project: fewsteprl
149
+ run_name: meanflownft_mix
150
+ entity: ''
151
+ tags:
152
+ - sd35m
153
+ - fewstep-rl
154
+ - kl-u
155
+ - mixed
156
+ - klw1e-4
157
+ - align-old-stage3
158
+ - '512'
159
+ enabled: true
160
+ eval:
161
+ enabled: true
162
+ eval_interval: 50
163
+ eval_prompt_source: dataset
164
+ eval_prompt_path: prompts.json
165
+ dataset: drawbench
166
+ dataset_path: dataset
167
+ reward_fn:
168
+ aesthetic: 1.0
169
+ clipscore: 1.0
170
+ hpsv2: 1.0
171
+ ocr: 1.0
172
+ pickscore: 1.0
173
+ reward_dataset_map:
174
+ aesthetic: drawbench
175
+ clipscore: drawbench
176
+ hpsv2: drawbench
177
+ ocr: ocr
178
+ pickscore: drawbench
179
+ reward_ckpt_path: ./models/reward_ckpts
180
+ eval_num_steps: 4
181
+ eval_guidance_scale: 1.0
182
+ eval_batch_size: 16
183
+ num_media_images: 64
184
+ eval_seed: 12345
185
+ prompt_path: dataset/pickscore
186
+ output_dir: ./outputs/meanflownft_mix
meanflownft_ocr/checkpoint-200/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2a3252faaa568aa84318098be22ba846bab7e69da219ad7645f9c7322a78466e
3
+ size 43203307
meanflownft_ocr/config.yaml ADDED
@@ -0,0 +1,220 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 200
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.001
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ ocr: 1.0
103
+ reward_ckpt_path: ./models/reward_ckpts
104
+ log_sample_images: 8
105
+ central_diff_epsilon: 5.0
106
+ cd_velocity_source: noise_minus_x0
107
+ share_cd_with_old: true
108
+ disable_cd: false
109
+ num_training_timesteps_per_sample: 4
110
+ rl_min_timestep: 0
111
+ rl_max_timestep: 1000
112
+ diffusion_ratio: 0.5
113
+ consistency_ratio: 0.25
114
+ use_ref_v_model: false
115
+ guidance_scale: 1.0
116
+ aux_v_enabled: false
117
+ max_aux_sample: 0
118
+ renoise_per_epoch: 10
119
+ aux_v_gradient_accum: 1
120
+ aux_v_batch_size: 0
121
+ rollout_type: sde
122
+ reuse_x_t: false
123
+ opsd_enabled: false
124
+ opsd_rho: 0.1
125
+ opsd_query_sigma: 0.278
126
+ opsd_n_ascent: 2
127
+ opsd_eta_tgt: 1.0
128
+ opsd_reward_kind: hpsv2
129
+ opsd_target_chunk: 6
130
+ opsd_eps_grad: 1.0e-12
131
+ opsd_eps_gamma: 1.0e-05
132
+ opsd_dual_neg: true
133
+ opsd_state_mode: forward
134
+ opsd_dir_mode: grad
135
+ opsd_sigma_mode: fixed
136
+ opsd_logit_mean: -2.053
137
+ opsd_logit_std: 0.6
138
+ renoise_distribution: uniform
139
+ renoise_logit_mean: -2.053
140
+ renoise_logit_std: 0.6
141
+ opsd_r_mode: zero
142
+ opsd_adv_gain: 1.0
143
+ refl_use_advantage: false
144
+ solver:
145
+ generator:
146
+ lr: 3.0e-06
147
+ warmup_steps: 0
148
+ beta1: 0.9
149
+ beta2: 0.999
150
+ eps: 1.0e-08
151
+ weight_decay: 0.0001
152
+ max_grad_norm: 1.0
153
+ fake_score:
154
+ lr: 1.0e-06
155
+ warmup_steps: 500
156
+ beta1: 0.9
157
+ beta2: 0.999
158
+ eps: 1.0e-08
159
+ weight_decay: 0.0
160
+ max_grad_norm: 1.0
161
+ aux_v:
162
+ lr: 1.0e-05
163
+ warmup_steps: 0
164
+ beta1: 0.9
165
+ beta2: 0.999
166
+ eps: 1.0e-08
167
+ weight_decay: 0.0
168
+ max_grad_norm: 1.0
169
+ train:
170
+ total_steps: 0
171
+ save_interval: 100
172
+ log_interval: 1
173
+ seed: 42
174
+ batch_size: 9
175
+ gradient_accumulation_steps: 1
176
+ autocast_dtype: bf16
177
+ resume_from: ''
178
+ use_ema: true
179
+ ema_decay: 0.9
180
+ ema_every: 1
181
+ distributed:
182
+ strategy: fsdp
183
+ fsdp_sharding: full_shard
184
+ ddp_find_unused_parameters: null
185
+ logging:
186
+ project: fewsteprl
187
+ run_name: meanflownft_ocr
188
+ entity: ''
189
+ tags:
190
+ - sd35m
191
+ - fewstep-rl
192
+ - kl-u
193
+ - ocr
194
+ - klw1e-4
195
+ - align-old-stage3
196
+ - '512'
197
+ enabled: true
198
+ eval:
199
+ enabled: true
200
+ eval_interval: 50
201
+ eval_prompt_source: dataset
202
+ eval_prompt_path: prompts.json
203
+ dataset: ocr
204
+ dataset_path: dataset
205
+ reward_fn:
206
+ aesthetic: 1.0
207
+ clipscore: 1.0
208
+ ocr: 1.0
209
+ reward_dataset_map:
210
+ aesthetic: ocr
211
+ clipscore: ocr
212
+ ocr: ocr
213
+ reward_ckpt_path: ./models/reward_ckpts
214
+ eval_num_steps: 4
215
+ eval_guidance_scale: 1.0
216
+ eval_batch_size: 16
217
+ num_media_images: 64
218
+ eval_seed: 12345
219
+ prompt_path: dataset/ocr
220
+ output_dir: ./outputs/meanflownft_ocr
meanflowopsd_mix/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ccb3092eac0581ec9c48b9cb7b7f05b51e383611bc66f6742cbc90dc9e1793bb
3
+ size 43203307
meanflowopsd_mix/config.yaml ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 400
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.001
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ clipscore: 1.0
103
+ hpsv2: 1.0
104
+ pickscore: 1.0
105
+ reward_ckpt_path: ./models/reward_ckpts
106
+ log_sample_images: 8
107
+ central_diff_epsilon: 5.0
108
+ cd_velocity_source: noise_minus_x0
109
+ share_cd_with_old: true
110
+ disable_cd: true
111
+ num_training_timesteps_per_sample: 4
112
+ rl_min_timestep: 0
113
+ rl_max_timestep: 1000
114
+ diffusion_ratio: 0.5
115
+ consistency_ratio: 0.25
116
+ use_ref_v_model: false
117
+ guidance_scale: 1.0
118
+ aux_v_enabled: false
119
+ max_aux_sample: 0
120
+ renoise_per_epoch: 10
121
+ aux_v_gradient_accum: 1
122
+ aux_v_batch_size: 0
123
+ rollout_type: ode
124
+ reuse_x_t: false
125
+ opsd_enabled: true
126
+ opsd_rho: 0.1
127
+ opsd_query_sigma: 0.278
128
+ opsd_n_ascent: 2
129
+ opsd_eta_tgt: 1.0
130
+ opsd_reward_kind: mixed
131
+ opsd_target_chunk: 6
132
+ opsd_eps_grad: 1.0e-12
133
+ opsd_eps_gamma: 1.0e-05
134
+ opsd_dual_neg: true
135
+ opsd_state_mode: forward
136
+ opsd_dir_mode: grad
137
+ opsd_sigma_mode: lognormal
138
+ opsd_logit_mean: -2.053
139
+ opsd_logit_std: 0.6
140
+ renoise_distribution: uniform
141
+ renoise_logit_mean: -2.053
142
+ renoise_logit_std: 0.6
143
+ opsd_r_mode: given_t
144
+ opsd_adv_gain: 1.0
145
+ refl_use_advantage: false
146
+ solver:
147
+ generator:
148
+ lr: 3.0e-06
149
+ warmup_steps: 0
150
+ beta1: 0.9
151
+ beta2: 0.999
152
+ eps: 1.0e-08
153
+ weight_decay: 0.0001
154
+ max_grad_norm: 1.0
155
+ fake_score:
156
+ lr: 1.0e-06
157
+ warmup_steps: 500
158
+ beta1: 0.9
159
+ beta2: 0.999
160
+ eps: 1.0e-08
161
+ weight_decay: 0.0
162
+ max_grad_norm: 1.0
163
+ aux_v:
164
+ lr: 1.0e-05
165
+ warmup_steps: 0
166
+ beta1: 0.9
167
+ beta2: 0.999
168
+ eps: 1.0e-08
169
+ weight_decay: 0.0
170
+ max_grad_norm: 1.0
171
+ train:
172
+ total_steps: 0
173
+ save_interval: 100
174
+ log_interval: 1
175
+ seed: 42
176
+ batch_size: 9
177
+ gradient_accumulation_steps: 1
178
+ autocast_dtype: bf16
179
+ resume_from: ''
180
+ use_ema: true
181
+ ema_decay: 0.9
182
+ ema_every: 1
183
+ distributed:
184
+ strategy: fsdp
185
+ fsdp_sharding: full_shard
186
+ ddp_find_unused_parameters: null
187
+ logging:
188
+ project: fewsteprl
189
+ run_name: meanflowopsd_mix
190
+ entity: ''
191
+ tags:
192
+ - sd35m
193
+ - fewstep-rl
194
+ - mixed
195
+ - kl-u
196
+ - '512'
197
+ - opsd
198
+ - opsd-vtln
199
+ enabled: true
200
+ eval:
201
+ enabled: true
202
+ eval_interval: 50
203
+ eval_prompt_source: dataset
204
+ eval_prompt_path: prompts.json
205
+ dataset: drawbench
206
+ dataset_path: dataset
207
+ reward_fn:
208
+ aesthetic: 1.0
209
+ clipscore: 1.0
210
+ hpsv2: 1.0
211
+ ocr: 1.0
212
+ pickscore: 1.0
213
+ reward_dataset_map:
214
+ aesthetic: drawbench
215
+ clipscore: drawbench
216
+ hpsv2: drawbench
217
+ ocr: ocr
218
+ pickscore: drawbench
219
+ reward_ckpt_path: ./models/reward_ckpts
220
+ eval_num_steps: 4
221
+ eval_guidance_scale: 1.0
222
+ eval_batch_size: 16
223
+ num_media_images: 64
224
+ eval_seed: 12345
225
+ prompt_path: dataset/pickscore
226
+ output_dir: ./outputs/meanflowopsd_mix
nft_mix/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6faf97432f041db6fcf03e0e688b753fe7ccfb2bb1cd5cdb77307b76f6f13419
3
+ size 43203307
nft_mix/config.yaml ADDED
@@ -0,0 +1,227 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 401
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.001
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ clipscore: 1.0
103
+ hpsv2: 1.0
104
+ pickscore: 1.0
105
+ reward_ckpt_path: ./models/reward_ckpts
106
+ log_sample_images: 8
107
+ central_diff_epsilon: 5.0
108
+ cd_velocity_source: noise_minus_x0
109
+ share_cd_with_old: true
110
+ disable_cd: false
111
+ num_training_timesteps_per_sample: 4
112
+ rl_min_timestep: 0
113
+ rl_max_timestep: 1000
114
+ diffusion_ratio: 0.5
115
+ consistency_ratio: 0.25
116
+ use_ref_v_model: false
117
+ guidance_scale: 1.0
118
+ aux_v_enabled: false
119
+ max_aux_sample: 0
120
+ renoise_per_epoch: 10
121
+ aux_v_gradient_accum: 1
122
+ aux_v_batch_size: 0
123
+ rollout_type: sde
124
+ reuse_x_t: false
125
+ opsd_enabled: false
126
+ opsd_rho: 0.1
127
+ opsd_query_sigma: 0.278
128
+ opsd_n_ascent: 2
129
+ opsd_eta_tgt: 1.0
130
+ opsd_reward_kind: hpsv2
131
+ opsd_target_chunk: 6
132
+ opsd_eps_grad: 1.0e-12
133
+ opsd_eps_gamma: 1.0e-05
134
+ opsd_dual_neg: true
135
+ opsd_state_mode: forward
136
+ opsd_dir_mode: grad
137
+ opsd_sigma_mode: fixed
138
+ opsd_logit_mean: -2.053
139
+ opsd_logit_std: 0.6
140
+ renoise_distribution: uniform
141
+ renoise_logit_mean: -2.053
142
+ renoise_logit_std: 0.6
143
+ opsd_r_mode: zero
144
+ opsd_adv_gain: 1.0
145
+ refl_use_advantage: false
146
+ solver:
147
+ generator:
148
+ lr: 3.0e-06
149
+ warmup_steps: 0
150
+ beta1: 0.9
151
+ beta2: 0.999
152
+ eps: 1.0e-08
153
+ weight_decay: 0.0001
154
+ max_grad_norm: 1.0
155
+ fake_score:
156
+ lr: 1.0e-06
157
+ warmup_steps: 500
158
+ beta1: 0.9
159
+ beta2: 0.999
160
+ eps: 1.0e-08
161
+ weight_decay: 0.0
162
+ max_grad_norm: 1.0
163
+ aux_v:
164
+ lr: 1.0e-05
165
+ warmup_steps: 0
166
+ beta1: 0.9
167
+ beta2: 0.999
168
+ eps: 1.0e-08
169
+ weight_decay: 0.0
170
+ max_grad_norm: 1.0
171
+ train:
172
+ total_steps: 0
173
+ save_interval: 100
174
+ log_interval: 1
175
+ seed: 42
176
+ batch_size: 9
177
+ gradient_accumulation_steps: 1
178
+ autocast_dtype: bf16
179
+ resume_from: ''
180
+ use_ema: true
181
+ ema_decay: 0.9
182
+ ema_every: 1
183
+ distributed:
184
+ strategy: fsdp
185
+ fsdp_sharding: full_shard
186
+ ddp_find_unused_parameters: null
187
+ logging:
188
+ project: fewsteprl
189
+ run_name: nft_mix
190
+ entity: ''
191
+ tags:
192
+ - no-s-term
193
+ - sd35m
194
+ - fewstep-rl
195
+ - kl-u
196
+ - mixed
197
+ - klw1e-4
198
+ - align-old-stage3
199
+ - '512'
200
+ enabled: true
201
+ eval:
202
+ enabled: true
203
+ eval_interval: 50
204
+ eval_prompt_source: dataset
205
+ eval_prompt_path: prompts.json
206
+ dataset: drawbench
207
+ dataset_path: dataset
208
+ reward_fn:
209
+ aesthetic: 1.0
210
+ clipscore: 1.0
211
+ hpsv2: 1.0
212
+ ocr: 1.0
213
+ pickscore: 1.0
214
+ reward_dataset_map:
215
+ aesthetic: drawbench
216
+ clipscore: drawbench
217
+ hpsv2: drawbench
218
+ ocr: ocr
219
+ pickscore: drawbench
220
+ reward_ckpt_path: ./models/reward_ckpts
221
+ eval_num_steps: 4
222
+ eval_guidance_scale: 1.0
223
+ eval_batch_size: 16
224
+ num_media_images: 64
225
+ eval_seed: 12345
226
+ prompt_path: dataset/pickscore
227
+ output_dir: ./outputs/nft_mix
reflmx_mix/checkpoint-400/generator_ema.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b25ffea29fef57ee39611bfa7d2987fd6308a628b57b34d7c35013452e05e489
3
+ size 43203307
reflmx_mix/config.yaml ADDED
@@ -0,0 +1,228 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ model:
2
+ pretrained_path: models/stable-diffusion-3.5-medium
3
+ generator_init_path: ''
4
+ fake_score_init_path: ''
5
+ teacher_init_path: ''
6
+ aux_v_init_path: ''
7
+ model_type: sd35
8
+ gradient_checkpointing: true
9
+ dtype: bf16
10
+ image_resolution: 512
11
+ generator_lora:
12
+ enabled: true
13
+ rank: 32
14
+ lora_alpha: 64
15
+ target_modules:
16
+ - attn.to_q
17
+ - attn.to_k
18
+ - attn.to_v
19
+ - attn.to_out.0
20
+ - attn.add_k_proj
21
+ - attn.add_v_proj
22
+ - attn.add_q_proj
23
+ - attn.to_add_out
24
+ init_lora_weights: gaussian
25
+ load_path: ./models/pretrained/meanflownft_loras/anyflow_onpolicy/generator_ema.pt
26
+ pre_merge_paths:
27
+ - ./models/pretrained/meanflownft_loras/anyflow_pretrain/generator_ema.pt
28
+ merge_before_training: true
29
+ fake_score_lora:
30
+ enabled: false
31
+ rank: 32
32
+ lora_alpha: 8
33
+ target_modules:
34
+ - attn.to_q
35
+ - attn.to_k
36
+ - attn.to_v
37
+ - attn.to_out.0
38
+ init_lora_weights: gaussian
39
+ load_path: ''
40
+ pre_merge_paths: []
41
+ merge_before_training: false
42
+ flowmap_model:
43
+ enabled: true
44
+ gate_value: 0.25
45
+ deltatime_type: r
46
+ anyflow_pretrain:
47
+ diffusion_ratio: 0.25
48
+ consistency_ratio: 0.25
49
+ epsilon: 0.5
50
+ cd_velocity_source: noise_minus_x0
51
+ v_target_source: noise_minus_x0
52
+ forward_guidance_scale: 1.0
53
+ weight_type: gaussian
54
+ negative_embedding_path: ''
55
+ drop_text_ratio: 0.1
56
+ ema_warmup_steps: 0
57
+ anyflow_onpolicy:
58
+ num_inference_steps_list:
59
+ - 4
60
+ - 8
61
+ - 16
62
+ - 32
63
+ dmd_weight: 1.0
64
+ dmd_batch_size: 0
65
+ dmd_min_timestep: 0
66
+ dmd_max_timestep: 1000
67
+ real_guidance_scale: 0.0
68
+ gradient_normalization: true
69
+ cotrain_forward_kl: true
70
+ discriminator_update_ratio: 1
71
+ rollout_detach_between_jumps: false
72
+ rollout_full_steps: false
73
+ anyflow_dmd:
74
+ rollout_batch_size: 0
75
+ fm_inner_batchsize: 0
76
+ num_inner_epochs_fake: 1
77
+ freq_generator_update: 1
78
+ dmd_batch_size: 0
79
+ generator_rollout_source: fresh_with_grad
80
+ fewstep_rl:
81
+ num_epochs: 400
82
+ num_batches_per_epoch: 16
83
+ num_inner_epochs: 1
84
+ rl_inner_batch_size: 0
85
+ rl_start_epoch: 0
86
+ num_image_per_prompt: 24
87
+ rl_sample_use_ema: false
88
+ sampling_num_steps: 4
89
+ sampling_guidance_scale: 1.0
90
+ text_max_sequence_length: 128
91
+ per_prompt_stat_tracking: true
92
+ per_prompt_global_std: true
93
+ weight_advantages: false
94
+ adv_clip_max: 5.0
95
+ beta: 0.1
96
+ kl_weight: 0.0001
97
+ clip_range: 0.001
98
+ grpo_noise_level: 0.7
99
+ decay_type: 1
100
+ rl_decay_interval: 1
101
+ reward_fn:
102
+ clipscore: 1.0
103
+ hpsv2: 1.0
104
+ pickscore: 1.0
105
+ reward_ckpt_path: ./models/reward_ckpts
106
+ log_sample_images: 8
107
+ central_diff_epsilon: 5.0
108
+ cd_velocity_source: noise_minus_x0
109
+ share_cd_with_old: true
110
+ disable_cd: true
111
+ num_training_timesteps_per_sample: 4
112
+ rl_min_timestep: 0
113
+ rl_max_timestep: 1000
114
+ diffusion_ratio: 0.5
115
+ consistency_ratio: 0.25
116
+ use_ref_v_model: false
117
+ guidance_scale: 1.0
118
+ aux_v_enabled: false
119
+ max_aux_sample: 0
120
+ renoise_per_epoch: 10
121
+ aux_v_gradient_accum: 1
122
+ aux_v_batch_size: 0
123
+ rollout_type: ode
124
+ reuse_x_t: false
125
+ opsd_enabled: true
126
+ opsd_rho: 0.1
127
+ opsd_query_sigma: 0.278
128
+ opsd_n_ascent: 1
129
+ opsd_eta_tgt: 1.0
130
+ opsd_reward_kind: mixed
131
+ opsd_target_chunk: 6
132
+ opsd_eps_grad: 1.0e-12
133
+ opsd_eps_gamma: 1.0e-05
134
+ opsd_dual_neg: false
135
+ opsd_state_mode: forward
136
+ opsd_dir_mode: grad
137
+ opsd_sigma_mode: lognormal
138
+ opsd_logit_mean: -2.053
139
+ opsd_logit_std: 0.6
140
+ renoise_distribution: uniform
141
+ renoise_logit_mean: -2.053
142
+ renoise_logit_std: 0.6
143
+ opsd_r_mode: given_t
144
+ opsd_adv_gain: 1.0
145
+ refl_use_advantage: false
146
+ solver:
147
+ generator:
148
+ lr: 3.0e-06
149
+ warmup_steps: 0
150
+ beta1: 0.9
151
+ beta2: 0.999
152
+ eps: 1.0e-08
153
+ weight_decay: 0.0001
154
+ max_grad_norm: 1.0
155
+ fake_score:
156
+ lr: 1.0e-06
157
+ warmup_steps: 500
158
+ beta1: 0.9
159
+ beta2: 0.999
160
+ eps: 1.0e-08
161
+ weight_decay: 0.0
162
+ max_grad_norm: 1.0
163
+ aux_v:
164
+ lr: 1.0e-05
165
+ warmup_steps: 0
166
+ beta1: 0.9
167
+ beta2: 0.999
168
+ eps: 1.0e-08
169
+ weight_decay: 0.0
170
+ max_grad_norm: 1.0
171
+ train:
172
+ total_steps: 0
173
+ save_interval: 50
174
+ log_interval: 1
175
+ seed: 42
176
+ batch_size: 9
177
+ gradient_accumulation_steps: 1
178
+ autocast_dtype: bf16
179
+ resume_from: ''
180
+ use_ema: true
181
+ ema_decay: 0.9
182
+ ema_every: 1
183
+ distributed:
184
+ strategy: fsdp
185
+ fsdp_sharding: full_shard
186
+ ddp_find_unused_parameters: null
187
+ logging:
188
+ project: fewsteprl
189
+ run_name: reflmx_mix
190
+ entity: ''
191
+ tags:
192
+ - sd35m
193
+ - fewstep-rl
194
+ - mixed
195
+ - kl-u
196
+ - '512'
197
+ - opsd
198
+ - refl
199
+ - refl-corner
200
+ - refl-mixed
201
+ enabled: true
202
+ eval:
203
+ enabled: true
204
+ eval_interval: 50
205
+ eval_prompt_source: dataset
206
+ eval_prompt_path: prompts.json
207
+ dataset: drawbench
208
+ dataset_path: dataset
209
+ reward_fn:
210
+ aesthetic: 1.0
211
+ clipscore: 1.0
212
+ hpsv2: 1.0
213
+ ocr: 1.0
214
+ pickscore: 1.0
215
+ reward_dataset_map:
216
+ aesthetic: drawbench
217
+ clipscore: drawbench
218
+ hpsv2: drawbench
219
+ ocr: ocr
220
+ pickscore: drawbench
221
+ reward_ckpt_path: ./models/reward_ckpts
222
+ eval_num_steps: 4
223
+ eval_guidance_scale: 1.0
224
+ eval_batch_size: 16
225
+ num_media_images: 64
226
+ eval_seed: 12345
227
+ prompt_path: dataset/pickscore
228
+ output_dir: ./outputs/reflmx_mix