diff --git a/.gitattributes b/.gitattributes index 06fb6ca0af4f55d673780e57388423b5fc83485f..dba338ac7c4cb1c8a6cdc5a1ed74d187bd75fd29 100644 --- a/.gitattributes +++ b/.gitattributes @@ -39,3 +39,8 @@ loras/be_v11/loss_log.db filter=lfs diff=lfs merge=lfs -text loras/be_v14/loss_log.db filter=lfs diff=lfs merge=lfs -text loras/be_v15/loss_log.db filter=lfs diff=lfs merge=lfs -text loras/be_v16/loss_log.db filter=lfs diff=lfs merge=lfs -text +loras/be_v7/loss_log.db filter=lfs diff=lfs merge=lfs -text +loras/be_v8/loss_log.db filter=lfs diff=lfs merge=lfs -text +loras/be_v9/loss_log.db filter=lfs diff=lfs merge=lfs -text +loras/exp_v13/loss_log.db filter=lfs diff=lfs merge=lfs -text +loras/output/wan22_ti2v_BE_v1/samples/1787456236235__000000000_0.webp filter=lfs diff=lfs merge=lfs -text diff --git a/loras/be_v7/be_v7_000000750.safetensors b/loras/be_v7/be_v7_000000750.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..cac09b64863e43ecb68b9a02fe693c7e3d743041 --- /dev/null +++ b/loras/be_v7/be_v7_000000750.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6857299ea6a5f8afc50896d8d93fbeceeefbfc56083166bce79dbbb03bbd7d6 +size 674249552 diff --git a/loras/be_v7/be_v7_000000800.safetensors b/loras/be_v7/be_v7_000000800.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a0a9493a06aa4281c2a446f838fe98d80b56cf15 --- /dev/null +++ b/loras/be_v7/be_v7_000000800.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:acf1c7f6cd0cda12c26186bf7472d91b1267bf68c2bc2408bd383a3d2c3f84a8 +size 674249552 diff --git a/loras/be_v7/be_v7_000000850.safetensors b/loras/be_v7/be_v7_000000850.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a9f3ed08351382e8c7b7930c0c083244aef97caa --- /dev/null +++ b/loras/be_v7/be_v7_000000850.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f709f71d12d66246ba077cca139fbe0a122c2da62efd1f8e1cb563240e0a3e5 +size 674249552 diff --git a/loras/be_v7/config.yaml b/loras/be_v7/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..6981e86e4acdfbb04745e30d9599f228ecea45f1 --- /dev/null +++ b/loras/be_v7/config.yaml @@ -0,0 +1,126 @@ +job: extension +config: + name: be_v7 + process: + - type: diffusion_trainer + training_folder: /mnt/runpod-slim/models/loras + sqlite_db_path: /app/ai-toolkit/aitk_db.db + device: cuda + trigger_word: null + performance_log_every: 10 + network: + type: lora + linear: 32 + linear_alpha: 32 + conv: 16 + conv_alpha: 16 + lokr_full_rank: true + lokr_factor: -1 + network_kwargs: + ignore_if_contains: [] + save: + dtype: bf16 + save_every: 50 + max_step_saves_to_keep: 20 + save_format: diffusers + push_to_hub: false + datasets: + - folder_path: /mnt/runpod-slim/datasets/be_v7 + mask_path: null + mask_min_value: 0.1 + default_caption: '' + caption_ext: txt + caption_dropout_rate: 0.05 + cache_latents_to_disk: true + is_reg: false + network_weight: 1 + resolution: + - 512 + controls: [] + shrink_video_to_frames: true + num_frames: 1 + flip_x: false + flip_y: false + num_repeats: 1 + do_i2v: true + fps: 24 + auto_frame_count: true + train: + batch_size: 1 + bypass_guidance_embedding: false + steps: 1250 + gradient_accumulation: 1 + train_unet: true + train_text_encoder: false + gradient_checkpointing: true + noise_scheduler: flowmatch + optimizer: adamw8bit + timestep_type: weighted + content_or_style: content + optimizer_params: + weight_decay: 0.0001 + unload_text_encoder: false + cache_text_embeddings: true + lr: 0.0001 + ema_config: + use_ema: false + ema_decay: 0.99 + skip_first_sample: true + force_first_sample: false + disable_sampling: true + dtype: bf16 + diff_output_preservation: false + diff_output_preservation_multiplier: 1 + diff_output_preservation_class: person + switch_boundary_every: 1 + loss_type: mse + audio_loss_multiplier: 1 + logging: + log_every: 1 + use_ui_logger: true + model: + name_or_path: Lightricks/LTX-2.3/ltx-2.3-22b-dev.safetensors + quantize: false + qtype: qfloat8 + quantize_te: false + qtype_te: qfloat8 + arch: ltx2.3 + low_vram: false + model_kwargs: {} + compile: false + layer_offloading: false + layer_offloading_text_encoder_percent: 1 + layer_offloading_transformer_percent: 1 + sample: + sampler: flowmatch + sample_every: 250 + sample_start_step: 0 + width: 768 + height: 768 + samples: + - prompt: woman with red hair, playing chess at the park, bomb going off in + the background + - prompt: a woman holding a coffee cup, in a beanie, sitting at a cafe + - prompt: a horse is a DJ at a night club, fish eye lens, smoke machine, lazer + lights, holding a martini + - prompt: a man showing off his cool new t shirt at the beach, a shark is jumping + out of the water in the background + - prompt: a bear building a log cabin in the snow covered mountains + - prompt: woman playing the guitar, on stage, singing a song, laser lights, + punk rocker + - prompt: hipster man with a beard, building a chair, in a wood shop + - prompt: photo of a man, white background, medium shot, modeling clothing, + studio lighting, white backdrop + - prompt: a man holding a sign that says, 'this is a sign' + - prompt: a bulldog, in a post apocalyptic world, with a shotgun, in a leather + jacket, in a desert, with a motorcycle + neg: '' + seed: 42 + walk_seed: true + guidance_scale: 4 + sample_steps: 30 + num_frames: 121 + fps: 24 +meta: + name: be_v7 + version: '1.0' diff --git a/loras/be_v7/log.txt b/loras/be_v7/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..4a73c74523d4ec139209ebd3218fd749e7f3d12c --- /dev/null +++ b/loras/be_v7/log.txt @@ -0,0 +1,266 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/mnt/runpod-slim/models/loras", + "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": null, + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 32, + "linear_alpha": 32, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 50, + "max_step_saves_to_keep": 20, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/mnt/runpod-slim/datasets/be_v7", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 1, + "flip_x": false, + "flip_y": false, + "num_repeats": 1, + "do_i2v": true, + "fps": 24, + "auto_frame_count": true + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 1250, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "weighted", + "content_or_style": "content", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0001, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": true, + "force_first_sample": false, + "disable_sampling": true, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 1, + "loss_type": "mse", + "audio_loss_multiplier": 1 + }, + "logging": { + "log_every": 1, + "use_ui_logger": true + }, + "model": { + "name_or_path": "Lightricks/LTX-2.3/ltx-2.3-22b-dev.safetensors", + "quantize": false, + "qtype": "qfloat8", + "quantize_te": false, + "qtype_te": "qfloat8", + "arch": "ltx2.3", + "low_vram": false, + "model_kwargs": {}, + "compile": false, + "layer_offloading": false, + "layer_offloading_text_encoder_percent": 1, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 250, + "sample_start_step": 0, + "width": 768, + "height": 768, + "samples": [ + { + "prompt": "woman with red hair, playing chess at the park, bomb going off in the background" + }, + { + "prompt": "a woman holding a coffee cup, in a beanie, sitting at a cafe" + }, + { + "prompt": "a horse is a DJ at a night club, fish eye lens, smoke machine, lazer lights, holding a martini" + }, + { + "prompt": "a man showing off his cool new t shirt at the beach, a shark is jumping out of the water in the background" + }, + { + "prompt": "a bear building a log cabin in the snow covered mountains" + }, + { + "prompt": "woman playing the guitar, on stage, singing a song, laser lights, punk rocker" + }, + { + "prompt": "hipster man with a beard, building a chair, in a wood shop" + }, + { + "prompt": "photo of a man, white background, medium shot, modeling clothing, studio lighting, white backdrop" + }, + { + "prompt": "a man holding a sign that says, 'this is a sign'" + }, + { + "prompt": "a bulldog, in a post apocalyptic world, with a shotgun, in a leather jacket, in a desert, with a motorcycle" + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 30, + "num_frames": 121, + "fps": 24 + } +} +Changing sample extention to animated webp +Using SQLite database at /app/ai-toolkit/aitk_db.db +Job ID: "50dc67ef-1258-4817-b9d9-f23660cee77b" + +############################################# +# Running job: be_v7 +############################################# + + +Running 1 process +Loading LTX2 model +Loading transformer +Loading text encoder + Loading weights: 0%| | 0/1065 [00:00' not supported between instances of 'NoneType' and 'int' + +======================================== +Result: + - 0 completed jobs + - 1 failure +======================================== +Traceback (most recent call last): + File "/app/ai-toolkit/run.py", line 147, in + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2252, in run + self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/data_loader.py", line 707, in get_dataloader_from_datasets + dataset = AiToolkitDataset(config, batch_size=batch_size, sd=sd) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/data_loader.py", line 396, in __init__ + self.is_video = dataset_config.num_frames > 1 or dataset_config.auto_frame_count + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: '>' not supported between instances of 'NoneType' and 'int' + +Job process terminated with exit code 1. diff --git a/loras/output/wan22_ti2v_BE_v1/logs/0_log.txt b/loras/output/wan22_ti2v_BE_v1/logs/0_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..14a31f014bb918679f95229f4a438ad3a5750c9e --- /dev/null +++ b/loras/output/wan22_ti2v_BE_v1/logs/0_log.txt @@ -0,0 +1,428 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/workspace/runpod-slim/ComfyUI/models/loras/output", + "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": "BR3AST3XPANSION", + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 64, + "linear_alpha": 64, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 200, + "max_step_saves_to_keep": 40, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/workspace/runpod-slim/datasets/wan22_ti2v_be_v1", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "BR3AST3XPANSION", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": false, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512, + 768, + 1024 + ], + "controls": [], + "shrink_video_to_frames": false, + "num_frames": 81, + "flip_x": false, + "flip_y": false, + "num_repeats": 1, + "fps": 16 + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 5000, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "linear", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": true, + "cache_text_embeddings": true, + "lr": 0.0004, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 10, + "loss_type": "mse" + }, + "logging": { + "log_every": 1, + "use_ui_logger": true + }, + "model": { + "name_or_path": "ai-toolkit/Wan2.2-I2V-A14B-Diffusers-bf16", + "quantize": false, + "qtype": "qfloat8", + "quantize_te": false, + "qtype_te": "qfloat8", + "arch": "wan22_14b_i2v", + "low_vram": false, + "model_kwargs": { + "train_high_noise": true, + "train_low_noise": true + }, + "compile": true, + "layer_offloading": false, + "layer_offloading_text_encoder_percent": 1, + "layer_offloading_transformer_percent": 1, + "block_compile": true + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 200, + "sample_start_step": 0, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "A woman bartender with wavy pink hair wearing a pink tank top standing behind the bar with one hand on a drink mixer looks at the camera before picking up the shaker and beginning to shake a cocktail drink when a BR3AST3XPANSION occurs causing her breasts to dramatically grow in weight and size almost in sync with her mixer shakes and the subsequent breasts side to side wobble, she looks at the camera unbothered by the dramatic breast size changes happening as they continue to grow and stretch her pink tank top before she looks down at her chest as her breasts reach and maintain their largest size. Dark indoor nightlife venue lighting, high detail and dramatic smooth fluid breast motion dynamics." + }, + { + "prompt": "A woman with brown hair wearing a white lace nightie thats tied in the front standing in a forest taking a selfie video looks down at her chest with a look of slight concern, suddenly a BR3AST3XPANSION occurs causing a rapid doubling of her breast size which strains her top to losing a piece, she smiles confidently and shakes her breasts side to side playfully as they continue to grow. Natural outdoor lighting and captivating smooth fluid motion." + }, + { + "prompt": "A woman with red hair sitting profile at an office cubicle desk wearing a thin black long sleeve shirt, as she types on her computer a BR3AST3XPANSION occurs causing her breasts to dramatically and significantly expand horizontally forward toward her computer, she looks down and tries to continue typing unbothered while her breasts continue to grow to their peak volume and weight. Indoor fluorescent office lighting and quick but smooth fluid movement dynamics." + }, + { + "prompt": "A woman scientist with brown hair and safety glasses leaning casually toward the camera smiling when suddenly a look of surprise comes across her face and a BR3AST3XPANSION occurs causing her breasts to dramatically balloon to a much larger size and weight evident by her breasts nearly lunging at the camera before reaching their max size and falling to the table she was leaning on. Classroom bright lighting and dramatic smooth fluid motion." + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 30, + "num_frames": 61, + "fps": 16 + } +} +Changing sample extention to animated webp +Using SQLite database at /app/ai-toolkit/aitk_db.db +Job ID: "389b8bda-5e1e-4e33-b692-5a98d569086c" + +############################################# +# Running job: wan22_ti2v_BE_v1 +############################################# + + +Running 1 process +Loading Wan model +Loading transformer 1 + config.json: 0%| | 0.00/550 [00:00 + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2481, in run + self.sample(self.step_num) + File "/app/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 408, in sample + super().sample(step, is_first) + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 378, in sample + self.sd.generate_images(gen_img_config_list, sampler=sample_config.sampler) + File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 124, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/models/base_model.py", line 702, in generate_images + img = self.generate_single_image( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/diffusion_models/wan22/wan22_14b_i2v_model.py", line 83, in generate_single_image + output = pipeline( + ^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/diffusion_models/wan22/wan22_pipeline.py", line 269, in __call__ + latents = self.scheduler.step( + ^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/diffusers/schedulers/scheduling_flow_match_euler_discrete.py", line 514, in step + prev_sample = sample + dt * model_output + ~~~~~~~^~~~~~~~~~~~~~~~~~~ +RuntimeError: The size of tensor a (36) must match the size of tensor b (16) at non-singleton dimension 1 + +Job process terminated with exit code 1. diff --git a/loras/output/wan22_ti2v_BE_v1/logs/10_log.txt b/loras/output/wan22_ti2v_BE_v1/logs/10_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a6b4b536b7fda29c418d33c8cf530d10885f8bf9 --- /dev/null +++ b/loras/output/wan22_ti2v_BE_v1/logs/10_log.txt @@ -0,0 +1,265 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/workspace/runpod-slim/ComfyUI/models/loras/output", + "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": "BR3AST3XPANSION", + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 32, + "linear_alpha": 32, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 200, + "max_step_saves_to_keep": 40, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/workspace/runpod-slim/datasets/wan22_ti2v_be_v1", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "BR3AST3XPANSION", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512, + 768 + ], + "controls": [], + "shrink_video_to_frames": true, + "num_frames": 81, + "flip_x": false, + "flip_y": false, + "num_repeats": 1, + "fps": 16, + "auto_frame_count": true + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 3000, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "linear", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0003, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": true, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 10, + "loss_type": "mse" + }, + "logging": { + "log_every": 1, + "use_ui_logger": true + }, + "model": { + "name_or_path": "ai-toolkit/Wan2.2-I2V-A14B-Diffusers-bf16", + "quantize": true, + "qtype": "qfloat8", + "quantize_te": true, + "qtype_te": "qfloat8", + "arch": "wan22_14b_i2v", + "low_vram": false, + "model_kwargs": { + "train_high_noise": true, + "train_low_noise": false + }, + "compile": false, + "layer_offloading": false, + "layer_offloading_text_encoder_percent": 1, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 400, + "sample_start_step": 0, + "width": 512, + "height": 512, + "samples": [], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 30, + "num_frames": 81, + "fps": 16 + } +} +Changing sample extention to animated webp +Using SQLite database at /app/ai-toolkit/aitk_db.db +Job ID: "389b8bda-5e1e-4e33-b692-5a98d569086c" + +############################################# +# Running job: wan22_ti2v_BE_v1 +############################################# + + +Running 1 process +Loading Wan model +Loading transformer 1 + Downloading bytes: | 0.00B + Reconstructing (incomplete total...): | | 0.00B / 0.00B  + + Fetching 3 files: 0%| | 0/3 [00:00 + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2633, in run + loss_dict = self.hook_train_loop(batch_list) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2263, in hook_train_loop + loss = self.train_single_accumulation(batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2152, in train_single_accumulation + noise_pred = self.predict_noise( + ^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 1364, in predict_noise + return self.sd.predict_noise( + ^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/models/base_model.py", line 974, in predict_noise + noise_pred = self.get_noise_prediction( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/diffusion_models/wan22/wan22_14b_i2v_model.py", line 123, in get_noise_prediction + if len(frames.shape) == 4: + ^^^^^^^^^^^^ +AttributeError: 'NoneType' object has no attribute 'shape' + wan22_ti2v_BE_v1: 0%| | 0/3000 [00:00 + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2633, in run + loss_dict = self.hook_train_loop(batch_list) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2263, in hook_train_loop + loss = self.train_single_accumulation(batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2152, in train_single_accumulation + noise_pred = self.predict_noise( + ^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 1364, in predict_noise + return self.sd.predict_noise( + ^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/models/base_model.py", line 974, in predict_noise + noise_pred = self.get_noise_prediction( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/diffusion_models/wan22/wan22_14b_i2v_model.py", line 123, in get_noise_prediction + if len(frames.shape) == 4: + ^^^^^^^^^^^^ +AttributeError: 'NoneType' object has no attribute 'shape' + wan22_ti2v_BE_v1: 0%| | 0/3000 [00:00' not supported between instances of 'NoneType' and 'int' + +======================================== +Result: + - 0 completed jobs + - 1 failure +======================================== +Traceback (most recent call last): + File "/app/ai-toolkit/run.py", line 147, in + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2252, in run + self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/data_loader.py", line 707, in get_dataloader_from_datasets + dataset = AiToolkitDataset(config, batch_size=batch_size, sd=sd) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/data_loader.py", line 396, in __init__ + self.is_video = dataset_config.num_frames > 1 or dataset_config.auto_frame_count + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: '>' not supported between instances of 'NoneType' and 'int' + +Job process terminated with exit code 1. diff --git a/loras/output/wan22_ti2v_BE_v1/logs/14_log.txt b/loras/output/wan22_ti2v_BE_v1/logs/14_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..73f4a8914007c8ef6bbdd3954cc5adf7c0f5c6dd --- /dev/null +++ b/loras/output/wan22_ti2v_BE_v1/logs/14_log.txt @@ -0,0 +1,209 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/workspace/runpod-slim/ComfyUI/models/loras/output", + "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": "BR3AST3XPANSION", + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 32, + "linear_alpha": 32, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 200, + "max_step_saves_to_keep": 40, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/workspace/runpod-slim/datasets/wan22_ti2v_be_v1", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "BR3AST3XPANSION", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512, + 768 + ], + "controls": [], + "shrink_video_to_frames": false, + "num_frames": null, + "flip_x": false, + "flip_y": false, + "num_repeats": 1, + "fps": 16, + "auto_frame_count": true + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 3000, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "linear", + "content_or_style": "content", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0003, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": true, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 10, + "loss_type": "mse" + }, + "logging": { + "log_every": 1, + "use_ui_logger": true + }, + "model": { + "name_or_path": "ai-toolkit/Wan2.2-I2V-A14B-Diffusers-bf16", + "quantize": true, + "qtype": "qfloat8", + "quantize_te": true, + "qtype_te": "qfloat8", + "arch": "wan22_14b_i2v", + "low_vram": false, + "model_kwargs": { + "train_high_noise": true, + "train_low_noise": false + }, + "compile": false, + "layer_offloading": false, + "layer_offloading_text_encoder_percent": 1, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 400, + "sample_start_step": 0, + "width": 512, + "height": 512, + "samples": [], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 30, + "num_frames": 81, + "fps": 16 + } +} +Changing sample extention to animated webp +Using SQLite database at /app/ai-toolkit/aitk_db.db +Job ID: "389b8bda-5e1e-4e33-b692-5a98d569086c" + +############################################# +# Running job: wan22_ti2v_BE_v1 +############################################# + + +Running 1 process +Loading Wan model +Loading transformer 1 + Downloading bytes: | 0.00B + Reconstructing (incomplete total...): | | 0.00B / 0.00B  + + Fetching 3 files: 0%| | 0/3 [00:00' not supported between instances of 'NoneType' and 'int' + +======================================== +Result: + - 0 completed jobs + - 1 failure +======================================== +Traceback (most recent call last): + File "/app/ai-toolkit/run.py", line 147, in + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2252, in run + self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/data_loader.py", line 707, in get_dataloader_from_datasets + dataset = AiToolkitDataset(config, batch_size=batch_size, sd=sd) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/data_loader.py", line 396, in __init__ + self.is_video = dataset_config.num_frames > 1 or dataset_config.auto_frame_count + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: '>' not supported between instances of 'NoneType' and 'int' + +Job process terminated with exit code 1. diff --git a/loras/output/wan22_ti2v_BE_v1/logs/15_log.txt b/loras/output/wan22_ti2v_BE_v1/logs/15_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..5f3cefdc70d06cbd6577f12c69c8e6103bf5689e --- /dev/null +++ b/loras/output/wan22_ti2v_BE_v1/logs/15_log.txt @@ -0,0 +1,299 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/workspace/runpod-slim/ComfyUI/models/loras/output", + "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": "BR3AST3XPANSION", + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 32, + "linear_alpha": 32, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 200, + "max_step_saves_to_keep": 40, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/workspace/runpod-slim/datasets/wan22_ti2v_be_v1", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "BR3AST3XPANSION", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": true, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512, + 768 + ], + "controls": [], + "shrink_video_to_frames": false, + "num_frames": null, + "flip_x": false, + "flip_y": false, + "num_repeats": 1, + "do_i2v": true, + "do_audio": true, + "fps": 24, + "auto_frame_count": true + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 3000, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "weighted", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0003, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": true, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 10, + "loss_type": "mse", + "audio_loss_multiplier": 1 + }, + "logging": { + "log_every": 1, + "use_ui_logger": true + }, + "model": { + "name_or_path": "Lightricks/LTX-2.3/ltx-2.3-22b-dev.safetensors", + "quantize": false, + "qtype": "qfloat8", + "quantize_te": false, + "qtype_te": "qfloat8", + "arch": "ltx2.3", + "low_vram": false, + "model_kwargs": {}, + "compile": false, + "layer_offloading": false, + "layer_offloading_text_encoder_percent": 1, + "layer_offloading_transformer_percent": 1 + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 400, + "sample_start_step": 0, + "width": 768, + "height": 768, + "samples": [], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 30, + "num_frames": 121, + "fps": 24 + } +} +Changing sample extention to animated webp +Using SQLite database at /app/ai-toolkit/aitk_db.db +Job ID: "389b8bda-5e1e-4e33-b692-5a98d569086c" + +############################################# +# Running job: wan22_ti2v_BE_v1 +############################################# + + +Running 1 process +Loading LTX2 model +Loading transformer + + ltx-2.3-22b-dev.safetensors: reconstructing file: 0%| | 0.00B / 46.1GB  ltx-2.3-22b-dev.safetensors: downloading bytes: | 0.00B ltx-2.3-22b-dev.safetensors: downloading bytes: | 601kB ltx-2.3-22b-dev.safetensors: downloading bytes: | 4.33MB, 58.7kB/s ltx-2.3-22b-dev.safetensors: downloading bytes: | 26.4MB, 421kB/s ltx-2.3-22b-dev.safetensors: downloading bytes: | 140MB, 2.56MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: | 222MB, 13.6MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: | 392MB, 21.3MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ▏ | 637MB, 37.5MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 0%| | 134MB / 46.1GB  ltx-2.3-22b-dev.safetensors: downloading bytes: ▏ | 912MB, 60.6MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 1%| | 402MB / 46.1GB, 12.8MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ▎ | 1.26GB, 98.3MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 2%|▏ | 1.01GB / 46.1GB, 38.1MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ▎ | 1.47GB, 118MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ▍ | 1.78GB, 136MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ▍ | 2.06GB, 175MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 4%|▍ | 2.01GB / 46.1GB, 107MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ▌ | 2.38GB, 188MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ▌ | 2.64GB, 217MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ▋ | 3.03GB, 249MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ▊ | 3.48GB, 282MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 7%|▋ | 3.22GB / 46.1GB, 197MB/s  + ltx-2.3-22b-dev.safetensors: reconstructing file: 8%|▊ | 3.55GB / 46.1GB, 287MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ▊ | 3.94GB, 323MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ▉ | 4.34GB, 350MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 10%|▉ | 4.49GB / 46.1GB, 315MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ▉ | 4.58GB, 374MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █ | 4.84GB, 392MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █ | 5.09GB, 412MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █▏ | 5.48GB, 437MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █▎ | 5.82GB, 458MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █▎ | 6.10GB, 495MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █▍ | 6.60GB, 513MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █▍ | 6.88GB, 537MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 13%|█▎ | 6.10GB / 46.1GB, 392MB/s  + ltx-2.3-22b-dev.safetensors: reconstructing file: 14%|█▍ | 6.44GB / 46.1GB, 499MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: █▌ | 7.17GB, 566MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 15%|█▍ | 6.91GB / 46.1GB, 524MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: █▋ | 7.76GB, 585MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 17%|█▋ | 7.78GB / 46.1GB, 556MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: █▊ | 8.15GB, 628MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █▊ | 8.45GB, 638MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █▉ | 8.94GB, 652MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 20%|██ | 9.26GB / 46.1GB, 623MB/s  + ltx-2.3-22b-dev.safetensors: reconstructing file: 21%|██ | 9.66GB / 46.1GB, 724MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ██ | 9.42GB, 699MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██ | 9.81GB, 719MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██▏ | 10.3GB, 734MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 23%|██▎ | 10.5GB / 46.1GB, 753MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ██▎ | 10.8GB, 782MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██▍ | 11.4GB, 806MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 24%|██▍ | 11.1GB / 46.1GB, 775MB/s  + ltx-2.3-22b-dev.safetensors: reconstructing file: 26%|██▌ | 12.1GB / 46.1GB, 820MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ██▌ | 11.7GB, 842MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 27%|██▋ | 12.7GB / 46.1GB, 893MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ██▋ | 12.1GB, 852MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██▋ | 12.5GB, 864MB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 29%|██▉ | 13.3GB / 46.1GB, 936MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ██▊ | 12.8GB, 885MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██▉ | 13.4GB, 914MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██▉ | 13.7GB, 930MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███ | 14.1GB, 965MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███ | 14.4GB, 967MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▏ | 14.7GB, 989MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▏ | 15.0GB, 994MB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▎ | 15.3GB, 1.01GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▍ | 15.6GB, 1.03GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▍ | 15.9GB, 1.04GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▌ | 16.2GB, 1.05GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▋ | 16.8GB, 1.07GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▋ | 17.2GB, 1.09GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▊ | 17.5GB, 1.11GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▊ | 17.8GB, 1.12GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███▉ | 18.2GB, 1.14GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████ | 18.6GB, 1.15GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████ | 18.9GB, 1.17GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████▏ | 19.3GB, 1.19GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████▏ | 19.6GB, 1.19GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████▎ | 20.0GB, 1.20GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████▍ | 20.3GB, 1.23GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████▌ | 20.8GB, 1.24GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████▌ | 21.1GB, 1.25GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████▋ | 21.6GB, 1.26GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 33%|███▎ | 15.1GB / 46.1GB, 971MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ████▊ | 21.9GB, 1.29GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 36%|███▌ | 16.7GB / 46.1GB, 830MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ████▊ | 22.4GB, 1.30GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████▉ | 22.8GB, 1.31GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 40%|███▉ | 18.3GB / 46.1GB, 946MB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: █████ | 23.2GB, 1.33GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 43%|████▎ | 19.9GB / 46.1GB, 1.06GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: █████ | 23.6GB, 1.35GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 47%|████▋ | 21.5GB / 46.1GB, 1.17GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: █████▏ | 23.8GB, 1.35GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 50%|████▉ | 23.1GB / 46.1GB, 1.28GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: █████▎ | 24.3GB, 1.36GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 53%|█████▎ | 24.7GB / 46.1GB, 1.38GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: █████▎ | 24.7GB, 1.37GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █████▍ | 25.0GB, 1.38GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █████▌ | 25.4GB, 1.40GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █████▌ | 25.8GB, 1.40GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █████▋ | 26.2GB, 1.41GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █████▊ | 26.6GB, 1.43GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █████▊ | 26.9GB, 1.44GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █████▉ | 27.2GB, 1.45GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: █████▉ | 27.6GB, 1.46GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████ | 28.0GB, 1.47GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▏ | 28.5GB, 1.48GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▎ | 28.9GB, 1.50GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▎ | 29.2GB, 1.50GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▍ | 29.6GB, 1.52GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▍ | 29.9GB, 1.52GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▌ | 30.3GB, 1.54GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▋ | 30.6GB, 1.54GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▋ | 31.0GB, 1.56GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▊ | 31.2GB, 1.56GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▊ | 31.5GB, 1.57GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████▉ | 32.0GB, 1.58GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███████ | 32.5GB, 1.59GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▏ | 32.9GB, 1.60GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▏ | 33.3GB, 1.61GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 57%|█████▋ | 26.3GB / 46.1GB, 1.49GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▎ | 33.8GB, 1.62GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 60%|██████ | 27.9GB / 46.1GB, 1.23GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▍ | 34.1GB, 1.63GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 64%|██████▍ | 29.5GB / 46.1GB, 1.33GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▍ | 34.6GB, 1.65GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▌ | 34.9GB, 1.65GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 67%|██████▋ | 31.1GB / 46.1GB, 1.43GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▋ | 35.3GB, 1.66GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 71%|███████ | 32.7GB / 46.1GB, 1.52GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▋ | 35.8GB, 1.67GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 74%|███████▍ | 34.3GB / 46.1GB, 1.62GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▊ | 36.3GB, 1.68GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 78%|███████▊ | 35.9GB / 46.1GB, 1.72GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ███████▉ | 36.8GB, 1.69GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 81%|████████ | 37.5GB / 46.1GB, 1.81GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ████████ | 37.1GB, 1.70GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████████ | 37.4GB, 1.70GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 85%|████████▍ | 39.1GB / 46.1GB, 1.90GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ████████▏ | 38.1GB, 1.71GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████████▎ | 38.3GB, 1.73GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████████▍ | 38.7GB, 1.73GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████████▍ | 39.1GB, 1.73GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 87%|████████▋ | 40.0GB / 46.1GB, 1.99GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ████████▌ | 39.4GB, 1.74GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ████████▌ | 39.8GB, 1.76GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 90%|█████████ | 41.6GB / 46.1GB, 1.96GB/s  + ltx-2.3-22b-dev.safetensors: reconstructing file: 92%|█████████▏| 42.5GB / 46.1GB, 2.02GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ████████▋ | 40.1GB, 1.73GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 95%|█████████▍| 43.8GB / 46.1GB, 2.07GB/s  + ltx-2.3-22b-dev.safetensors: reconstructing file: 98%|█████████▊| 45.4GB / 46.1GB, 2.13GB/s  + ltx-2.3-22b-dev.safetensors: reconstructing file: 100%|██████████| 46.1GB / 46.1GB, 2.17GB/s  ltx-2.3-22b-dev.safetensors: downloading bytes: ██████████| 40.2GB, 1.68GB/s ltx-2.3-22b-dev.safetensors: downloading bytes: ██████████| 40.2GB, 1.68GB/s + ltx-2.3-22b-dev.safetensors: reconstructing file: 100%|██████████| 46.1GB / 46.1GB, 2.21GB/s +Loading text encoder + tokenizer_config.json: 0%| | 0.00/1.16M [00:00' not supported between instances of 'NoneType' and 'int' + +======================================== +Result: + - 0 completed jobs + - 1 failure +======================================== +Traceback (most recent call last): + File "/app/ai-toolkit/run.py", line 147, in + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2252, in run + self.data_loader = get_dataloader_from_datasets(self.datasets, self.train_config.batch_size, self.sd) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/data_loader.py", line 707, in get_dataloader_from_datasets + dataset = AiToolkitDataset(config, batch_size=batch_size, sd=sd) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/data_loader.py", line 396, in __init__ + self.is_video = dataset_config.num_frames > 1 or dataset_config.auto_frame_count + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +TypeError: '>' not supported between instances of 'NoneType' and 'int' + +Job process terminated with exit code 1. diff --git a/loras/output/wan22_ti2v_BE_v1/logs/1_log.txt b/loras/output/wan22_ti2v_BE_v1/logs/1_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..a3078d3bf46f1aa7909d6a2c9e6a2b6837c696d5 --- /dev/null +++ b/loras/output/wan22_ti2v_BE_v1/logs/1_log.txt @@ -0,0 +1,290 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/workspace/runpod-slim/ComfyUI/models/loras/output", + "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": "BR3AST3XPANSION", + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 32, + "linear_alpha": 32, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 200, + "max_step_saves_to_keep": 40, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/workspace/runpod-slim/datasets/wan22_ti2v_be_v1", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "BR3AST3XPANSION", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": false, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512, + 768, + 1024 + ], + "controls": [], + "shrink_video_to_frames": false, + "num_frames": 81, + "flip_x": false, + "flip_y": false, + "num_repeats": 1, + "fps": 16 + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 5000, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "linear", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": true, + "cache_text_embeddings": true, + "lr": 0.0004, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 10, + "loss_type": "mse" + }, + "logging": { + "log_every": 1, + "use_ui_logger": true + }, + "model": { + "name_or_path": "ai-toolkit/Wan2.2-I2V-A14B-Diffusers-bf16", + "quantize": false, + "qtype": "qfloat8", + "quantize_te": false, + "qtype_te": "qfloat8", + "arch": "wan22_14b_i2v", + "low_vram": false, + "model_kwargs": { + "train_high_noise": true, + "train_low_noise": true + }, + "compile": true, + "layer_offloading": false, + "layer_offloading_text_encoder_percent": 1, + "layer_offloading_transformer_percent": 1, + "block_compile": true + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 200, + "sample_start_step": 0, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "A woman bartender with wavy pink hair wearing a pink tank top standing behind the bar with one hand on a drink mixer looks at the camera before picking up the shaker and beginning to shake a cocktail drink when a BR3AST3XPANSION occurs causing her breasts to dramatically grow in weight and size almost in sync with her mixer shakes and the subsequent breasts side to side wobble, she looks at the camera unbothered by the dramatic breast size changes happening as they continue to grow and stretch her pink tank top before she looks down at her chest as her breasts reach and maintain their largest size. Dark indoor nightlife venue lighting, high detail and dramatic smooth fluid breast motion dynamics." + }, + { + "prompt": "A woman with brown hair wearing a white lace nightie thats tied in the front standing in a forest taking a selfie video looks down at her chest with a look of slight concern, suddenly a BR3AST3XPANSION occurs causing a rapid doubling of her breast size which strains her top to losing a piece, she smiles confidently and shakes her breasts side to side playfully as they continue to grow. Natural outdoor lighting and captivating smooth fluid motion." + }, + { + "prompt": "A woman with red hair sitting profile at an office cubicle desk wearing a thin black long sleeve shirt, as she types on her computer a BR3AST3XPANSION occurs causing her breasts to dramatically and significantly expand horizontally forward toward her computer, she looks down and tries to continue typing unbothered while her breasts continue to grow to their peak volume and weight. Indoor fluorescent office lighting and quick but smooth fluid movement dynamics." + }, + { + "prompt": "A woman scientist with brown hair and safety glasses leaning casually toward the camera smiling when suddenly a look of surprise comes across her face and a BR3AST3XPANSION occurs causing her breasts to dramatically balloon to a much larger size and weight evident by her breasts nearly lunging at the camera before reaching their max size and falling to the table she was leaning on. Classroom bright lighting and dramatic smooth fluid motion." + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 30, + "num_frames": 81, + "fps": 16 + } +} +Changing sample extention to animated webp +Using SQLite database at /app/ai-toolkit/aitk_db.db +Job ID: "389b8bda-5e1e-4e33-b692-5a98d569086c" + +############################################# +# Running job: wan22_ti2v_BE_v1 +############################################# + + +Running 1 process +Loading Wan model +Loading transformer 1 + Downloading bytes: | 0.00B + Reconstructing (incomplete total...): | | 0.00B / 0.00B  + + Fetching 3 files: 0%| | 0/3 [00:00 + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2481, in run + self.sample(self.step_num) + File "/app/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 408, in sample + super().sample(step, is_first) + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 378, in sample + self.sd.generate_images(gen_img_config_list, sampler=sample_config.sampler) + File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 124, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/models/base_model.py", line 702, in generate_images + img = self.generate_single_image( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/diffusion_models/wan22/wan22_14b_i2v_model.py", line 83, in generate_single_image + output = pipeline( + ^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/diffusion_models/wan22/wan22_pipeline.py", line 269, in __call__ + latents = self.scheduler.step( + ^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/diffusers/schedulers/scheduling_flow_match_euler_discrete.py", line 514, in step + prev_sample = sample + dt * model_output + ~~~~~~~^~~~~~~~~~~~~~~~~~~ +RuntimeError: The size of tensor a (36) must match the size of tensor b (16) at non-singleton dimension 1 + +Job process terminated with exit code 1. diff --git a/loras/output/wan22_ti2v_BE_v1/logs/2_log.txt b/loras/output/wan22_ti2v_BE_v1/logs/2_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..3a27599c904f23efccf3b0d2de813886eff370e2 --- /dev/null +++ b/loras/output/wan22_ti2v_BE_v1/logs/2_log.txt @@ -0,0 +1,290 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/workspace/runpod-slim/ComfyUI/models/loras/output", + "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": "BR3AST3XPANSION", + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 32, + "linear_alpha": 32, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 200, + "max_step_saves_to_keep": 40, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/workspace/runpod-slim/datasets/wan22_ti2v_be_v1", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "BR3AST3XPANSION", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": false, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512, + 768, + 1024 + ], + "controls": [], + "shrink_video_to_frames": false, + "num_frames": 81, + "flip_x": false, + "flip_y": false, + "num_repeats": 1, + "fps": 16 + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 5000, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "linear", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0004, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 10, + "loss_type": "mse" + }, + "logging": { + "log_every": 1, + "use_ui_logger": true + }, + "model": { + "name_or_path": "ai-toolkit/Wan2.2-I2V-A14B-Diffusers-bf16", + "quantize": false, + "qtype": "qfloat8", + "quantize_te": false, + "qtype_te": "qfloat8", + "arch": "wan22_14b_i2v", + "low_vram": false, + "model_kwargs": { + "train_high_noise": true, + "train_low_noise": true + }, + "compile": true, + "layer_offloading": false, + "layer_offloading_text_encoder_percent": 1, + "layer_offloading_transformer_percent": 1, + "block_compile": true + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 200, + "sample_start_step": 0, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "A woman bartender with wavy pink hair wearing a pink tank top standing behind the bar with one hand on a drink mixer looks at the camera before picking up the shaker and beginning to shake a cocktail drink when a BR3AST3XPANSION occurs causing her breasts to dramatically grow in weight and size almost in sync with her mixer shakes and the subsequent breasts side to side wobble, she looks at the camera unbothered by the dramatic breast size changes happening as they continue to grow and stretch her pink tank top before she looks down at her chest as her breasts reach and maintain their largest size. Dark indoor nightlife venue lighting, high detail and dramatic smooth fluid breast motion dynamics." + }, + { + "prompt": "A woman with brown hair wearing a white lace nightie thats tied in the front standing in a forest taking a selfie video looks down at her chest with a look of slight concern, suddenly a BR3AST3XPANSION occurs causing a rapid doubling of her breast size which strains her top to losing a piece, she smiles confidently and shakes her breasts side to side playfully as they continue to grow. Natural outdoor lighting and captivating smooth fluid motion." + }, + { + "prompt": "A woman with red hair sitting profile at an office cubicle desk wearing a thin black long sleeve shirt, as she types on her computer a BR3AST3XPANSION occurs causing her breasts to dramatically and significantly expand horizontally forward toward her computer, she looks down and tries to continue typing unbothered while her breasts continue to grow to their peak volume and weight. Indoor fluorescent office lighting and quick but smooth fluid movement dynamics." + }, + { + "prompt": "A woman scientist with brown hair and safety glasses leaning casually toward the camera smiling when suddenly a look of surprise comes across her face and a BR3AST3XPANSION occurs causing her breasts to dramatically balloon to a much larger size and weight evident by her breasts nearly lunging at the camera before reaching their max size and falling to the table she was leaning on. Classroom bright lighting and dramatic smooth fluid motion." + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 30, + "num_frames": 81, + "fps": 16 + } +} +Changing sample extention to animated webp +Using SQLite database at /app/ai-toolkit/aitk_db.db +Job ID: "389b8bda-5e1e-4e33-b692-5a98d569086c" + +############################################# +# Running job: wan22_ti2v_BE_v1 +############################################# + + +Running 1 process +Loading Wan model +Loading transformer 1 + Downloading bytes: | 0.00B + Reconstructing (incomplete total...): | | 0.00B / 0.00B  + + Fetching 3 files: 0%| | 0/3 [00:00 + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2481, in run + self.sample(self.step_num) + File "/app/ai-toolkit/extensions_built_in/sd_trainer/DiffusionTrainer.py", line 408, in sample + super().sample(step, is_first) + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 378, in sample + self.sd.generate_images(gen_img_config_list, sampler=sample_config.sampler) + File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 124, in decorate_context + return func(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/models/base_model.py", line 702, in generate_images + img = self.generate_single_image( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/diffusion_models/wan22/wan22_14b_i2v_model.py", line 83, in generate_single_image + output = pipeline( + ^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/diffusion_models/wan22/wan22_pipeline.py", line 269, in __call__ + latents = self.scheduler.step( + ^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/diffusers/schedulers/scheduling_flow_match_euler_discrete.py", line 514, in step + prev_sample = sample + dt * model_output + ~~~~~~~^~~~~~~~~~~~~~~~~~~ +RuntimeError: The size of tensor a (36) must match the size of tensor b (16) at non-singleton dimension 1 + +Job process terminated with exit code 1. diff --git a/loras/output/wan22_ti2v_BE_v1/logs/3_log.txt b/loras/output/wan22_ti2v_BE_v1/logs/3_log.txt new file mode 100644 index 0000000000000000000000000000000000000000..968d1c26aac8fa79409dda4f855792f52eb714d9 --- /dev/null +++ b/loras/output/wan22_ti2v_BE_v1/logs/3_log.txt @@ -0,0 +1,302 @@ +Running 1 job +{ + "type": "diffusion_trainer", + "training_folder": "/workspace/runpod-slim/ComfyUI/models/loras/output", + "sqlite_db_path": "/app/ai-toolkit/aitk_db.db", + "device": "cuda", + "trigger_word": "BR3AST3XPANSION", + "performance_log_every": 10, + "network": { + "type": "lora", + "linear": 32, + "linear_alpha": 32, + "conv": 16, + "conv_alpha": 16, + "lokr_full_rank": true, + "lokr_factor": -1, + "network_kwargs": { + "ignore_if_contains": [] + } + }, + "save": { + "dtype": "bf16", + "save_every": 200, + "max_step_saves_to_keep": 40, + "save_format": "diffusers", + "push_to_hub": false + }, + "datasets": [ + { + "folder_path": "/workspace/runpod-slim/datasets/wan22_ti2v_be_v1", + "mask_path": null, + "mask_min_value": 0.1, + "default_caption": "BR3AST3XPANSION", + "caption_ext": "txt", + "caption_dropout_rate": 0.05, + "cache_latents_to_disk": false, + "is_reg": false, + "network_weight": 1, + "resolution": [ + 512, + 768, + 1024 + ], + "controls": [], + "shrink_video_to_frames": false, + "num_frames": 81, + "flip_x": false, + "flip_y": false, + "num_repeats": 1, + "fps": 16 + } + ], + "train": { + "batch_size": 1, + "bypass_guidance_embedding": false, + "steps": 5000, + "gradient_accumulation": 1, + "train_unet": true, + "train_text_encoder": false, + "gradient_checkpointing": true, + "noise_scheduler": "flowmatch", + "optimizer": "adamw8bit", + "timestep_type": "linear", + "content_or_style": "balanced", + "optimizer_params": { + "weight_decay": 0.0001 + }, + "unload_text_encoder": false, + "cache_text_embeddings": true, + "lr": 0.0004, + "ema_config": { + "use_ema": false, + "ema_decay": 0.99 + }, + "skip_first_sample": false, + "force_first_sample": false, + "disable_sampling": false, + "dtype": "bf16", + "diff_output_preservation": false, + "diff_output_preservation_multiplier": 1, + "diff_output_preservation_class": "person", + "switch_boundary_every": 10, + "loss_type": "mse" + }, + "logging": { + "log_every": 1, + "use_ui_logger": true + }, + "model": { + "name_or_path": "ai-toolkit/Wan2.2-I2V-A14B-Diffusers-bf16", + "quantize": false, + "qtype": "qfloat8", + "quantize_te": false, + "qtype_te": "qfloat8", + "arch": "wan22_14b_i2v", + "low_vram": false, + "model_kwargs": { + "train_high_noise": true, + "train_low_noise": true + }, + "compile": true, + "layer_offloading": false, + "layer_offloading_text_encoder_percent": 1, + "layer_offloading_transformer_percent": 1, + "block_compile": true + }, + "sample": { + "sampler": "flowmatch", + "sample_every": 400, + "sample_start_step": 0, + "width": 512, + "height": 512, + "samples": [ + { + "prompt": "A woman bartender with wavy pink hair wearing a pink tank top standing behind the bar with one hand on a drink mixer looks at the camera before picking up the shaker and beginning to shake a cocktail drink when a BR3AST3XPANSION occurs causing her breasts to dramatically grow in weight and size almost in sync with her mixer shakes and the subsequent breasts side to side wobble, she looks at the camera unbothered by the dramatic breast size changes happening as they continue to grow and stretch her pink tank top before she looks down at her chest as her breasts reach and maintain their largest size. Dark indoor nightlife venue lighting, high detail and dramatic smooth fluid breast motion dynamics.", + "ctrl_img": "/app/ai-toolkit/data/images/a027c383-cde8-4d29-9367-261022eccd7b.png" + }, + { + "prompt": "A woman with brown hair wearing a white lace nightie thats tied in the front standing in a forest taking a selfie video looks down at her chest with a look of slight concern, suddenly a BR3AST3XPANSION occurs causing a rapid doubling of her breast size which strains her top to losing a piece, she smiles confidently and shakes her breasts side to side playfully as they continue to grow. Natural outdoor lighting and captivating smooth fluid motion.", + "ctrl_img": "/app/ai-toolkit/data/images/1e9a4f88-bc13-43b5-b804-6e58f577e1b5.png" + }, + { + "prompt": "A woman with red hair sitting profile at an office cubicle desk wearing a thin black long sleeve shirt, as she types on her computer a BR3AST3XPANSION occurs causing her breasts to dramatically and significantly expand horizontally forward toward her computer, she looks down and tries to continue typing unbothered while her breasts continue to grow to their peak volume and weight. Indoor fluorescent office lighting and quick but smooth fluid movement dynamics.", + "ctrl_img": "/app/ai-toolkit/data/images/5354c6aa-6340-45e0-b3ea-ef2fae27acd1.png" + }, + { + "prompt": "A woman scientist with brown hair and safety glasses leaning casually toward the camera smiling when suddenly a look of surprise comes across her face and a BR3AST3XPANSION occurs causing her breasts to dramatically balloon to a much larger size and weight evident by her breasts nearly lunging at the camera before reaching their max size and falling to the table she was leaning on. Classroom bright lighting and dramatic smooth fluid motion.", + "ctrl_img": "/app/ai-toolkit/data/images/7f24a43b-ff74-4144-a1d8-130f13c29ff5.png" + } + ], + "neg": "", + "seed": 42, + "walk_seed": true, + "guidance_scale": 4, + "sample_steps": 30, + "num_frames": 81, + "fps": 16 + } +} +Changing sample extention to animated webp +Using SQLite database at /app/ai-toolkit/aitk_db.db +Job ID: "389b8bda-5e1e-4e33-b692-5a98d569086c" + +############################################# +# Running job: wan22_ti2v_BE_v1 +############################################# + + +Running 1 process +Loading Wan model +Loading transformer 1 + Downloading bytes: | 0.00B + Reconstructing (incomplete total...): | | 0.00B / 0.00B  + + Fetching 3 files: 0%| | 0/3 [00:00 + main() + File "/app/ai-toolkit/run.py", line 132, in main + raise e + File "/app/ai-toolkit/run.py", line 120, in main + job.run() + File "/app/ai-toolkit/jobs/ExtensionJob.py", line 22, in run + process.run() + File "/app/ai-toolkit/jobs/process/BaseSDTrainProcess.py", line 2633, in run + loss_dict = self.hook_train_loop(batch_list) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2263, in hook_train_loop + loss = self.train_single_accumulation(batch) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 2152, in train_single_accumulation + noise_pred = self.predict_noise( + ^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/sd_trainer/SDTrainer.py", line 1364, in predict_noise + return self.sd.predict_noise( + ^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/toolkit/models/base_model.py", line 974, in predict_noise + noise_pred = self.get_noise_prediction( + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/app/ai-toolkit/extensions_built_in/diffusion_models/wan22/wan22_14b_i2v_model.py", line 123, in get_noise_prediction + if len(frames.shape) == 4: + ^^^^^^^^^^^^ +AttributeError: 'NoneType' object has no attribute 'shape' + wan22_ti2v_BE_v1: 0%| | 0/3000 [00:00