diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/dataset.json b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/dataset.json new file mode 100755 index 0000000000000000000000000000000000000000..9608c46ac273366c44471cc138453d4196bb8eaa --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/dataset.json @@ -0,0 +1,17 @@ +{ + "channel_names": { + "0": "CT" + }, + "labels": { + "background": 0, + "Bladder": 1, + "Anorectum": 2, + "Bag_Bowel": 3, + "Femur_Head_L": 4, + "Femur_Head_R": 5, + "Penilebulb": 6 + }, + "numTraining": 60, + "file_ending": ".nii.gz", + "numTest": 0 +} \ No newline at end of file diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/dataset_fingerprint.json b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/dataset_fingerprint.json new file mode 100755 index 0000000000000000000000000000000000000000..fd7943e498aa4df6950c7812c08c480b7f93806c --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/dataset_fingerprint.json @@ -0,0 +1,618 @@ +{ + "foreground_intensity_properties_per_channel": { + "0": { + "max": 1620.0, + "mean": -38.229164123535156, + "median": -54.0, + "min": -1000.0, + "percentile_00_5": -941.0, + "percentile_99_5": 897.0, + "std": 192.37086486816406 + } + }, + "median_relative_size_after_cropping": 1.0, + "shapes_after_crop": [ + [ + 230, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 260, + 512, + 512 + ], + [ + 215, + 512, + 512 + ], + [ + 260, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 265, + 512, + 512 + ], + [ + 229, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 243, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 245, + 512, + 512 + ], + [ + 235, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 242, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 255, + 512, + 512 + ], + [ + 246, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 245, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 249, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 244, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 235, + 512, + 512 + ], + [ + 260, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 190, + 512, + 512 + ], + [ + 255, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 255, + 512, + 512 + ], + [ + 236, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 245, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 266, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 280, + 512, + 512 + ], + [ + 260, + 512, + 512 + ], + [ + 245, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 226, + 512, + 512 + ], + [ + 240, + 512, + 512 + ] + ], + "spacings": [ + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ] + ] +} \ No newline at end of file diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/checkpoint_best.pth b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/checkpoint_best.pth new file mode 100755 index 0000000000000000000000000000000000000000..83b1b8369e5c88f9bcf4bbce6d19e89affd4de93 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/checkpoint_best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e27d5b54b180e4e0e2f27bff5ed8d071cf2d431d4861a3b1fcee4f2f76ba0da8 +size 246768929 diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/checkpoint_latest.pth b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/checkpoint_latest.pth new file mode 100755 index 0000000000000000000000000000000000000000..95217b8bece6544b0fe147ab3efb93e6594a9997 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/checkpoint_latest.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:15ec58f8a56fa17e53739749487a8cbf54fb0282167c5e1c9bf5c0c89082e96a +size 246952469 diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/debug.json b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/debug.json new file mode 100755 index 0000000000000000000000000000000000000000..ba05b6bace861cbb4678c5909249f934ab5bf810 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/debug.json @@ -0,0 +1,52 @@ +{ + "_best_ema": "0.86605509930085", + "batch_size": "2", + "configuration_manager": "{'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [241.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True}", + "configuration_name": "3d_fullres", + "cudnn_version": 8500, + "current_epoch": "650", + "dataloader_train": "", + "dataloader_train.generator": "", + "dataloader_train.num_processes": "12", + "dataloader_train.transform": "Compose ( [SpatialTransform( independent_scale_for_each_axis = False, p_rot_per_sample = 0.2, p_scale_per_sample = 0.2, p_el_per_sample = 0, data_key = 'data', label_key = 'seg', patch_size = [80, 192, 160], patch_center_dist_from_border = None, do_elastic_deform = False, alpha = (0, 0), sigma = (0, 0), do_rotation = True, angle_x = (-0.5235987755982988, 0.5235987755982988), angle_y = (-0.5235987755982988, 0.5235987755982988), angle_z = (-0.5235987755982988, 0.5235987755982988), do_scale = True, scale = (0.7, 1.4), border_mode_data = 'constant', border_cval_data = 0, order_data = 3, border_mode_seg = 'constant', border_cval_seg = -1, order_seg = 1, random_crop = False, p_rot_per_axis = 1, p_independent_scale_per_axis = 1 ), GaussianNoiseTransform( p_per_sample = 0.1, data_key = 'data', noise_variance = (0, 0.1), p_per_channel = 1, per_channel = False ), GaussianBlurTransform( p_per_sample = 0.2, different_sigma_per_channel = True, p_per_channel = 0.5, data_key = 'data', blur_sigma = (0.5, 1.0), different_sigma_per_axis = False, p_isotropic = 0 ), BrightnessMultiplicativeTransform( p_per_sample = 0.15, data_key = 'data', multiplier_range = (0.75, 1.25), per_channel = True ), ContrastAugmentationTransform( p_per_sample = 0.15, data_key = 'data', contrast_range = (0.75, 1.25), preserve_range = True, per_channel = True, p_per_channel = 1 ), SimulateLowResolutionTransform( order_upsample = 3, order_downsample = 0, channels = None, per_channel = True, p_per_channel = 0.5, p_per_sample = 0.25, data_key = 'data', zoom_range = (0.5, 1), ignore_axes = None ), GammaTransform( p_per_sample = 0.1, retain_stats = True, per_channel = True, data_key = 'data', gamma_range = (0.7, 1.5), invert_image = True ), GammaTransform( p_per_sample = 0.3, retain_stats = True, per_channel = True, data_key = 'data', gamma_range = (0.7, 1.5), invert_image = False ), RemoveLabelTransform( output_key = 'seg', input_key = 'seg', replace_with = 0, remove_label = -1 ), RenameTransform( delete_old = True, out_key = 'target', in_key = 'seg' ), DownsampleSegForDSTransform2( axes = None, output_key = 'target', input_key = 'target', order = 0, ds_scales = [[1.0, 1.0, 1.0], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]] ), NumpyToTensor( keys = ['data', 'target'], cast_to = 'float' )] )", + "dataloader_val": "", + "dataloader_val.generator": "", + "dataloader_val.num_processes": "6", + "dataloader_val.transform": "Compose ( [RemoveLabelTransform( output_key = 'seg', input_key = 'seg', replace_with = 0, remove_label = -1 ), RenameTransform( delete_old = True, out_key = 'target', in_key = 'seg' ), DownsampleSegForDSTransform2( axes = None, output_key = 'target', input_key = 'target', order = 0, ds_scales = [[1.0, 1.0, 1.0], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]] ), NumpyToTensor( keys = ['data', 'target'], cast_to = 'float' )] )", + "dataset_json": "{'channel_names': {'0': 'CT'}, 'labels': {'background': 0, 'Bladder': 1, 'Anorectum': 2, 'Bag_Bowel': 3, 'Femur_Head_L': 4, 'Femur_Head_R': 5, 'Penilebulb': 6}, 'numTraining': 60, 'file_ending': '.nii.gz', 'numTest': 0}", + "device": "cuda:0", + "disable_checkpointing": "False", + "fold": "0", + "folder_with_segs_from_previous_stage": "None", + "gpu_name": "NVIDIA GeForce GTX 1080 Ti", + "grad_scaler": "", + "hostname": "vipadmin-Z10PE-D16-WS", + "inference_allowed_mirroring_axes": "None", + "initial_lr": "0.01", + "is_cascaded": "False", + "is_ddp": "False", + "label_manager": "", + "local_rank": "0", + "log_file": "./data/nnUNet_results/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_16_11_52_25.txt", + "logger": "", + "loss": "DeepSupervisionWrapper(\n (loss): DC_and_CE_loss(\n (ce): RobustCrossEntropyLoss()\n (dc): MemoryEfficientSoftDiceLoss()\n )\n)", + "lr_scheduler": "", + "my_init_kwargs": "{'plans': {'dataset_name': 'Dataset720_TSPrime', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [241, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'configurations': {'2d': {'data_identifier': 'nnUNetPlans_2d', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 12, 'patch_size': [512, 512], 'median_image_size_in_voxels': [512.0, 512.0], 'spacing': [1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2, 2, 2], 'num_pool_per_axis': [7, 7], 'pool_op_kernel_sizes': [[1, 1], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2]], 'conv_kernel_sizes': [[3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3]], 'unet_max_num_features': 512, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True}, '3d_lowres': {'data_identifier': 'nnUNetPlans_3d_lowres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [130, 275, 275], 'spacing': [4.650736429273743, 2.361701649461784, 2.361701649461784], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': False, 'next_stage': '3d_cascade_fullres'}, '3d_fullres': {'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [241.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True}, '3d_cascade_fullres': {'inherits_from': '3d_fullres', 'previous_stage': '3d_lowres'}}, 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 1620.0, 'mean': -38.229164123535156, 'median': -54.0, 'min': -1000.0, 'percentile_00_5': -941.0, 'percentile_99_5': 897.0, 'std': 192.37086486816406}}}, 'configuration': '3d_fullres', 'fold': 0, 'dataset_json': {'channel_names': {'0': 'CT'}, 'labels': {'background': 0, 'Bladder': 1, 'Anorectum': 2, 'Bag_Bowel': 3, 'Femur_Head_L': 4, 'Femur_Head_R': 5, 'Penilebulb': 6}, 'numTraining': 60, 'file_ending': '.nii.gz', 'numTest': 0}, 'unpack_dataset': True, 'device': device(type='cuda')}", + "network": "PlainConvUNet", + "num_epochs": "1000", + "num_input_channels": "1", + "num_iterations_per_epoch": "250", + "num_val_iterations_per_epoch": "50", + "optimizer": "SGD (\nParameter Group 0\n dampening: 0\n differentiable: False\n foreach: None\n initial_lr: 0.01\n lr: 0.003897412779133726\n maximize: False\n momentum: 0.99\n nesterov: True\n weight_decay: 3e-05\n)", + "output_folder": "./data/nnUNet_results/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0", + "output_folder_base": "./data/nnUNet_results/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres", + "oversample_foreground_percent": "0.33", + "plans_manager": "{'dataset_name': 'Dataset720_TSPrime', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [241, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'configurations': {'2d': {'data_identifier': 'nnUNetPlans_2d', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 12, 'patch_size': [512, 512], 'median_image_size_in_voxels': [512.0, 512.0], 'spacing': [1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2, 2, 2], 'num_pool_per_axis': [7, 7], 'pool_op_kernel_sizes': [[1, 1], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2]], 'conv_kernel_sizes': [[3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3]], 'unet_max_num_features': 512, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True}, '3d_lowres': {'data_identifier': 'nnUNetPlans_3d_lowres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [130, 275, 275], 'spacing': [4.650736429273743, 2.361701649461784, 2.361701649461784], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': False, 'next_stage': '3d_cascade_fullres'}, '3d_fullres': {'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [241.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True}, '3d_cascade_fullres': {'inherits_from': '3d_fullres', 'previous_stage': '3d_lowres'}}, 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 1620.0, 'mean': -38.229164123535156, 'median': -54.0, 'min': -1000.0, 'percentile_00_5': -941.0, 'percentile_99_5': 897.0, 'std': 192.37086486816406}}}", + "preprocessed_dataset_folder": "./data/nnUNet_preprocessed/Dataset720_TSPrime/nnUNetPlans_3d_fullres", + "preprocessed_dataset_folder_base": "./data/nnUNet_preprocessed/Dataset720_TSPrime", + "save_every": "50", + "torch_version": "2.0.1+cu117", + "unpack_dataset": "True", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/network_architecture b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/network_architecture new file mode 100755 index 0000000000000000000000000000000000000000..e8dd9af306f1958d25af867d5c2090c7ffb8a8b1 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/network_architecture @@ -0,0 +1,171 @@ +digraph { + graph [bgcolor="#FFFFFF" color="#000000" fontcolor="#000000" fontname=Times fontsize=10 margin="0,0" pad="1.0,0.5" rankdir=LR] + node [color="#000000" fillcolor="#E8E8E8" fontcolor="#000000" fontname=Times fontsize=10 margin="0,0" shape=box style=filled] + edge [color="#000000" fontcolor="#000000" fontname=Times fontsize=10 style=solid] + "/outputs/109" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/110" [label=<
InstanceNormalization
>] + "/outputs/111" [label=<
LeakyRelu
>] + "/outputs/112" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/113" [label=<
InstanceNormalization
>] + "/outputs/114" [label=<
LeakyRelu
>] + "/outputs/115" [label=<
Conv, kernel_size: [3, 3, 3], stride: [2, 2, 2]
>] + "/outputs/116" [label=<
InstanceNormalization
>] + "/outputs/117" [label=<
LeakyRelu
>] + "/outputs/118" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/119" [label=<
InstanceNormalization
>] + "/outputs/120" [label=<
LeakyRelu
>] + "/outputs/121" [label=<
Conv, kernel_size: [3, 3, 3], stride: [2, 2, 2]
>] + "/outputs/122" [label=<
InstanceNormalization
>] + "/outputs/123" [label=<
LeakyRelu
>] + "/outputs/124" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/125" [label=<
InstanceNormalization
>] + "/outputs/126" [label=<
LeakyRelu
>] + "/outputs/127" [label=<
Conv, kernel_size: [3, 3, 3], stride: [2, 2, 2]
>] + "/outputs/128" [label=<
InstanceNormalization
>] + "/outputs/129" [label=<
LeakyRelu
>] + "/outputs/130" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/131" [label=<
InstanceNormalization
>] + "/outputs/132" [label=<
LeakyRelu
>] + "/outputs/133" [label=<
Conv, kernel_size: [3, 3, 3], stride: [2, 2, 2]
>] + "/outputs/134" [label=<
InstanceNormalization
>] + "/outputs/135" [label=<
LeakyRelu
>] + "/outputs/136" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/137" [label=<
InstanceNormalization
>] + "/outputs/138" [label=<
LeakyRelu
>] + "/outputs/139" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 2, 2]
>] + "/outputs/140" [label=<
InstanceNormalization
>] + "/outputs/141" [label=<
LeakyRelu
>] + "/outputs/142" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/143" [label=<
InstanceNormalization
>] + "/outputs/144" [label=<
LeakyRelu
>] + "/outputs/145" [label=<
ConvTranspose, kernel_size: [1, 2, 2], stride: [1, 2, 2]
>] + "/outputs/146" [label=<
Concat
>] + "/outputs/147" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/148" [label=<
InstanceNormalization
>] + "/outputs/149" [label=<
LeakyRelu
>] + "/outputs/150" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/151" [label=<
InstanceNormalization
>] + "/outputs/152" [label=<
LeakyRelu
>] + "/outputs/153" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/154" [label=<
ConvTranspose, kernel_size: [2, 2, 2], stride: [2, 2, 2]
>] + "/outputs/155" [label=<
Concat
>] + "/outputs/156" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/157" [label=<
InstanceNormalization
>] + "/outputs/158" [label=<
LeakyRelu
>] + "/outputs/159" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/160" [label=<
InstanceNormalization
>] + "/outputs/161" [label=<
LeakyRelu
>] + "/outputs/162" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/163" [label=<
ConvTranspose, kernel_size: [2, 2, 2], stride: [2, 2, 2]
>] + "/outputs/164" [label=<
Concat
>] + "/outputs/165" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/166" [label=<
InstanceNormalization
>] + "/outputs/167" [label=<
LeakyRelu
>] + "/outputs/168" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/169" [label=<
InstanceNormalization
>] + "/outputs/170" [label=<
LeakyRelu
>] + "/outputs/171" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/172" [label=<
ConvTranspose, kernel_size: [2, 2, 2], stride: [2, 2, 2]
>] + "/outputs/173" [label=<
Concat
>] + "/outputs/174" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/175" [label=<
InstanceNormalization
>] + "/outputs/176" [label=<
LeakyRelu
>] + "/outputs/177" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/178" [label=<
InstanceNormalization
>] + "/outputs/179" [label=<
LeakyRelu
>] + "/outputs/180" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/181" [label=<
ConvTranspose, kernel_size: [2, 2, 2], stride: [2, 2, 2]
>] + "/outputs/182" [label=<
Concat
>] + "/outputs/183" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/184" [label=<
InstanceNormalization
>] + "/outputs/185" [label=<
LeakyRelu
>] + "/outputs/186" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/187" [label=<
InstanceNormalization
>] + "/outputs/188" [label=<
LeakyRelu
>] + "/outputs/189" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/109" -> "/outputs/110" [label="1x32x80x192x160"] + "/outputs/110" -> "/outputs/111" [label="1x32x80x192x160"] + "/outputs/111" -> "/outputs/112" [label="1x32x80x192x160"] + "/outputs/112" -> "/outputs/113" [label="1x32x80x192x160"] + "/outputs/113" -> "/outputs/114" [label="1x32x80x192x160"] + "/outputs/114" -> "/outputs/115" [label="1x32x80x192x160"] + "/outputs/114" -> "/outputs/182" [label="1x32x80x192x160"] + "/outputs/115" -> "/outputs/116" [label="1x64x40x96x80"] + "/outputs/116" -> "/outputs/117" [label="1x64x40x96x80"] + "/outputs/117" -> "/outputs/118" [label="1x64x40x96x80"] + "/outputs/118" -> "/outputs/119" [label="1x64x40x96x80"] + "/outputs/119" -> "/outputs/120" [label="1x64x40x96x80"] + "/outputs/120" -> "/outputs/121" [label="1x64x40x96x80"] + "/outputs/120" -> "/outputs/173" [label="1x64x40x96x80"] + "/outputs/121" -> "/outputs/122" [label="1x128x20x48x40"] + "/outputs/122" -> "/outputs/123" [label="1x128x20x48x40"] + "/outputs/123" -> "/outputs/124" [label="1x128x20x48x40"] + "/outputs/124" -> "/outputs/125" [label="1x128x20x48x40"] + "/outputs/125" -> "/outputs/126" [label="1x128x20x48x40"] + "/outputs/126" -> "/outputs/127" [label="1x128x20x48x40"] + "/outputs/126" -> "/outputs/164" [label="1x128x20x48x40"] + "/outputs/127" -> "/outputs/128" [label="1x256x10x24x20"] + "/outputs/128" -> "/outputs/129" [label="1x256x10x24x20"] + "/outputs/129" -> "/outputs/130" [label="1x256x10x24x20"] + "/outputs/130" -> "/outputs/131" [label="1x256x10x24x20"] + "/outputs/131" -> "/outputs/132" [label="1x256x10x24x20"] + "/outputs/132" -> "/outputs/133" [label="1x256x10x24x20"] + "/outputs/132" -> "/outputs/155" [label="1x256x10x24x20"] + "/outputs/133" -> "/outputs/134" [label="1x320x5x12x10"] + "/outputs/134" -> "/outputs/135" [label="1x320x5x12x10"] + "/outputs/135" -> "/outputs/136" [label="1x320x5x12x10"] + "/outputs/136" -> "/outputs/137" [label="1x320x5x12x10"] + "/outputs/137" -> "/outputs/138" [label="1x320x5x12x10"] + "/outputs/138" -> "/outputs/139" [label="1x320x5x12x10"] + "/outputs/138" -> "/outputs/146" [label="1x320x5x12x10"] + "/outputs/139" -> "/outputs/140" [label="1x320x5x6x5"] + "/outputs/140" -> "/outputs/141" [label="1x320x5x6x5"] + "/outputs/141" -> "/outputs/142" [label="1x320x5x6x5"] + "/outputs/142" -> "/outputs/143" [label="1x320x5x6x5"] + "/outputs/143" -> "/outputs/144" [label="1x320x5x6x5"] + "/outputs/144" -> "/outputs/145" [label="1x320x5x6x5"] + "/outputs/145" -> "/outputs/146" [label="1x320x5x12x10"] + "/outputs/146" -> "/outputs/147" [label="1x640x5x12x10"] + "/outputs/147" -> "/outputs/148" [label="1x320x5x12x10"] + "/outputs/148" -> "/outputs/149" [label="1x320x5x12x10"] + "/outputs/149" -> "/outputs/150" [label="1x320x5x12x10"] + "/outputs/150" -> "/outputs/151" [label="1x320x5x12x10"] + "/outputs/151" -> "/outputs/152" [label="1x320x5x12x10"] + "/outputs/152" -> "/outputs/153" [label="1x320x5x12x10"] + "/outputs/152" -> "/outputs/154" [label="1x320x5x12x10"] + "/outputs/154" -> "/outputs/155" [label="1x256x10x24x20"] + "/outputs/155" -> "/outputs/156" [label="1x512x10x24x20"] + "/outputs/156" -> "/outputs/157" [label="1x256x10x24x20"] + "/outputs/157" -> "/outputs/158" [label="1x256x10x24x20"] + "/outputs/158" -> "/outputs/159" [label="1x256x10x24x20"] + "/outputs/159" -> "/outputs/160" [label="1x256x10x24x20"] + "/outputs/160" -> "/outputs/161" [label="1x256x10x24x20"] + "/outputs/161" -> "/outputs/162" [label="1x256x10x24x20"] + "/outputs/161" -> "/outputs/163" [label="1x256x10x24x20"] + "/outputs/163" -> "/outputs/164" [label="1x128x20x48x40"] + "/outputs/164" -> "/outputs/165" [label="1x256x20x48x40"] + "/outputs/165" -> "/outputs/166" [label="1x128x20x48x40"] + "/outputs/166" -> "/outputs/167" [label="1x128x20x48x40"] + "/outputs/167" -> "/outputs/168" [label="1x128x20x48x40"] + "/outputs/168" -> "/outputs/169" [label="1x128x20x48x40"] + "/outputs/169" -> "/outputs/170" [label="1x128x20x48x40"] + "/outputs/170" -> "/outputs/171" [label="1x128x20x48x40"] + "/outputs/170" -> "/outputs/172" [label="1x128x20x48x40"] + "/outputs/172" -> "/outputs/173" [label="1x64x40x96x80"] + "/outputs/173" -> "/outputs/174" [label="1x128x40x96x80"] + "/outputs/174" -> "/outputs/175" [label="1x64x40x96x80"] + "/outputs/175" -> "/outputs/176" [label="1x64x40x96x80"] + "/outputs/176" -> "/outputs/177" [label="1x64x40x96x80"] + "/outputs/177" -> "/outputs/178" [label="1x64x40x96x80"] + "/outputs/178" -> "/outputs/179" [label="1x64x40x96x80"] + "/outputs/179" -> "/outputs/180" [label="1x64x40x96x80"] + "/outputs/179" -> "/outputs/181" [label="1x64x40x96x80"] + "/outputs/181" -> "/outputs/182" [label="1x32x80x192x160"] + "/outputs/182" -> "/outputs/183" [label="1x64x80x192x160"] + "/outputs/183" -> "/outputs/184" [label="1x32x80x192x160"] + "/outputs/184" -> "/outputs/185" [label="1x32x80x192x160"] + "/outputs/185" -> "/outputs/186" [label="1x32x80x192x160"] + "/outputs/186" -> "/outputs/187" [label="1x32x80x192x160"] + "/outputs/187" -> "/outputs/188" [label="1x32x80x192x160"] + "/outputs/188" -> "/outputs/189" [label="1x32x80x192x160"] +} diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/progress.png b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/progress.png new file mode 100755 index 0000000000000000000000000000000000000000..72ff3af06abf144ff647206337938a701e8f1064 Binary files /dev/null and b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/progress.png differ diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_11_21_50_08.txt b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_11_21_50_08.txt new file mode 100755 index 0000000000000000000000000000000000000000..1baedd9f470212c0fdb8a85ac345093173d4efa4 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_11_21_50_08.txt @@ -0,0 +1,26 @@ + +####################################################################### +Please cite the following paper when using nnU-Net: +Isensee, F., Jaeger, P. F., Kohl, S. A., Petersen, J., & Maier-Hein, K. H. (2021). nnU-Net: a self-configuring method for deep learning-based biomedical image segmentation. Nature methods, 18(2), 203-211. +####################################################################### + + +This is the configuration used by this training: +Configuration name: 3d_fullres + {'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [240.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True} + +These are the global plan.json settings: + {'dataset_name': 'Dataset720_TSPrime', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [240, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 3071.0, 'mean': -12.232562065124512, 'median': -21.0, 'min': -1000.0, 'percentile_00_5': -793.0, 'percentile_99_5': 974.0, 'std': 168.24203491210938}}} + +2023-10-11 21:50:09.984604: unpacking dataset... +2023-10-11 21:51:22.972373: unpacking done... +2023-10-11 21:51:23.374131: do_dummy_2d_data_aug: False +2023-10-11 21:51:23.375596: Using splits from existing split file: ./data/nnUNet_preprocessed/Dataset720_TSPrime/splits_final.json +2023-10-11 21:51:23.541955: The split file contains 5 splits. +2023-10-11 21:51:23.542130: Desired fold for training: 0 +2023-10-11 21:51:23.542256: This split has 20 training and 5 validation cases. +2023-10-11 21:52:06.109724: Unable to plot network architecture: +2023-10-11 21:52:06.109830: failed to execute PosixPath('dot'), make sure the Graphviz executables are on your systems' PATH +2023-10-11 21:52:06.212852: +2023-10-11 21:52:06.212922: Epoch 0 +2023-10-11 21:52:06.213036: Current learning rate: 0.01 diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_11_21_56_01.txt b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_11_21_56_01.txt new file mode 100755 index 0000000000000000000000000000000000000000..40094a84e8ad78a2bef1f3a162307eec3744fc7b --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_11_21_56_01.txt @@ -0,0 +1,26 @@ + +####################################################################### +Please cite the following paper when using nnU-Net: +Isensee, F., Jaeger, P. F., Kohl, S. A., Petersen, J., & Maier-Hein, K. H. (2021). nnU-Net: a self-configuring method for deep learning-based biomedical image segmentation. Nature methods, 18(2), 203-211. +####################################################################### + + +This is the configuration used by this training: +Configuration name: 3d_fullres + {'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 3, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [240.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True} + +These are the global plan.json settings: + {'dataset_name': 'Dataset720_TSPrime', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [240, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 3071.0, 'mean': -12.232562065124512, 'median': -21.0, 'min': -1000.0, 'percentile_00_5': -793.0, 'percentile_99_5': 974.0, 'std': 168.24203491210938}}} + +2023-10-11 21:56:02.926755: unpacking dataset... +2023-10-11 21:56:06.110042: unpacking done... +2023-10-11 21:56:06.110639: do_dummy_2d_data_aug: False +2023-10-11 21:56:06.111117: Using splits from existing split file: ./data/nnUNet_preprocessed/Dataset720_TSPrime/splits_final.json +2023-10-11 21:56:06.111239: The split file contains 5 splits. +2023-10-11 21:56:06.111284: Desired fold for training: 0 +2023-10-11 21:56:06.111325: This split has 20 training and 5 validation cases. +2023-10-11 21:56:27.774188: Unable to plot network architecture: +2023-10-11 21:56:27.774289: failed to execute PosixPath('dot'), make sure the Graphviz executables are on your systems' PATH +2023-10-11 21:56:27.879786: +2023-10-11 21:56:27.879845: Epoch 0 +2023-10-11 21:56:27.879956: Current learning rate: 0.01 diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_11_22_29_34.txt b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_11_22_29_34.txt new file mode 100755 index 0000000000000000000000000000000000000000..ddb4cd68c5b52ba6575cbafcb387c62717567e49 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_11_22_29_34.txt @@ -0,0 +1,4247 @@ + +####################################################################### +Please cite the following paper when using nnU-Net: +Isensee, F., Jaeger, P. F., Kohl, S. A., Petersen, J., & Maier-Hein, K. H. (2021). nnU-Net: a self-configuring method for deep learning-based biomedical image segmentation. Nature methods, 18(2), 203-211. +####################################################################### + + +This is the configuration used by this training: +Configuration name: 3d_fullres + {'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [241.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True} + +These are the global plan.json settings: + {'dataset_name': 'Dataset720_TSPrime', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [241, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 1620.0, 'mean': -38.229164123535156, 'median': -54.0, 'min': -1000.0, 'percentile_00_5': -941.0, 'percentile_99_5': 897.0, 'std': 192.37086486816406}}} + +2023-10-11 22:29:36.372318: unpacking dataset... +2023-10-11 22:30:36.052502: unpacking done... +2023-10-11 22:30:36.053181: do_dummy_2d_data_aug: False +2023-10-11 22:30:36.053809: Creating new 5-fold cross-validation split... +2023-10-11 22:30:36.055167: Desired fold for training: 0 +2023-10-11 22:30:36.055283: This split has 48 training and 12 validation cases. +2023-10-11 22:31:00.037469: Unable to plot network architecture: +2023-10-11 22:31:00.037561: failed to execute PosixPath('dot'), make sure the Graphviz executables are on your systems' PATH +2023-10-11 22:31:00.187478: +2023-10-11 22:31:00.187545: Epoch 0 +2023-10-11 22:31:00.187664: Current learning rate: 0.01 +2023-10-11 22:38:33.687418: train_loss 0.3305 +2023-10-11 22:38:33.687695: val_loss 0.1569 +2023-10-11 22:38:33.687808: Pseudo dice [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] +2023-10-11 22:38:33.687905: Epoch time: 453.5 s +2023-10-11 22:38:33.687979: Yayy! New best EMA pseudo Dice: 0.0 +2023-10-11 22:38:35.118591: +2023-10-11 22:38:35.118784: Epoch 1 +2023-10-11 22:38:35.118942: Current learning rate: 0.00999 +2023-10-11 22:44:17.746039: train_loss 0.1 +2023-10-11 22:44:17.746197: val_loss 0.0196 +2023-10-11 22:44:17.746302: Pseudo dice [0.0, 0.0, 0.6067, 0.0, 0.0, 0.0] +2023-10-11 22:44:17.746385: Epoch time: 342.63 s +2023-10-11 22:44:17.746453: Yayy! New best EMA pseudo Dice: 0.0101 +2023-10-11 22:44:20.693798: +2023-10-11 22:44:20.693964: Epoch 2 +2023-10-11 22:44:20.694131: Current learning rate: 0.00998 +2023-10-11 22:50:03.333036: train_loss -0.0647 +2023-10-11 22:50:03.333194: val_loss -0.2109 +2023-10-11 22:50:03.333298: Pseudo dice [0.8093, 0.0, 0.735, 0.045, 0.5273, 0.0] +2023-10-11 22:50:03.333383: Epoch time: 342.64 s +2023-10-11 22:50:03.333452: Yayy! New best EMA pseudo Dice: 0.0444 +2023-10-11 22:50:06.360998: +2023-10-11 22:50:06.361099: Epoch 3 +2023-10-11 22:50:06.361211: Current learning rate: 0.00997 +2023-10-11 22:55:49.088002: train_loss -0.2264 +2023-10-11 22:55:49.088167: val_loss -0.3036 +2023-10-11 22:55:49.088279: Pseudo dice [0.8018, 0.0, 0.736, 0.6794, 0.716, 0.0] +2023-10-11 22:55:49.088363: Epoch time: 342.73 s +2023-10-11 22:55:49.088431: Yayy! New best EMA pseudo Dice: 0.0888 +2023-10-11 22:55:51.940247: +2023-10-11 22:55:51.940346: Epoch 4 +2023-10-11 22:55:51.940458: Current learning rate: 0.00996 +2023-10-11 23:01:34.799937: train_loss -0.3008 +2023-10-11 23:01:34.800092: val_loss -0.3693 +2023-10-11 23:01:34.800195: Pseudo dice [0.8557, 0.6345, 0.7703, 0.7855, 0.7798, 0.0] +2023-10-11 23:01:34.800279: Epoch time: 342.86 s +2023-10-11 23:01:34.800346: Yayy! New best EMA pseudo Dice: 0.1437 +2023-10-11 23:01:37.979902: +2023-10-11 23:01:37.980081: Epoch 5 +2023-10-11 23:01:37.980230: Current learning rate: 0.00995 +2023-10-11 23:07:20.691968: train_loss -0.4007 +2023-10-11 23:07:20.692130: val_loss -0.4888 +2023-10-11 23:07:20.692261: Pseudo dice [0.8907, 0.7035, 0.8122, 0.7887, 0.8243, 0.0] +2023-10-11 23:07:20.692346: Epoch time: 342.71 s +2023-10-11 23:07:20.692414: Yayy! New best EMA pseudo Dice: 0.1963 +2023-10-11 23:07:23.499679: +2023-10-11 23:07:23.499862: Epoch 6 +2023-10-11 23:07:23.500035: Current learning rate: 0.00995 +2023-10-11 23:13:06.144582: train_loss -0.4332 +2023-10-11 23:13:06.144751: val_loss -0.4604 +2023-10-11 23:13:06.144855: Pseudo dice [0.861, 0.73, 0.7818, 0.7915, 0.8178, 0.0] +2023-10-11 23:13:06.144939: Epoch time: 342.65 s +2023-10-11 23:13:06.145007: Yayy! New best EMA pseudo Dice: 0.2431 +2023-10-11 23:13:09.078362: +2023-10-11 23:13:09.078465: Epoch 7 +2023-10-11 23:13:09.078587: Current learning rate: 0.00994 +2023-10-11 23:18:51.648893: train_loss -0.4474 +2023-10-11 23:18:51.649032: val_loss -0.5032 +2023-10-11 23:18:51.649149: Pseudo dice [0.913, 0.7546, 0.8447, 0.8258, 0.8079, 0.0] +2023-10-11 23:18:51.649236: Epoch time: 342.57 s +2023-10-11 23:18:51.649306: Yayy! New best EMA pseudo Dice: 0.2879 +2023-10-11 23:18:54.520814: +2023-10-11 23:18:54.520916: Epoch 8 +2023-10-11 23:18:54.521026: Current learning rate: 0.00993 +2023-10-11 23:24:37.289511: train_loss -0.4881 +2023-10-11 23:24:37.289653: val_loss -0.4913 +2023-10-11 23:24:37.289769: Pseudo dice [0.9085, 0.7704, 0.8537, 0.8218, 0.8026, 0.0] +2023-10-11 23:24:37.289853: Epoch time: 342.77 s +2023-10-11 23:24:37.289920: Yayy! New best EMA pseudo Dice: 0.3284 +2023-10-11 23:24:40.216775: +2023-10-11 23:24:40.217001: Epoch 9 +2023-10-11 23:24:40.217103: Current learning rate: 0.00992 +2023-10-11 23:30:22.797530: train_loss -0.4902 +2023-10-11 23:30:22.797673: val_loss -0.5356 +2023-10-11 23:30:22.797789: Pseudo dice [0.9146, 0.7888, 0.8284, 0.8295, 0.8006, 0.0] +2023-10-11 23:30:22.797873: Epoch time: 342.58 s +2023-10-11 23:30:22.797939: Yayy! New best EMA pseudo Dice: 0.3649 +2023-10-11 23:30:25.758161: +2023-10-11 23:30:25.758345: Epoch 10 +2023-10-11 23:30:25.758482: Current learning rate: 0.00991 +2023-10-11 23:36:08.547217: train_loss -0.4997 +2023-10-11 23:36:08.547385: val_loss -0.5129 +2023-10-11 23:36:08.547490: Pseudo dice [0.9026, 0.7708, 0.8408, 0.8528, 0.8346, 0.0] +2023-10-11 23:36:08.547574: Epoch time: 342.79 s +2023-10-11 23:36:08.547641: Yayy! New best EMA pseudo Dice: 0.3984 +2023-10-11 23:36:11.542623: +2023-10-11 23:36:11.542796: Epoch 11 +2023-10-11 23:36:11.542936: Current learning rate: 0.0099 +2023-10-11 23:41:54.202858: train_loss -0.4955 +2023-10-11 23:41:54.203000: val_loss -0.4787 +2023-10-11 23:41:54.203117: Pseudo dice [0.9209, 0.8089, 0.8654, 0.8144, 0.764, 0.0] +2023-10-11 23:41:54.203265: Epoch time: 342.66 s +2023-10-11 23:41:54.203400: Yayy! New best EMA pseudo Dice: 0.4281 +2023-10-11 23:41:57.017556: +2023-10-11 23:41:57.017736: Epoch 12 +2023-10-11 23:41:57.017895: Current learning rate: 0.00989 +2023-10-11 23:47:39.721987: train_loss -0.5109 +2023-10-11 23:47:39.722129: val_loss -0.5359 +2023-10-11 23:47:39.722256: Pseudo dice [0.9311, 0.8118, 0.8609, 0.8103, 0.8442, 0.0] +2023-10-11 23:47:39.722341: Epoch time: 342.71 s +2023-10-11 23:47:39.722409: Yayy! New best EMA pseudo Dice: 0.4563 +2023-10-11 23:47:42.742083: +2023-10-11 23:47:42.742293: Epoch 13 +2023-10-11 23:47:42.742465: Current learning rate: 0.00988 +2023-10-11 23:53:25.410082: train_loss -0.5398 +2023-10-11 23:53:25.410223: val_loss -0.4869 +2023-10-11 23:53:25.410341: Pseudo dice [0.9182, 0.8271, 0.8478, 0.8612, 0.801, 0.0] +2023-10-11 23:53:25.410427: Epoch time: 342.67 s +2023-10-11 23:53:25.410503: Yayy! New best EMA pseudo Dice: 0.4816 +2023-10-11 23:53:28.331167: +2023-10-11 23:53:28.331286: Epoch 14 +2023-10-11 23:53:28.331388: Current learning rate: 0.00987 +2023-10-11 23:59:11.074016: train_loss -0.5407 +2023-10-11 23:59:11.074182: val_loss -0.5519 +2023-10-11 23:59:11.074285: Pseudo dice [0.9377, 0.8028, 0.8768, 0.8591, 0.829, 0.0] +2023-10-11 23:59:11.074369: Epoch time: 342.74 s +2023-10-11 23:59:11.074436: Yayy! New best EMA pseudo Dice: 0.5052 +2023-10-11 23:59:13.988856: +2023-10-11 23:59:13.988958: Epoch 15 +2023-10-11 23:59:13.989069: Current learning rate: 0.00986 +2023-10-12 00:04:56.570926: train_loss -0.5517 +2023-10-12 00:04:56.571094: val_loss -0.5715 +2023-10-12 00:04:56.571198: Pseudo dice [0.934, 0.8172, 0.8664, 0.8845, 0.8525, 0.0] +2023-10-12 00:04:56.571281: Epoch time: 342.58 s +2023-10-12 00:04:56.571347: Yayy! New best EMA pseudo Dice: 0.5272 +2023-10-12 00:04:59.549285: +2023-10-12 00:04:59.549387: Epoch 16 +2023-10-12 00:04:59.549503: Current learning rate: 0.00986 +2023-10-12 00:10:42.199685: train_loss -0.5591 +2023-10-12 00:10:42.199848: val_loss -0.5948 +2023-10-12 00:10:42.199950: Pseudo dice [0.9399, 0.8401, 0.8654, 0.8417, 0.8713, 0.0] +2023-10-12 00:10:42.200033: Epoch time: 342.65 s +2023-10-12 00:10:42.200099: Yayy! New best EMA pseudo Dice: 0.5472 +2023-10-12 00:10:45.176628: +2023-10-12 00:10:45.176836: Epoch 17 +2023-10-12 00:10:45.176981: Current learning rate: 0.00985 +2023-10-12 00:16:27.844501: train_loss -0.5701 +2023-10-12 00:16:27.844658: val_loss -0.5614 +2023-10-12 00:16:27.844759: Pseudo dice [0.9418, 0.8429, 0.88, 0.8683, 0.8667, 0.0] +2023-10-12 00:16:27.844842: Epoch time: 342.67 s +2023-10-12 00:16:27.844907: Yayy! New best EMA pseudo Dice: 0.5658 +2023-10-12 00:16:30.977469: +2023-10-12 00:16:30.977576: Epoch 18 +2023-10-12 00:16:30.977688: Current learning rate: 0.00984 +2023-10-12 00:22:13.482676: train_loss -0.5577 +2023-10-12 00:22:13.482824: val_loss -0.524 +2023-10-12 00:22:13.482958: Pseudo dice [0.9248, 0.8559, 0.8652, 0.8643, 0.8672, 0.0] +2023-10-12 00:22:13.483040: Epoch time: 342.51 s +2023-10-12 00:22:13.483106: Yayy! New best EMA pseudo Dice: 0.5821 +2023-10-12 00:22:16.384195: +2023-10-12 00:22:16.384302: Epoch 19 +2023-10-12 00:22:16.384414: Current learning rate: 0.00983 +2023-10-12 00:27:58.942358: train_loss -0.5519 +2023-10-12 00:27:58.942523: val_loss -0.5175 +2023-10-12 00:27:58.942628: Pseudo dice [0.9353, 0.8425, 0.8601, 0.7908, 0.781, 0.0] +2023-10-12 00:27:58.942716: Epoch time: 342.56 s +2023-10-12 00:27:58.942788: Yayy! New best EMA pseudo Dice: 0.5941 +2023-10-12 00:28:01.850435: +2023-10-12 00:28:01.850570: Epoch 20 +2023-10-12 00:28:01.850674: Current learning rate: 0.00982 +2023-10-12 00:33:44.570221: train_loss -0.517 +2023-10-12 00:33:44.570361: val_loss -0.5811 +2023-10-12 00:33:44.570476: Pseudo dice [0.9507, 0.8329, 0.8846, 0.8649, 0.8595, 0.0] +2023-10-12 00:33:44.570568: Epoch time: 342.72 s +2023-10-12 00:33:44.570635: Yayy! New best EMA pseudo Dice: 0.6079 +2023-10-12 00:33:47.529000: +2023-10-12 00:33:47.529245: Epoch 21 +2023-10-12 00:33:47.529409: Current learning rate: 0.00981 +2023-10-12 00:39:30.235863: train_loss -0.5526 +2023-10-12 00:39:30.236005: val_loss -0.5554 +2023-10-12 00:39:30.236123: Pseudo dice [0.9491, 0.8424, 0.8665, 0.8802, 0.869, 0.0] +2023-10-12 00:39:30.236208: Epoch time: 342.71 s +2023-10-12 00:39:30.236274: Yayy! New best EMA pseudo Dice: 0.6206 +2023-10-12 00:39:33.221772: +2023-10-12 00:39:33.221872: Epoch 22 +2023-10-12 00:39:33.221993: Current learning rate: 0.0098 +2023-10-12 00:45:15.981112: train_loss -0.5585 +2023-10-12 00:45:15.981260: val_loss -0.5775 +2023-10-12 00:45:15.981365: Pseudo dice [0.9463, 0.8491, 0.877, 0.8735, 0.8828, 0.0] +2023-10-12 00:45:15.981449: Epoch time: 342.76 s +2023-10-12 00:45:15.981516: Yayy! New best EMA pseudo Dice: 0.6323 +2023-10-12 00:45:18.891257: +2023-10-12 00:45:18.891359: Epoch 23 +2023-10-12 00:45:18.891469: Current learning rate: 0.00979 +2023-10-12 00:51:01.616500: train_loss -0.5534 +2023-10-12 00:51:01.616653: val_loss -0.5812 +2023-10-12 00:51:01.616764: Pseudo dice [0.9351, 0.8302, 0.8701, 0.8399, 0.8528, 0.0] +2023-10-12 00:51:01.616854: Epoch time: 342.73 s +2023-10-12 00:51:01.616927: Yayy! New best EMA pseudo Dice: 0.6412 +2023-10-12 00:51:04.684643: +2023-10-12 00:51:04.684751: Epoch 24 +2023-10-12 00:51:04.684861: Current learning rate: 0.00978 +2023-10-12 00:56:47.442679: train_loss -0.5944 +2023-10-12 00:56:47.442824: val_loss -0.6473 +2023-10-12 00:56:47.442926: Pseudo dice [0.9483, 0.8309, 0.8741, 0.8906, 0.8572, 0.5147] +2023-10-12 00:56:47.443011: Epoch time: 342.76 s +2023-10-12 00:56:47.443076: Yayy! New best EMA pseudo Dice: 0.659 +2023-10-12 00:56:50.391332: +2023-10-12 00:56:50.391534: Epoch 25 +2023-10-12 00:56:50.391695: Current learning rate: 0.00977 +2023-10-12 01:02:33.109845: train_loss -0.6224 +2023-10-12 01:02:33.110010: val_loss -0.6117 +2023-10-12 01:02:33.110114: Pseudo dice [0.9563, 0.8463, 0.8835, 0.84, 0.8385, 0.6043] +2023-10-12 01:02:33.110199: Epoch time: 342.72 s +2023-10-12 01:02:33.110265: Yayy! New best EMA pseudo Dice: 0.6759 +2023-10-12 01:02:36.121195: +2023-10-12 01:02:36.121370: Epoch 26 +2023-10-12 01:02:36.121526: Current learning rate: 0.00977 +2023-10-12 01:08:18.789280: train_loss -0.6489 +2023-10-12 01:08:18.789443: val_loss -0.6464 +2023-10-12 01:08:18.789547: Pseudo dice [0.9347, 0.8268, 0.8829, 0.871, 0.8726, 0.5889] +2023-10-12 01:08:18.789632: Epoch time: 342.67 s +2023-10-12 01:08:18.789698: Yayy! New best EMA pseudo Dice: 0.6913 +2023-10-12 01:08:21.695619: +2023-10-12 01:08:21.695788: Epoch 27 +2023-10-12 01:08:21.695943: Current learning rate: 0.00976 +2023-10-12 01:14:04.459669: train_loss -0.6472 +2023-10-12 01:14:04.459808: val_loss -0.6443 +2023-10-12 01:14:04.459924: Pseudo dice [0.9497, 0.8591, 0.8796, 0.8993, 0.8484, 0.6367] +2023-10-12 01:14:04.460010: Epoch time: 342.76 s +2023-10-12 01:14:04.460077: Yayy! New best EMA pseudo Dice: 0.7067 +2023-10-12 01:14:07.386564: +2023-10-12 01:14:07.386698: Epoch 28 +2023-10-12 01:14:07.386843: Current learning rate: 0.00975 +2023-10-12 01:19:50.016520: train_loss -0.623 +2023-10-12 01:19:50.016670: val_loss -0.6463 +2023-10-12 01:19:50.016785: Pseudo dice [0.9265, 0.845, 0.8795, 0.8767, 0.8752, 0.617] +2023-10-12 01:19:50.016870: Epoch time: 342.63 s +2023-10-12 01:19:50.016936: Yayy! New best EMA pseudo Dice: 0.7197 +2023-10-12 01:19:53.001982: +2023-10-12 01:19:53.002169: Epoch 29 +2023-10-12 01:19:53.002318: Current learning rate: 0.00974 +2023-10-12 01:25:35.783812: train_loss -0.6172 +2023-10-12 01:25:35.783977: val_loss -0.5865 +2023-10-12 01:25:35.784088: Pseudo dice [0.9265, 0.8455, 0.8764, 0.8203, 0.8404, 0.5675] +2023-10-12 01:25:35.784180: Epoch time: 342.78 s +2023-10-12 01:25:35.784253: Yayy! New best EMA pseudo Dice: 0.729 +2023-10-12 01:25:38.625413: +2023-10-12 01:25:38.625515: Epoch 30 +2023-10-12 01:25:38.625627: Current learning rate: 0.00973 +2023-10-12 01:31:21.328883: train_loss -0.6353 +2023-10-12 01:31:21.329023: val_loss -0.6142 +2023-10-12 01:31:21.329140: Pseudo dice [0.9566, 0.831, 0.8805, 0.8746, 0.8843, 0.6147] +2023-10-12 01:31:21.329224: Epoch time: 342.7 s +2023-10-12 01:31:21.329290: Yayy! New best EMA pseudo Dice: 0.7401 +2023-10-12 01:31:24.429391: +2023-10-12 01:31:24.429501: Epoch 31 +2023-10-12 01:31:24.429613: Current learning rate: 0.00972 +2023-10-12 01:37:07.158125: train_loss -0.639 +2023-10-12 01:37:07.158267: val_loss -0.6231 +2023-10-12 01:37:07.158381: Pseudo dice [0.9482, 0.8382, 0.8953, 0.8494, 0.8876, 0.5817] +2023-10-12 01:37:07.158476: Epoch time: 342.73 s +2023-10-12 01:37:07.158552: Yayy! New best EMA pseudo Dice: 0.7495 +2023-10-12 01:37:10.026229: +2023-10-12 01:37:10.026357: Epoch 32 +2023-10-12 01:37:10.026456: Current learning rate: 0.00971 +2023-10-12 01:42:52.822203: train_loss -0.6082 +2023-10-12 01:42:52.822344: val_loss -0.6614 +2023-10-12 01:42:52.822460: Pseudo dice [0.9518, 0.8594, 0.8803, 0.8726, 0.8759, 0.6035] +2023-10-12 01:42:52.822555: Epoch time: 342.8 s +2023-10-12 01:42:52.822622: Yayy! New best EMA pseudo Dice: 0.7586 +2023-10-12 01:42:55.673703: +2023-10-12 01:42:55.673932: Epoch 33 +2023-10-12 01:42:55.674134: Current learning rate: 0.0097 +2023-10-12 01:48:38.505801: train_loss -0.6797 +2023-10-12 01:48:38.505965: val_loss -0.6487 +2023-10-12 01:48:38.506079: Pseudo dice [0.958, 0.8636, 0.8891, 0.8728, 0.887, 0.6317] +2023-10-12 01:48:38.506172: Epoch time: 342.83 s +2023-10-12 01:48:38.506248: Yayy! New best EMA pseudo Dice: 0.7678 +2023-10-12 01:48:41.508611: +2023-10-12 01:48:41.508802: Epoch 34 +2023-10-12 01:48:41.508956: Current learning rate: 0.00969 +2023-10-12 01:54:24.225562: train_loss -0.6525 +2023-10-12 01:54:24.225725: val_loss -0.6808 +2023-10-12 01:54:24.225835: Pseudo dice [0.9591, 0.8662, 0.8827, 0.8999, 0.8602, 0.6213] +2023-10-12 01:54:24.225919: Epoch time: 342.72 s +2023-10-12 01:54:24.225984: Yayy! New best EMA pseudo Dice: 0.7758 +2023-10-12 01:54:27.185761: +2023-10-12 01:54:27.185879: Epoch 35 +2023-10-12 01:54:27.185980: Current learning rate: 0.00968 +2023-10-12 02:00:09.886006: train_loss -0.6528 +2023-10-12 02:00:09.886148: val_loss -0.6571 +2023-10-12 02:00:09.886266: Pseudo dice [0.9584, 0.8622, 0.8909, 0.8638, 0.8552, 0.6143] +2023-10-12 02:00:09.886357: Epoch time: 342.7 s +2023-10-12 02:00:09.886424: Yayy! New best EMA pseudo Dice: 0.7823 +2023-10-12 02:00:12.820828: +2023-10-12 02:00:12.821004: Epoch 36 +2023-10-12 02:00:12.821142: Current learning rate: 0.00968 +2023-10-12 02:05:55.568484: train_loss -0.6566 +2023-10-12 02:05:55.568649: val_loss -0.6458 +2023-10-12 02:05:55.568751: Pseudo dice [0.9531, 0.8673, 0.8747, 0.8155, 0.8778, 0.6109] +2023-10-12 02:05:55.568835: Epoch time: 342.75 s +2023-10-12 02:05:55.568902: Yayy! New best EMA pseudo Dice: 0.7874 +2023-10-12 02:05:58.438680: +2023-10-12 02:05:58.438787: Epoch 37 +2023-10-12 02:05:58.438954: Current learning rate: 0.00967 +2023-10-12 02:11:41.179587: train_loss -0.6662 +2023-10-12 02:11:41.179727: val_loss -0.6137 +2023-10-12 02:11:41.179843: Pseudo dice [0.9578, 0.8733, 0.8759, 0.8445, 0.8892, 0.6538] +2023-10-12 02:11:41.179927: Epoch time: 342.74 s +2023-10-12 02:11:41.179992: Yayy! New best EMA pseudo Dice: 0.7936 +2023-10-12 02:11:44.303593: +2023-10-12 02:11:44.303703: Epoch 38 +2023-10-12 02:11:44.303822: Current learning rate: 0.00966 +2023-10-12 02:17:26.972083: train_loss -0.6685 +2023-10-12 02:17:26.972223: val_loss -0.6539 +2023-10-12 02:17:26.972318: Pseudo dice [0.9526, 0.8474, 0.9028, 0.8639, 0.8578, 0.5606] +2023-10-12 02:17:26.972396: Epoch time: 342.67 s +2023-10-12 02:17:26.972457: Yayy! New best EMA pseudo Dice: 0.7973 +2023-10-12 02:17:30.031160: +2023-10-12 02:17:30.031424: Epoch 39 +2023-10-12 02:17:30.031568: Current learning rate: 0.00965 +2023-10-12 02:23:12.805249: train_loss -0.6623 +2023-10-12 02:23:12.805413: val_loss -0.6468 +2023-10-12 02:23:12.805516: Pseudo dice [0.9641, 0.8535, 0.8901, 0.8526, 0.8458, 0.6642] +2023-10-12 02:23:12.805599: Epoch time: 342.77 s +2023-10-12 02:23:12.805668: Yayy! New best EMA pseudo Dice: 0.8021 +2023-10-12 02:23:15.669306: +2023-10-12 02:23:15.669412: Epoch 40 +2023-10-12 02:23:15.669523: Current learning rate: 0.00964 +2023-10-12 02:28:58.333120: train_loss -0.649 +2023-10-12 02:28:58.333281: val_loss -0.6801 +2023-10-12 02:28:58.333383: Pseudo dice [0.9605, 0.8591, 0.8981, 0.875, 0.867, 0.6115] +2023-10-12 02:28:58.333466: Epoch time: 342.66 s +2023-10-12 02:28:58.333533: Yayy! New best EMA pseudo Dice: 0.8064 +2023-10-12 02:29:01.269073: +2023-10-12 02:29:01.269283: Epoch 41 +2023-10-12 02:29:01.269450: Current learning rate: 0.00963 +2023-10-12 02:34:43.944252: train_loss -0.6771 +2023-10-12 02:34:43.944388: val_loss -0.6759 +2023-10-12 02:34:43.944504: Pseudo dice [0.9576, 0.8639, 0.8814, 0.8736, 0.8853, 0.5876] +2023-10-12 02:34:43.944587: Epoch time: 342.68 s +2023-10-12 02:34:43.944652: Yayy! New best EMA pseudo Dice: 0.8099 +2023-10-12 02:34:46.943379: +2023-10-12 02:34:46.943635: Epoch 42 +2023-10-12 02:34:46.943798: Current learning rate: 0.00962 +2023-10-12 02:40:29.601466: train_loss -0.6515 +2023-10-12 02:40:29.601615: val_loss -0.6721 +2023-10-12 02:40:29.601732: Pseudo dice [0.9654, 0.8628, 0.9028, 0.8678, 0.8729, 0.5543] +2023-10-12 02:40:29.601815: Epoch time: 342.66 s +2023-10-12 02:40:29.601882: Yayy! New best EMA pseudo Dice: 0.8127 +2023-10-12 02:40:32.505672: +2023-10-12 02:40:32.505788: Epoch 43 +2023-10-12 02:40:32.505918: Current learning rate: 0.00961 +2023-10-12 02:46:15.183424: train_loss -0.6771 +2023-10-12 02:46:15.183583: val_loss -0.7023 +2023-10-12 02:46:15.183686: Pseudo dice [0.9628, 0.8788, 0.8994, 0.8529, 0.8701, 0.6573] +2023-10-12 02:46:15.183770: Epoch time: 342.68 s +2023-10-12 02:46:15.183836: Yayy! New best EMA pseudo Dice: 0.8168 +2023-10-12 02:46:18.055420: +2023-10-12 02:46:18.055524: Epoch 44 +2023-10-12 02:46:18.055635: Current learning rate: 0.0096 +2023-10-12 02:52:00.799792: train_loss -0.6935 +2023-10-12 02:52:00.799942: val_loss -0.6716 +2023-10-12 02:52:00.800045: Pseudo dice [0.9639, 0.8725, 0.8994, 0.8427, 0.8738, 0.5512] +2023-10-12 02:52:00.800130: Epoch time: 342.75 s +2023-10-12 02:52:00.800197: Yayy! New best EMA pseudo Dice: 0.8185 +2023-10-12 02:52:03.794958: +2023-10-12 02:52:03.795069: Epoch 45 +2023-10-12 02:52:03.795179: Current learning rate: 0.00959 +2023-10-12 02:57:46.430653: train_loss -0.6698 +2023-10-12 02:57:46.430804: val_loss -0.6114 +2023-10-12 02:57:46.430939: Pseudo dice [0.9602, 0.8766, 0.8886, 0.8633, 0.9038, 0.5267] +2023-10-12 02:57:46.431023: Epoch time: 342.64 s +2023-10-12 02:57:46.431090: Yayy! New best EMA pseudo Dice: 0.8203 +2023-10-12 02:57:49.277206: +2023-10-12 02:57:49.277392: Epoch 46 +2023-10-12 02:57:49.277533: Current learning rate: 0.00959 +2023-10-12 03:03:31.985590: train_loss -0.6711 +2023-10-12 03:03:31.985733: val_loss -0.6669 +2023-10-12 03:03:31.985848: Pseudo dice [0.9555, 0.8576, 0.8802, 0.8617, 0.8848, 0.6323] +2023-10-12 03:03:31.985934: Epoch time: 342.71 s +2023-10-12 03:03:31.986000: Yayy! New best EMA pseudo Dice: 0.8228 +2023-10-12 03:03:34.897163: +2023-10-12 03:03:34.897288: Epoch 47 +2023-10-12 03:03:34.897399: Current learning rate: 0.00958 +2023-10-12 03:09:17.649697: train_loss -0.673 +2023-10-12 03:09:17.649856: val_loss -0.6772 +2023-10-12 03:09:17.649958: Pseudo dice [0.9605, 0.8647, 0.8952, 0.8883, 0.895, 0.5998] +2023-10-12 03:09:17.650042: Epoch time: 342.75 s +2023-10-12 03:09:17.650110: Yayy! New best EMA pseudo Dice: 0.8256 +2023-10-12 03:09:20.483692: +2023-10-12 03:09:20.483812: Epoch 48 +2023-10-12 03:09:20.483913: Current learning rate: 0.00957 +2023-10-12 03:15:03.284327: train_loss -0.6809 +2023-10-12 03:15:03.284477: val_loss -0.6497 +2023-10-12 03:15:03.284581: Pseudo dice [0.9623, 0.8676, 0.8863, 0.8477, 0.8741, 0.6658] +2023-10-12 03:15:03.284665: Epoch time: 342.8 s +2023-10-12 03:15:03.284731: Yayy! New best EMA pseudo Dice: 0.8281 +2023-10-12 03:15:06.229518: +2023-10-12 03:15:06.229706: Epoch 49 +2023-10-12 03:15:06.229843: Current learning rate: 0.00956 +2023-10-12 03:20:48.823942: train_loss -0.7025 +2023-10-12 03:20:48.824103: val_loss -0.6927 +2023-10-12 03:20:48.824207: Pseudo dice [0.9601, 0.8755, 0.8947, 0.864, 0.8533, 0.6383] +2023-10-12 03:20:48.824291: Epoch time: 342.6 s +2023-10-12 03:20:49.090307: Yayy! New best EMA pseudo Dice: 0.83 +2023-10-12 03:20:51.928224: +2023-10-12 03:20:51.928394: Epoch 50 +2023-10-12 03:20:51.928543: Current learning rate: 0.00955 +2023-10-12 03:26:34.535890: train_loss -0.6878 +2023-10-12 03:26:34.536057: val_loss -0.6967 +2023-10-12 03:26:34.536161: Pseudo dice [0.9635, 0.878, 0.8803, 0.8999, 0.8848, 0.641] +2023-10-12 03:26:34.536254: Epoch time: 342.61 s +2023-10-12 03:26:34.536330: Yayy! New best EMA pseudo Dice: 0.8328 +2023-10-12 03:26:37.390921: +2023-10-12 03:26:37.391089: Epoch 51 +2023-10-12 03:26:37.391240: Current learning rate: 0.00954 +2023-10-12 03:32:20.085535: train_loss -0.6762 +2023-10-12 03:32:20.085674: val_loss -0.6512 +2023-10-12 03:32:20.085790: Pseudo dice [0.9656, 0.8607, 0.8825, 0.8757, 0.8576, 0.5627] +2023-10-12 03:32:20.085874: Epoch time: 342.7 s +2023-10-12 03:32:20.085939: Yayy! New best EMA pseudo Dice: 0.833 +2023-10-12 03:32:23.122293: +2023-10-12 03:32:23.122564: Epoch 52 +2023-10-12 03:32:23.122732: Current learning rate: 0.00953 +2023-10-12 03:38:05.884178: train_loss -0.6874 +2023-10-12 03:38:05.884315: val_loss -0.6659 +2023-10-12 03:38:05.884432: Pseudo dice [0.9606, 0.8448, 0.8838, 0.9044, 0.8555, 0.6129] +2023-10-12 03:38:05.884517: Epoch time: 342.76 s +2023-10-12 03:38:05.884582: Yayy! New best EMA pseudo Dice: 0.834 +2023-10-12 03:38:08.709363: +2023-10-12 03:38:08.709556: Epoch 53 +2023-10-12 03:38:08.709723: Current learning rate: 0.00952 +2023-10-12 03:43:51.522039: train_loss -0.6682 +2023-10-12 03:43:51.522176: val_loss -0.6825 +2023-10-12 03:43:51.522291: Pseudo dice [0.9635, 0.8694, 0.8873, 0.8545, 0.8823, 0.6455] +2023-10-12 03:43:51.522374: Epoch time: 342.81 s +2023-10-12 03:43:51.522439: Yayy! New best EMA pseudo Dice: 0.8357 +2023-10-12 03:43:54.469362: +2023-10-12 03:43:54.469526: Epoch 54 +2023-10-12 03:43:54.469666: Current learning rate: 0.00951 +2023-10-12 03:49:37.244408: train_loss -0.6646 +2023-10-12 03:49:37.244572: val_loss -0.6459 +2023-10-12 03:49:37.244677: Pseudo dice [0.958, 0.8648, 0.8894, 0.8212, 0.8473, 0.6225] +2023-10-12 03:49:37.244762: Epoch time: 342.78 s +2023-10-12 03:49:38.415977: +2023-10-12 03:49:38.416141: Epoch 55 +2023-10-12 03:49:38.416296: Current learning rate: 0.0095 +2023-10-12 03:55:21.031140: train_loss -0.6771 +2023-10-12 03:55:21.031288: val_loss -0.6434 +2023-10-12 03:55:21.031403: Pseudo dice [0.9622, 0.8653, 0.9004, 0.8651, 0.8921, 0.6596] +2023-10-12 03:55:21.031487: Epoch time: 342.62 s +2023-10-12 03:55:21.031554: Yayy! New best EMA pseudo Dice: 0.8377 +2023-10-12 03:55:23.850996: +2023-10-12 03:55:23.851180: Epoch 56 +2023-10-12 03:55:23.851328: Current learning rate: 0.00949 +2023-10-12 04:01:06.563989: train_loss -0.6761 +2023-10-12 04:01:06.564136: val_loss -0.6529 +2023-10-12 04:01:06.564255: Pseudo dice [0.9589, 0.8712, 0.8802, 0.8473, 0.8655, 0.6699] +2023-10-12 04:01:06.564337: Epoch time: 342.71 s +2023-10-12 04:01:06.564403: Yayy! New best EMA pseudo Dice: 0.8388 +2023-10-12 04:01:09.472729: +2023-10-12 04:01:09.472833: Epoch 57 +2023-10-12 04:01:09.472944: Current learning rate: 0.00949 +2023-10-12 04:06:52.253654: train_loss -0.7023 +2023-10-12 04:06:52.253812: val_loss -0.6734 +2023-10-12 04:06:52.253916: Pseudo dice [0.9629, 0.8755, 0.896, 0.8331, 0.9001, 0.6162] +2023-10-12 04:06:52.254001: Epoch time: 342.78 s +2023-10-12 04:06:52.254083: Yayy! New best EMA pseudo Dice: 0.8396 +2023-10-12 04:06:55.141241: +2023-10-12 04:06:55.141393: Epoch 58 +2023-10-12 04:06:55.141538: Current learning rate: 0.00948 +2023-10-12 04:12:37.833030: train_loss -0.718 +2023-10-12 04:12:37.833198: val_loss -0.6426 +2023-10-12 04:12:37.833310: Pseudo dice [0.9654, 0.8752, 0.8943, 0.9053, 0.878, 0.6342] +2023-10-12 04:12:37.833396: Epoch time: 342.69 s +2023-10-12 04:12:37.833463: Yayy! New best EMA pseudo Dice: 0.8416 +2023-10-12 04:12:40.818926: +2023-10-12 04:12:40.819038: Epoch 59 +2023-10-12 04:12:40.819149: Current learning rate: 0.00947 +2023-10-12 04:18:23.663529: train_loss -0.7037 +2023-10-12 04:18:23.663677: val_loss -0.6508 +2023-10-12 04:18:23.663778: Pseudo dice [0.9558, 0.8639, 0.8906, 0.9021, 0.8991, 0.6666] +2023-10-12 04:18:23.663862: Epoch time: 342.85 s +2023-10-12 04:18:23.663927: Yayy! New best EMA pseudo Dice: 0.8437 +2023-10-12 04:18:26.566714: +2023-10-12 04:18:26.566957: Epoch 60 +2023-10-12 04:18:26.567060: Current learning rate: 0.00946 +2023-10-12 04:24:09.448377: train_loss -0.6698 +2023-10-12 04:24:09.448532: val_loss -0.6743 +2023-10-12 04:24:09.448645: Pseudo dice [0.9649, 0.8556, 0.87, 0.9131, 0.8955, 0.6766] +2023-10-12 04:24:09.448729: Epoch time: 342.88 s +2023-10-12 04:24:09.448796: Yayy! New best EMA pseudo Dice: 0.8456 +2023-10-12 04:24:12.349111: +2023-10-12 04:24:12.349216: Epoch 61 +2023-10-12 04:24:12.349326: Current learning rate: 0.00945 +2023-10-12 04:29:55.242771: train_loss -0.678 +2023-10-12 04:29:55.242922: val_loss -0.6383 +2023-10-12 04:29:55.243038: Pseudo dice [0.9653, 0.8732, 0.8937, 0.8693, 0.8466, 0.6936] +2023-10-12 04:29:55.243124: Epoch time: 342.89 s +2023-10-12 04:29:55.243191: Yayy! New best EMA pseudo Dice: 0.8467 +2023-10-12 04:29:58.183920: +2023-10-12 04:29:58.184027: Epoch 62 +2023-10-12 04:29:58.184139: Current learning rate: 0.00944 +2023-10-12 04:35:41.021065: train_loss -0.6971 +2023-10-12 04:35:41.021258: val_loss -0.6741 +2023-10-12 04:35:41.021361: Pseudo dice [0.9645, 0.8764, 0.8967, 0.8681, 0.8859, 0.7067] +2023-10-12 04:35:41.021445: Epoch time: 342.84 s +2023-10-12 04:35:41.021511: Yayy! New best EMA pseudo Dice: 0.8487 +2023-10-12 04:35:43.993739: +2023-10-12 04:35:43.993844: Epoch 63 +2023-10-12 04:35:43.993958: Current learning rate: 0.00943 +2023-10-12 04:41:26.713228: train_loss -0.6903 +2023-10-12 04:41:26.713369: val_loss -0.6597 +2023-10-12 04:41:26.713486: Pseudo dice [0.9662, 0.8861, 0.896, 0.8593, 0.8683, 0.6197] +2023-10-12 04:41:26.713571: Epoch time: 342.72 s +2023-10-12 04:41:26.713637: Yayy! New best EMA pseudo Dice: 0.8488 +2023-10-12 04:41:29.507500: +2023-10-12 04:41:29.507668: Epoch 64 +2023-10-12 04:41:29.507817: Current learning rate: 0.00942 +2023-10-12 04:47:12.233486: train_loss -0.6981 +2023-10-12 04:47:12.233639: val_loss -0.6776 +2023-10-12 04:47:12.233745: Pseudo dice [0.9624, 0.8774, 0.894, 0.9048, 0.8989, 0.5812] +2023-10-12 04:47:12.233831: Epoch time: 342.73 s +2023-10-12 04:47:12.233899: Yayy! New best EMA pseudo Dice: 0.8492 +2023-10-12 04:47:15.031553: +2023-10-12 04:47:15.031721: Epoch 65 +2023-10-12 04:47:15.031874: Current learning rate: 0.00941 +2023-10-12 04:52:57.786306: train_loss -0.7058 +2023-10-12 04:52:57.786452: val_loss -0.6683 +2023-10-12 04:52:57.786578: Pseudo dice [0.9653, 0.8741, 0.9019, 0.8859, 0.9146, 0.5813] +2023-10-12 04:52:57.786662: Epoch time: 342.76 s +2023-10-12 04:52:57.786728: Yayy! New best EMA pseudo Dice: 0.8497 +2023-10-12 04:53:00.714078: +2023-10-12 04:53:00.714187: Epoch 66 +2023-10-12 04:53:00.714303: Current learning rate: 0.0094 +2023-10-12 04:58:43.447963: train_loss -0.7001 +2023-10-12 04:58:43.448120: val_loss -0.6358 +2023-10-12 04:58:43.448224: Pseudo dice [0.9591, 0.8658, 0.8803, 0.8455, 0.8603, 0.5671] +2023-10-12 04:58:43.448309: Epoch time: 342.73 s +2023-10-12 04:58:44.644797: +2023-10-12 04:58:44.644956: Epoch 67 +2023-10-12 04:58:44.645108: Current learning rate: 0.00939 +2023-10-12 05:04:27.412760: train_loss -0.6503 +2023-10-12 05:04:27.412901: val_loss -0.6497 +2023-10-12 05:04:27.413018: Pseudo dice [0.9653, 0.8763, 0.8684, 0.873, 0.9077, 0.5963] +2023-10-12 05:04:27.413103: Epoch time: 342.77 s +2023-10-12 05:04:28.635382: +2023-10-12 05:04:28.635491: Epoch 68 +2023-10-12 05:04:28.635605: Current learning rate: 0.00939 +2023-10-12 05:10:11.326452: train_loss -0.6898 +2023-10-12 05:10:11.326622: val_loss -0.6655 +2023-10-12 05:10:11.326789: Pseudo dice [0.9572, 0.8769, 0.9013, 0.8719, 0.9041, 0.6527] +2023-10-12 05:10:11.326876: Epoch time: 342.69 s +2023-10-12 05:10:12.538631: +2023-10-12 05:10:12.538848: Epoch 69 +2023-10-12 05:10:12.538953: Current learning rate: 0.00938 +2023-10-12 05:15:55.096313: train_loss -0.7111 +2023-10-12 05:15:55.096477: val_loss -0.6402 +2023-10-12 05:15:55.096580: Pseudo dice [0.9621, 0.8653, 0.8913, 0.8566, 0.8943, 0.6333] +2023-10-12 05:15:55.096664: Epoch time: 342.56 s +2023-10-12 05:15:56.309399: +2023-10-12 05:15:56.309568: Epoch 70 +2023-10-12 05:15:56.309737: Current learning rate: 0.00937 +2023-10-12 05:21:39.083196: train_loss -0.7008 +2023-10-12 05:21:39.083336: val_loss -0.6485 +2023-10-12 05:21:39.083451: Pseudo dice [0.9648, 0.8801, 0.8891, 0.8789, 0.9227, 0.5785] +2023-10-12 05:21:39.083545: Epoch time: 342.77 s +2023-10-12 05:21:40.295066: +2023-10-12 05:21:40.295240: Epoch 71 +2023-10-12 05:21:40.295396: Current learning rate: 0.00936 +2023-10-12 05:27:23.004569: train_loss -0.6906 +2023-10-12 05:27:23.004712: val_loss -0.683 +2023-10-12 05:27:23.004830: Pseudo dice [0.9688, 0.8794, 0.9001, 0.8954, 0.9265, 0.6339] +2023-10-12 05:27:23.004914: Epoch time: 342.71 s +2023-10-12 05:27:23.004981: Yayy! New best EMA pseudo Dice: 0.8512 +2023-10-12 05:27:26.045018: +2023-10-12 05:27:26.045234: Epoch 72 +2023-10-12 05:27:26.045420: Current learning rate: 0.00935 +2023-10-12 05:33:08.792828: train_loss -0.7004 +2023-10-12 05:33:08.792986: val_loss -0.6378 +2023-10-12 05:33:08.793119: Pseudo dice [0.9657, 0.8767, 0.8723, 0.8427, 0.8663, 0.5444] +2023-10-12 05:33:08.793202: Epoch time: 342.75 s +2023-10-12 05:33:10.018302: +2023-10-12 05:33:10.018414: Epoch 73 +2023-10-12 05:33:10.018535: Current learning rate: 0.00934 +2023-10-12 05:38:52.835397: train_loss -0.702 +2023-10-12 05:38:52.835539: val_loss -0.6368 +2023-10-12 05:38:52.835656: Pseudo dice [0.9696, 0.8778, 0.8856, 0.8689, 0.8448, 0.684] +2023-10-12 05:38:52.835740: Epoch time: 342.82 s +2023-10-12 05:38:54.047196: +2023-10-12 05:38:54.047383: Epoch 74 +2023-10-12 05:38:54.047541: Current learning rate: 0.00933 +2023-10-12 05:44:36.697304: train_loss -0.703 +2023-10-12 05:44:36.697446: val_loss -0.6854 +2023-10-12 05:44:36.697562: Pseudo dice [0.9585, 0.8799, 0.8905, 0.879, 0.9027, 0.577] +2023-10-12 05:44:36.697647: Epoch time: 342.65 s +2023-10-12 05:44:37.918354: +2023-10-12 05:44:37.918616: Epoch 75 +2023-10-12 05:44:37.918845: Current learning rate: 0.00932 +2023-10-12 05:50:20.651441: train_loss -0.6954 +2023-10-12 05:50:20.651585: val_loss -0.6339 +2023-10-12 05:50:20.651703: Pseudo dice [0.9617, 0.875, 0.8884, 0.8794, 0.8673, 0.6466] +2023-10-12 05:50:20.651786: Epoch time: 342.73 s +2023-10-12 05:50:21.867908: +2023-10-12 05:50:21.868014: Epoch 76 +2023-10-12 05:50:21.868127: Current learning rate: 0.00931 +2023-10-12 05:56:04.398144: train_loss -0.6866 +2023-10-12 05:56:04.398285: val_loss -0.6656 +2023-10-12 05:56:04.398402: Pseudo dice [0.9675, 0.886, 0.9094, 0.8934, 0.9055, 0.6442] +2023-10-12 05:56:04.398485: Epoch time: 342.53 s +2023-10-12 05:56:04.398570: Yayy! New best EMA pseudo Dice: 0.8515 +2023-10-12 05:56:07.266430: +2023-10-12 05:56:07.266559: Epoch 77 +2023-10-12 05:56:07.266659: Current learning rate: 0.0093 +2023-10-12 06:01:49.964866: train_loss -0.7097 +2023-10-12 06:01:49.965016: val_loss -0.7017 +2023-10-12 06:01:49.965119: Pseudo dice [0.9669, 0.8886, 0.8932, 0.8712, 0.8837, 0.6422] +2023-10-12 06:01:49.965202: Epoch time: 342.7 s +2023-10-12 06:01:49.965269: Yayy! New best EMA pseudo Dice: 0.8521 +2023-10-12 06:01:52.847392: +2023-10-12 06:01:52.847569: Epoch 78 +2023-10-12 06:01:52.847723: Current learning rate: 0.0093 +2023-10-12 06:07:35.723412: train_loss -0.7099 +2023-10-12 06:07:35.723552: val_loss -0.6979 +2023-10-12 06:07:35.723667: Pseudo dice [0.9595, 0.8722, 0.9012, 0.9019, 0.9163, 0.6194] +2023-10-12 06:07:35.723752: Epoch time: 342.88 s +2023-10-12 06:07:35.723819: Yayy! New best EMA pseudo Dice: 0.8531 +2023-10-12 06:07:38.777076: +2023-10-12 06:07:38.777188: Epoch 79 +2023-10-12 06:07:38.777308: Current learning rate: 0.00929 +2023-10-12 06:13:21.463235: train_loss -0.7063 +2023-10-12 06:13:21.463370: val_loss -0.6598 +2023-10-12 06:13:21.463487: Pseudo dice [0.9608, 0.8656, 0.9058, 0.8914, 0.8462, 0.6407] +2023-10-12 06:13:21.463571: Epoch time: 342.69 s +2023-10-12 06:13:22.696717: +2023-10-12 06:13:22.696831: Epoch 80 +2023-10-12 06:13:22.696944: Current learning rate: 0.00928 +2023-10-12 06:19:05.444056: train_loss -0.6896 +2023-10-12 06:19:05.444217: val_loss -0.6904 +2023-10-12 06:19:05.444321: Pseudo dice [0.9644, 0.8765, 0.9109, 0.8455, 0.8955, 0.6474] +2023-10-12 06:19:05.444406: Epoch time: 342.75 s +2023-10-12 06:19:05.444474: Yayy! New best EMA pseudo Dice: 0.8533 +2023-10-12 06:19:08.362256: +2023-10-12 06:19:08.362456: Epoch 81 +2023-10-12 06:19:08.362822: Current learning rate: 0.00927 +2023-10-12 06:24:51.197666: train_loss -0.7221 +2023-10-12 06:24:51.197809: val_loss -0.6425 +2023-10-12 06:24:51.197924: Pseudo dice [0.9617, 0.8814, 0.9006, 0.8604, 0.8789, 0.6372] +2023-10-12 06:24:51.198009: Epoch time: 342.84 s +2023-10-12 06:24:51.198075: Yayy! New best EMA pseudo Dice: 0.8534 +2023-10-12 06:24:53.975187: +2023-10-12 06:24:53.975383: Epoch 82 +2023-10-12 06:24:53.975539: Current learning rate: 0.00926 +2023-10-12 06:30:36.824612: train_loss -0.6657 +2023-10-12 06:30:36.824768: val_loss -0.6994 +2023-10-12 06:30:36.824870: Pseudo dice [0.9655, 0.8653, 0.9117, 0.8657, 0.8699, 0.6599] +2023-10-12 06:30:36.824955: Epoch time: 342.85 s +2023-10-12 06:30:36.825022: Yayy! New best EMA pseudo Dice: 0.8536 +2023-10-12 06:30:39.735789: +2023-10-12 06:30:39.735895: Epoch 83 +2023-10-12 06:30:39.736007: Current learning rate: 0.00925 +2023-10-12 06:36:22.355819: train_loss -0.6992 +2023-10-12 06:36:22.355960: val_loss -0.6828 +2023-10-12 06:36:22.356077: Pseudo dice [0.9581, 0.8807, 0.8973, 0.9106, 0.8701, 0.6439] +2023-10-12 06:36:22.356162: Epoch time: 342.62 s +2023-10-12 06:36:22.356229: Yayy! New best EMA pseudo Dice: 0.8543 +2023-10-12 06:36:25.142812: +2023-10-12 06:36:25.142931: Epoch 84 +2023-10-12 06:36:25.143040: Current learning rate: 0.00924 +2023-10-12 06:42:07.842564: train_loss -0.6848 +2023-10-12 06:42:07.842709: val_loss -0.6915 +2023-10-12 06:42:07.842812: Pseudo dice [0.9678, 0.8893, 0.8856, 0.8737, 0.829, 0.6135] +2023-10-12 06:42:07.842897: Epoch time: 342.7 s +2023-10-12 06:42:09.000732: +2023-10-12 06:42:09.000839: Epoch 85 +2023-10-12 06:42:09.000985: Current learning rate: 0.00923 +2023-10-12 06:47:51.741976: train_loss -0.6969 +2023-10-12 06:47:51.742143: val_loss -0.6832 +2023-10-12 06:47:51.742245: Pseudo dice [0.9659, 0.8689, 0.8918, 0.8595, 0.8642, 0.6104] +2023-10-12 06:47:51.742340: Epoch time: 342.74 s +2023-10-12 06:47:53.079365: +2023-10-12 06:47:53.079565: Epoch 86 +2023-10-12 06:47:53.079719: Current learning rate: 0.00922 +2023-10-12 06:53:35.932963: train_loss -0.7034 +2023-10-12 06:53:35.933101: val_loss -0.6869 +2023-10-12 06:53:35.933218: Pseudo dice [0.9659, 0.872, 0.904, 0.8485, 0.8782, 0.6494] +2023-10-12 06:53:35.933311: Epoch time: 342.85 s +2023-10-12 06:53:37.101764: +2023-10-12 06:53:37.101976: Epoch 87 +2023-10-12 06:53:37.102138: Current learning rate: 0.00921 +2023-10-12 06:59:19.852819: train_loss -0.7041 +2023-10-12 06:59:19.852976: val_loss -0.6249 +2023-10-12 06:59:19.853083: Pseudo dice [0.9656, 0.8525, 0.8874, 0.9227, 0.9013, 0.6356] +2023-10-12 06:59:19.853168: Epoch time: 342.75 s +2023-10-12 06:59:21.016698: +2023-10-12 06:59:21.016808: Epoch 88 +2023-10-12 06:59:21.016923: Current learning rate: 0.0092 +2023-10-12 07:05:03.896295: train_loss -0.6974 +2023-10-12 07:05:03.896439: val_loss -0.6882 +2023-10-12 07:05:03.896555: Pseudo dice [0.9633, 0.8873, 0.9035, 0.8989, 0.8855, 0.647] +2023-10-12 07:05:03.896637: Epoch time: 342.88 s +2023-10-12 07:05:05.058133: +2023-10-12 07:05:05.058331: Epoch 89 +2023-10-12 07:05:05.058472: Current learning rate: 0.0092 +2023-10-12 07:10:47.718593: train_loss -0.6985 +2023-10-12 07:10:47.718872: val_loss -0.66 +2023-10-12 07:10:47.719007: Pseudo dice [0.9691, 0.8814, 0.8949, 0.8762, 0.8743, 0.6387] +2023-10-12 07:10:47.719099: Epoch time: 342.66 s +2023-10-12 07:10:47.719173: Yayy! New best EMA pseudo Dice: 0.8544 +2023-10-12 07:10:50.653793: +2023-10-12 07:10:50.653900: Epoch 90 +2023-10-12 07:10:50.654014: Current learning rate: 0.00919 +2023-10-12 07:16:33.546522: train_loss -0.7004 +2023-10-12 07:16:33.546669: val_loss -0.6958 +2023-10-12 07:16:33.546772: Pseudo dice [0.9604, 0.8824, 0.899, 0.9158, 0.8762, 0.6541] +2023-10-12 07:16:33.546858: Epoch time: 342.89 s +2023-10-12 07:16:33.546925: Yayy! New best EMA pseudo Dice: 0.8554 +2023-10-12 07:16:36.377405: +2023-10-12 07:16:36.377552: Epoch 91 +2023-10-12 07:16:36.377711: Current learning rate: 0.00918 +2023-10-12 07:22:19.174566: train_loss -0.7095 +2023-10-12 07:22:19.174711: val_loss -0.7015 +2023-10-12 07:22:19.174826: Pseudo dice [0.9682, 0.8735, 0.9175, 0.8545, 0.8863, 0.6532] +2023-10-12 07:22:19.174912: Epoch time: 342.8 s +2023-10-12 07:22:19.174978: Yayy! New best EMA pseudo Dice: 0.8558 +2023-10-12 07:22:22.116857: +2023-10-12 07:22:22.116975: Epoch 92 +2023-10-12 07:22:22.117074: Current learning rate: 0.00917 +2023-10-12 07:28:04.724467: train_loss -0.701 +2023-10-12 07:28:04.724608: val_loss -0.7053 +2023-10-12 07:28:04.724729: Pseudo dice [0.9567, 0.8742, 0.9101, 0.8231, 0.9084, 0.6539] +2023-10-12 07:28:04.724813: Epoch time: 342.61 s +2023-10-12 07:28:06.252353: +2023-10-12 07:28:06.252460: Epoch 93 +2023-10-12 07:28:06.252578: Current learning rate: 0.00916 +2023-10-12 07:33:48.937072: train_loss -0.7295 +2023-10-12 07:33:48.937215: val_loss -0.6671 +2023-10-12 07:33:48.937332: Pseudo dice [0.9648, 0.8719, 0.8935, 0.8838, 0.8792, 0.644] +2023-10-12 07:33:48.937417: Epoch time: 342.69 s +2023-10-12 07:33:50.080722: +2023-10-12 07:33:50.080908: Epoch 94 +2023-10-12 07:33:50.081055: Current learning rate: 0.00915 +2023-10-12 07:39:32.667760: train_loss -0.6867 +2023-10-12 07:39:32.667897: val_loss -0.6532 +2023-10-12 07:39:32.668016: Pseudo dice [0.9675, 0.8708, 0.8867, 0.8352, 0.8922, 0.6069] +2023-10-12 07:39:32.668100: Epoch time: 342.59 s +2023-10-12 07:39:33.811063: +2023-10-12 07:39:33.811174: Epoch 95 +2023-10-12 07:39:33.811286: Current learning rate: 0.00914 +2023-10-12 07:45:16.183323: train_loss -0.6982 +2023-10-12 07:45:16.183482: val_loss -0.6877 +2023-10-12 07:45:16.183584: Pseudo dice [0.9711, 0.8747, 0.9095, 0.8966, 0.8604, 0.5875] +2023-10-12 07:45:16.183668: Epoch time: 342.37 s +2023-10-12 07:45:17.337628: +2023-10-12 07:45:17.337737: Epoch 96 +2023-10-12 07:45:17.337850: Current learning rate: 0.00913 +2023-10-12 07:50:59.978549: train_loss -0.7098 +2023-10-12 07:50:59.978710: val_loss -0.6297 +2023-10-12 07:50:59.978815: Pseudo dice [0.9671, 0.8688, 0.8859, 0.8562, 0.8601, 0.6018] +2023-10-12 07:50:59.978901: Epoch time: 342.64 s +2023-10-12 07:51:01.145121: +2023-10-12 07:51:01.145227: Epoch 97 +2023-10-12 07:51:01.145341: Current learning rate: 0.00912 +2023-10-12 07:56:43.914813: train_loss -0.688 +2023-10-12 07:56:43.914990: val_loss -0.6901 +2023-10-12 07:56:43.915093: Pseudo dice [0.9692, 0.8882, 0.9069, 0.8599, 0.8741, 0.6219] +2023-10-12 07:56:43.915186: Epoch time: 342.77 s +2023-10-12 07:56:45.089589: +2023-10-12 07:56:45.089692: Epoch 98 +2023-10-12 07:56:45.089805: Current learning rate: 0.00911 +2023-10-12 08:02:27.726690: train_loss -0.7182 +2023-10-12 08:02:27.726837: val_loss -0.6632 +2023-10-12 08:02:27.726940: Pseudo dice [0.9708, 0.8807, 0.9011, 0.8587, 0.8779, 0.6274] +2023-10-12 08:02:27.727023: Epoch time: 342.64 s +2023-10-12 08:02:28.901976: +2023-10-12 08:02:28.902079: Epoch 99 +2023-10-12 08:02:28.902190: Current learning rate: 0.0091 +2023-10-12 08:08:11.521181: train_loss -0.6938 +2023-10-12 08:08:11.521321: val_loss -0.6329 +2023-10-12 08:08:11.521438: Pseudo dice [0.9663, 0.8761, 0.8694, 0.8593, 0.8822, 0.6335] +2023-10-12 08:08:11.521523: Epoch time: 342.62 s +2023-10-12 08:08:14.476202: +2023-10-12 08:08:14.476400: Epoch 100 +2023-10-12 08:08:14.476566: Current learning rate: 0.0091 +2023-10-12 08:13:57.240213: train_loss -0.7033 +2023-10-12 08:13:57.240356: val_loss -0.6263 +2023-10-12 08:13:57.240474: Pseudo dice [0.967, 0.8668, 0.9058, 0.9035, 0.892, 0.6196] +2023-10-12 08:13:57.240558: Epoch time: 342.76 s +2023-10-12 08:13:58.414634: +2023-10-12 08:13:58.414752: Epoch 101 +2023-10-12 08:13:58.414852: Current learning rate: 0.00909 +2023-10-12 08:19:41.176234: train_loss -0.7 +2023-10-12 08:19:41.176375: val_loss -0.6654 +2023-10-12 08:19:41.176492: Pseudo dice [0.9611, 0.8782, 0.9, 0.8681, 0.8765, 0.6026] +2023-10-12 08:19:41.176576: Epoch time: 342.76 s +2023-10-12 08:19:42.344409: +2023-10-12 08:19:42.344521: Epoch 102 +2023-10-12 08:19:42.344632: Current learning rate: 0.00908 +2023-10-12 08:25:25.077127: train_loss -0.7336 +2023-10-12 08:25:25.077268: val_loss -0.6567 +2023-10-12 08:25:25.077382: Pseudo dice [0.9687, 0.8794, 0.9085, 0.8314, 0.8802, 0.6228] +2023-10-12 08:25:25.077476: Epoch time: 342.73 s +2023-10-12 08:25:26.253280: +2023-10-12 08:25:26.253471: Epoch 103 +2023-10-12 08:25:26.253610: Current learning rate: 0.00907 +2023-10-12 08:31:08.905929: train_loss -0.7293 +2023-10-12 08:31:08.906089: val_loss -0.6572 +2023-10-12 08:31:08.906191: Pseudo dice [0.9688, 0.8751, 0.911, 0.8885, 0.8879, 0.5588] +2023-10-12 08:31:08.906275: Epoch time: 342.65 s +2023-10-12 08:31:10.072607: +2023-10-12 08:31:10.072715: Epoch 104 +2023-10-12 08:31:10.072828: Current learning rate: 0.00906 +2023-10-12 08:36:52.762826: train_loss -0.7081 +2023-10-12 08:36:52.762977: val_loss -0.6649 +2023-10-12 08:36:52.763077: Pseudo dice [0.954, 0.8618, 0.8971, 0.8444, 0.8712, 0.6374] +2023-10-12 08:36:52.763160: Epoch time: 342.69 s +2023-10-12 08:36:53.932966: +2023-10-12 08:36:53.933086: Epoch 105 +2023-10-12 08:36:53.933186: Current learning rate: 0.00905 +2023-10-12 08:42:36.590616: train_loss -0.7119 +2023-10-12 08:42:36.590776: val_loss -0.6882 +2023-10-12 08:42:36.590878: Pseudo dice [0.9648, 0.8822, 0.8975, 0.9024, 0.8638, 0.6626] +2023-10-12 08:42:36.590962: Epoch time: 342.66 s +2023-10-12 08:42:37.758587: +2023-10-12 08:42:37.758751: Epoch 106 +2023-10-12 08:42:37.758898: Current learning rate: 0.00904 +2023-10-12 08:48:20.499249: train_loss -0.716 +2023-10-12 08:48:20.499392: val_loss -0.6908 +2023-10-12 08:48:20.499508: Pseudo dice [0.9705, 0.8739, 0.8974, 0.873, 0.8666, 0.6633] +2023-10-12 08:48:20.499592: Epoch time: 342.74 s +2023-10-12 08:48:21.675272: +2023-10-12 08:48:21.675390: Epoch 107 +2023-10-12 08:48:21.675490: Current learning rate: 0.00903 +2023-10-12 08:54:04.427676: train_loss -0.7107 +2023-10-12 08:54:04.427835: val_loss -0.6779 +2023-10-12 08:54:04.427946: Pseudo dice [0.9622, 0.8712, 0.8736, 0.8669, 0.9025, 0.6301] +2023-10-12 08:54:04.428030: Epoch time: 342.75 s +2023-10-12 08:54:05.788336: +2023-10-12 08:54:05.788512: Epoch 108 +2023-10-12 08:54:05.788669: Current learning rate: 0.00902 +2023-10-12 08:59:48.428390: train_loss -0.6994 +2023-10-12 08:59:48.428535: val_loss -0.6947 +2023-10-12 08:59:48.428633: Pseudo dice [0.9639, 0.8848, 0.9055, 0.856, 0.8779, 0.6246] +2023-10-12 08:59:48.428712: Epoch time: 342.64 s +2023-10-12 08:59:49.597537: +2023-10-12 08:59:49.597714: Epoch 109 +2023-10-12 08:59:49.597863: Current learning rate: 0.00901 +2023-10-12 09:05:32.351168: train_loss -0.7145 +2023-10-12 09:05:32.351316: val_loss -0.6889 +2023-10-12 09:05:32.351433: Pseudo dice [0.9543, 0.8747, 0.9061, 0.8692, 0.8758, 0.6647] +2023-10-12 09:05:32.351518: Epoch time: 342.75 s +2023-10-12 09:05:33.523484: +2023-10-12 09:05:33.523598: Epoch 110 +2023-10-12 09:05:33.523713: Current learning rate: 0.009 +2023-10-12 09:11:16.336903: train_loss -0.7183 +2023-10-12 09:11:16.337045: val_loss -0.6509 +2023-10-12 09:11:16.337162: Pseudo dice [0.9689, 0.8776, 0.8968, 0.8664, 0.8493, 0.6475] +2023-10-12 09:11:16.337248: Epoch time: 342.81 s +2023-10-12 09:11:17.507339: +2023-10-12 09:11:17.507487: Epoch 111 +2023-10-12 09:11:17.507586: Current learning rate: 0.009 +2023-10-12 09:17:00.293949: train_loss -0.7228 +2023-10-12 09:17:00.294088: val_loss -0.7134 +2023-10-12 09:17:00.294203: Pseudo dice [0.9692, 0.8801, 0.9031, 0.9116, 0.9008, 0.6058] +2023-10-12 09:17:00.294286: Epoch time: 342.79 s +2023-10-12 09:17:01.468265: +2023-10-12 09:17:01.468432: Epoch 112 +2023-10-12 09:17:01.468594: Current learning rate: 0.00899 +2023-10-12 09:22:44.133150: train_loss -0.7356 +2023-10-12 09:22:44.133337: val_loss -0.6241 +2023-10-12 09:22:44.133454: Pseudo dice [0.9726, 0.8816, 0.8877, 0.8443, 0.8852, 0.661] +2023-10-12 09:22:44.133538: Epoch time: 342.67 s +2023-10-12 09:22:45.305122: +2023-10-12 09:22:45.305225: Epoch 113 +2023-10-12 09:22:45.305336: Current learning rate: 0.00898 +2023-10-12 09:28:27.941352: train_loss -0.7106 +2023-10-12 09:28:27.941515: val_loss -0.6825 +2023-10-12 09:28:27.941618: Pseudo dice [0.9662, 0.8693, 0.8907, 0.7995, 0.8911, 0.6213] +2023-10-12 09:28:27.941702: Epoch time: 342.64 s +2023-10-12 09:28:29.112319: +2023-10-12 09:28:29.112427: Epoch 114 +2023-10-12 09:28:29.112538: Current learning rate: 0.00897 +2023-10-12 09:34:11.882530: train_loss -0.7254 +2023-10-12 09:34:11.882685: val_loss -0.6656 +2023-10-12 09:34:11.882788: Pseudo dice [0.971, 0.886, 0.8922, 0.8704, 0.9129, 0.5979] +2023-10-12 09:34:11.882871: Epoch time: 342.77 s +2023-10-12 09:34:13.229863: +2023-10-12 09:34:13.230054: Epoch 115 +2023-10-12 09:34:13.230227: Current learning rate: 0.00896 +2023-10-12 09:39:55.950988: train_loss -0.7292 +2023-10-12 09:39:55.951132: val_loss -0.6855 +2023-10-12 09:39:55.951250: Pseudo dice [0.9703, 0.8874, 0.8919, 0.8996, 0.8984, 0.6062] +2023-10-12 09:39:55.951334: Epoch time: 342.72 s +2023-10-12 09:39:57.148154: +2023-10-12 09:39:57.148329: Epoch 116 +2023-10-12 09:39:57.148503: Current learning rate: 0.00895 +2023-10-12 09:45:39.990070: train_loss -0.7096 +2023-10-12 09:45:39.990213: val_loss -0.7079 +2023-10-12 09:45:39.990309: Pseudo dice [0.9666, 0.8798, 0.9003, 0.8755, 0.8773, 0.6122] +2023-10-12 09:45:39.990387: Epoch time: 342.84 s +2023-10-12 09:45:41.183538: +2023-10-12 09:45:41.183711: Epoch 117 +2023-10-12 09:45:41.183862: Current learning rate: 0.00894 +2023-10-12 09:51:23.935846: train_loss -0.7157 +2023-10-12 09:51:23.935995: val_loss -0.662 +2023-10-12 09:51:23.936112: Pseudo dice [0.9674, 0.8675, 0.9005, 0.881, 0.8537, 0.6441] +2023-10-12 09:51:23.936189: Epoch time: 342.75 s +2023-10-12 09:51:25.128881: +2023-10-12 09:51:25.129070: Epoch 118 +2023-10-12 09:51:25.129220: Current learning rate: 0.00893 +2023-10-12 09:57:07.836169: train_loss -0.6938 +2023-10-12 09:57:07.836309: val_loss -0.6776 +2023-10-12 09:57:07.836403: Pseudo dice [0.9616, 0.8725, 0.9016, 0.8656, 0.8887, 0.7236] +2023-10-12 09:57:07.836482: Epoch time: 342.71 s +2023-10-12 09:57:09.029338: +2023-10-12 09:57:09.029476: Epoch 119 +2023-10-12 09:57:09.029574: Current learning rate: 0.00892 +2023-10-12 10:02:51.657176: train_loss -0.7066 +2023-10-12 10:02:51.657321: val_loss -0.6491 +2023-10-12 10:02:51.657424: Pseudo dice [0.9654, 0.8736, 0.887, 0.9152, 0.9205, 0.6693] +2023-10-12 10:02:51.657508: Epoch time: 342.63 s +2023-10-12 10:02:51.657589: Yayy! New best EMA pseudo Dice: 0.8564 +2023-10-12 10:02:54.541004: +2023-10-12 10:02:54.541125: Epoch 120 +2023-10-12 10:02:54.541224: Current learning rate: 0.00891 +2023-10-12 10:08:37.240723: train_loss -0.7125 +2023-10-12 10:08:37.240864: val_loss -0.6957 +2023-10-12 10:08:37.240959: Pseudo dice [0.9619, 0.8821, 0.8798, 0.8406, 0.9017, 0.6682] +2023-10-12 10:08:37.241040: Epoch time: 342.7 s +2023-10-12 10:08:38.437456: +2023-10-12 10:08:38.437560: Epoch 121 +2023-10-12 10:08:38.437653: Current learning rate: 0.0089 +2023-10-12 10:14:21.062388: train_loss -0.7035 +2023-10-12 10:14:21.062563: val_loss -0.6584 +2023-10-12 10:14:21.062666: Pseudo dice [0.9676, 0.8733, 0.9098, 0.8841, 0.8683, 0.6102] +2023-10-12 10:14:21.062762: Epoch time: 342.63 s +2023-10-12 10:14:22.439190: +2023-10-12 10:14:22.439375: Epoch 122 +2023-10-12 10:14:22.439590: Current learning rate: 0.00889 +2023-10-12 10:20:05.143665: train_loss -0.6971 +2023-10-12 10:20:05.143806: val_loss -0.6863 +2023-10-12 10:20:05.143902: Pseudo dice [0.9693, 0.8831, 0.8887, 0.8821, 0.9016, 0.6279] +2023-10-12 10:20:05.143978: Epoch time: 342.71 s +2023-10-12 10:20:06.340227: +2023-10-12 10:20:06.340336: Epoch 123 +2023-10-12 10:20:06.340435: Current learning rate: 0.00889 +2023-10-12 10:25:48.950250: train_loss -0.7136 +2023-10-12 10:25:48.950408: val_loss -0.6915 +2023-10-12 10:25:48.950520: Pseudo dice [0.9697, 0.8824, 0.8877, 0.9138, 0.8917, 0.6399] +2023-10-12 10:25:48.950606: Epoch time: 342.61 s +2023-10-12 10:25:48.950673: Yayy! New best EMA pseudo Dice: 0.857 +2023-10-12 10:25:51.862279: +2023-10-12 10:25:51.862391: Epoch 124 +2023-10-12 10:25:51.862491: Current learning rate: 0.00888 +2023-10-12 10:31:34.448738: train_loss -0.6884 +2023-10-12 10:31:34.448878: val_loss -0.6977 +2023-10-12 10:31:34.448973: Pseudo dice [0.9673, 0.8748, 0.9209, 0.8361, 0.8841, 0.6659] +2023-10-12 10:31:34.449051: Epoch time: 342.59 s +2023-10-12 10:31:34.449122: Yayy! New best EMA pseudo Dice: 0.8571 +2023-10-12 10:31:37.475441: +2023-10-12 10:31:37.475546: Epoch 125 +2023-10-12 10:31:37.475658: Current learning rate: 0.00887 +2023-10-12 10:37:20.285969: train_loss -0.7418 +2023-10-12 10:37:20.286215: val_loss -0.6551 +2023-10-12 10:37:20.286335: Pseudo dice [0.9598, 0.8687, 0.9071, 0.8698, 0.844, 0.5828] +2023-10-12 10:37:20.286418: Epoch time: 342.81 s +2023-10-12 10:37:21.479414: +2023-10-12 10:37:21.479569: Epoch 126 +2023-10-12 10:37:21.479704: Current learning rate: 0.00886 +2023-10-12 10:43:04.120555: train_loss -0.7262 +2023-10-12 10:43:04.120695: val_loss -0.6989 +2023-10-12 10:43:04.120790: Pseudo dice [0.9679, 0.871, 0.8808, 0.8784, 0.8876, 0.62] +2023-10-12 10:43:04.120867: Epoch time: 342.64 s +2023-10-12 10:43:05.311504: +2023-10-12 10:43:05.311735: Epoch 127 +2023-10-12 10:43:05.311889: Current learning rate: 0.00885 +2023-10-12 10:48:48.107555: train_loss -0.7398 +2023-10-12 10:48:48.107701: val_loss -0.6676 +2023-10-12 10:48:48.107793: Pseudo dice [0.9649, 0.8787, 0.9056, 0.8508, 0.8851, 0.6548] +2023-10-12 10:48:48.107871: Epoch time: 342.8 s +2023-10-12 10:48:49.318334: +2023-10-12 10:48:49.318436: Epoch 128 +2023-10-12 10:48:49.318552: Current learning rate: 0.00884 +2023-10-12 10:54:31.977488: train_loss -0.7126 +2023-10-12 10:54:31.977626: val_loss -0.6764 +2023-10-12 10:54:31.977720: Pseudo dice [0.9671, 0.8811, 0.9074, 0.8557, 0.9112, 0.6178] +2023-10-12 10:54:31.977798: Epoch time: 342.66 s +2023-10-12 10:54:33.357039: +2023-10-12 10:54:33.357152: Epoch 129 +2023-10-12 10:54:33.357269: Current learning rate: 0.00883 +2023-10-12 11:00:15.897394: train_loss -0.7324 +2023-10-12 11:00:15.897535: val_loss -0.6907 +2023-10-12 11:00:15.897633: Pseudo dice [0.9658, 0.8667, 0.8936, 0.8836, 0.8751, 0.6422] +2023-10-12 11:00:15.897712: Epoch time: 342.54 s +2023-10-12 11:00:17.102871: +2023-10-12 11:00:17.103132: Epoch 130 +2023-10-12 11:00:17.103325: Current learning rate: 0.00882 +2023-10-12 11:05:59.688554: train_loss -0.7338 +2023-10-12 11:05:59.688689: val_loss -0.6467 +2023-10-12 11:05:59.688785: Pseudo dice [0.9693, 0.8704, 0.892, 0.8906, 0.8847, 0.6612] +2023-10-12 11:05:59.688865: Epoch time: 342.59 s +2023-10-12 11:06:00.884331: +2023-10-12 11:06:00.884435: Epoch 131 +2023-10-12 11:06:00.884547: Current learning rate: 0.00881 +2023-10-12 11:11:43.649311: train_loss -0.697 +2023-10-12 11:11:43.649481: val_loss -0.6545 +2023-10-12 11:11:43.649586: Pseudo dice [0.9645, 0.8701, 0.9059, 0.8846, 0.8762, 0.6001] +2023-10-12 11:11:43.649680: Epoch time: 342.77 s +2023-10-12 11:11:44.845575: +2023-10-12 11:11:44.845733: Epoch 132 +2023-10-12 11:11:44.845901: Current learning rate: 0.0088 +2023-10-12 11:17:27.673482: train_loss -0.6975 +2023-10-12 11:17:27.673623: val_loss -0.7302 +2023-10-12 11:17:27.673719: Pseudo dice [0.9674, 0.8773, 0.9138, 0.8878, 0.8918, 0.6692] +2023-10-12 11:17:27.673797: Epoch time: 342.83 s +2023-10-12 11:17:28.875715: +2023-10-12 11:17:28.875899: Epoch 133 +2023-10-12 11:17:28.876049: Current learning rate: 0.00879 +2023-10-12 11:23:11.541055: train_loss -0.7337 +2023-10-12 11:23:11.541214: val_loss -0.6292 +2023-10-12 11:23:11.541316: Pseudo dice [0.9675, 0.8718, 0.8905, 0.8697, 0.8461, 0.6347] +2023-10-12 11:23:11.541400: Epoch time: 342.67 s +2023-10-12 11:23:12.747550: +2023-10-12 11:23:12.747735: Epoch 134 +2023-10-12 11:23:12.747881: Current learning rate: 0.00879 +2023-10-12 11:28:55.437577: train_loss -0.6988 +2023-10-12 11:28:55.437732: val_loss -0.6552 +2023-10-12 11:28:55.437834: Pseudo dice [0.9635, 0.8734, 0.9077, 0.8576, 0.8884, 0.6494] +2023-10-12 11:28:55.437918: Epoch time: 342.69 s +2023-10-12 11:28:56.656896: +2023-10-12 11:28:56.657040: Epoch 135 +2023-10-12 11:28:56.657170: Current learning rate: 0.00878 +2023-10-12 11:34:39.395446: train_loss -0.7113 +2023-10-12 11:34:39.395586: val_loss -0.6513 +2023-10-12 11:34:39.395681: Pseudo dice [0.9676, 0.8772, 0.8919, 0.862, 0.8558, 0.6567] +2023-10-12 11:34:39.395760: Epoch time: 342.74 s +2023-10-12 11:34:40.788161: +2023-10-12 11:34:40.788277: Epoch 136 +2023-10-12 11:34:40.788395: Current learning rate: 0.00877 +2023-10-12 11:40:23.571124: train_loss -0.7185 +2023-10-12 11:40:23.571274: val_loss -0.6318 +2023-10-12 11:40:23.571375: Pseudo dice [0.9663, 0.8431, 0.8916, 0.8614, 0.8697, 0.6768] +2023-10-12 11:40:23.571476: Epoch time: 342.78 s +2023-10-12 11:40:24.789135: +2023-10-12 11:40:24.789324: Epoch 137 +2023-10-12 11:40:24.789487: Current learning rate: 0.00876 +2023-10-12 11:46:07.383138: train_loss -0.7338 +2023-10-12 11:46:07.383281: val_loss -0.6667 +2023-10-12 11:46:07.383377: Pseudo dice [0.9623, 0.8665, 0.884, 0.9147, 0.8803, 0.5913] +2023-10-12 11:46:07.383455: Epoch time: 342.59 s +2023-10-12 11:46:08.608421: +2023-10-12 11:46:08.608620: Epoch 138 +2023-10-12 11:46:08.608784: Current learning rate: 0.00875 +2023-10-12 11:51:51.313194: train_loss -0.7063 +2023-10-12 11:51:51.313342: val_loss -0.7088 +2023-10-12 11:51:51.313436: Pseudo dice [0.9687, 0.8624, 0.8962, 0.9072, 0.8974, 0.566] +2023-10-12 11:51:51.313514: Epoch time: 342.71 s +2023-10-12 11:51:52.529474: +2023-10-12 11:51:52.529584: Epoch 139 +2023-10-12 11:51:52.529695: Current learning rate: 0.00874 +2023-10-12 11:57:35.292670: train_loss -0.742 +2023-10-12 11:57:35.292814: val_loss -0.7016 +2023-10-12 11:57:35.292908: Pseudo dice [0.9669, 0.8807, 0.9066, 0.8987, 0.8944, 0.639] +2023-10-12 11:57:35.292985: Epoch time: 342.76 s +2023-10-12 11:57:36.515310: +2023-10-12 11:57:36.515501: Epoch 140 +2023-10-12 11:57:36.515623: Current learning rate: 0.00873 +2023-10-12 12:03:19.330701: train_loss -0.7291 +2023-10-12 12:03:19.330850: val_loss -0.6461 +2023-10-12 12:03:19.330954: Pseudo dice [0.9694, 0.8785, 0.8999, 0.8789, 0.8677, 0.6303] +2023-10-12 12:03:19.331049: Epoch time: 342.82 s +2023-10-12 12:03:20.548048: +2023-10-12 12:03:20.548152: Epoch 141 +2023-10-12 12:03:20.548262: Current learning rate: 0.00872 +2023-10-12 12:09:03.292077: train_loss -0.7269 +2023-10-12 12:09:03.292224: val_loss -0.635 +2023-10-12 12:09:03.292319: Pseudo dice [0.966, 0.868, 0.8821, 0.9123, 0.9014, 0.5604] +2023-10-12 12:09:03.292397: Epoch time: 342.74 s +2023-10-12 12:09:04.509061: +2023-10-12 12:09:04.509162: Epoch 142 +2023-10-12 12:09:04.509274: Current learning rate: 0.00871 +2023-10-12 12:14:47.297504: train_loss -0.7423 +2023-10-12 12:14:47.297676: val_loss -0.6412 +2023-10-12 12:14:47.297768: Pseudo dice [0.9625, 0.8694, 0.8838, 0.905, 0.8723, 0.6718] +2023-10-12 12:14:47.297846: Epoch time: 342.79 s +2023-10-12 12:14:48.690770: +2023-10-12 12:14:48.690945: Epoch 143 +2023-10-12 12:14:48.691047: Current learning rate: 0.0087 +2023-10-12 12:20:31.388698: train_loss -0.7423 +2023-10-12 12:20:31.388841: val_loss -0.6711 +2023-10-12 12:20:31.388938: Pseudo dice [0.9683, 0.8707, 0.9013, 0.8365, 0.8866, 0.6409] +2023-10-12 12:20:31.389015: Epoch time: 342.7 s +2023-10-12 12:20:32.603032: +2023-10-12 12:20:32.603146: Epoch 144 +2023-10-12 12:20:32.603258: Current learning rate: 0.00869 +2023-10-12 12:26:15.439014: train_loss -0.7396 +2023-10-12 12:26:15.439192: val_loss -0.6503 +2023-10-12 12:26:15.439297: Pseudo dice [0.9678, 0.8738, 0.9108, 0.8655, 0.8443, 0.5917] +2023-10-12 12:26:15.439382: Epoch time: 342.84 s +2023-10-12 12:26:16.659086: +2023-10-12 12:26:16.659199: Epoch 145 +2023-10-12 12:26:16.659311: Current learning rate: 0.00868 +2023-10-12 12:31:59.434684: train_loss -0.7457 +2023-10-12 12:31:59.434832: val_loss -0.6641 +2023-10-12 12:31:59.434943: Pseudo dice [0.9701, 0.884, 0.8962, 0.8934, 0.8949, 0.6724] +2023-10-12 12:31:59.435021: Epoch time: 342.78 s +2023-10-12 12:32:00.651871: +2023-10-12 12:32:00.651981: Epoch 146 +2023-10-12 12:32:00.652093: Current learning rate: 0.00868 +2023-10-12 12:37:43.366633: train_loss -0.7118 +2023-10-12 12:37:43.366782: val_loss -0.6357 +2023-10-12 12:37:43.366882: Pseudo dice [0.9633, 0.8751, 0.9073, 0.8807, 0.8758, 0.5614] +2023-10-12 12:37:43.366967: Epoch time: 342.72 s +2023-10-12 12:37:44.579722: +2023-10-12 12:37:44.579847: Epoch 147 +2023-10-12 12:37:44.579958: Current learning rate: 0.00867 +2023-10-12 12:43:27.247730: train_loss -0.7211 +2023-10-12 12:43:27.247880: val_loss -0.6585 +2023-10-12 12:43:27.247975: Pseudo dice [0.9661, 0.882, 0.9009, 0.9046, 0.9145, 0.6541] +2023-10-12 12:43:27.248116: Epoch time: 342.67 s +2023-10-12 12:43:28.467836: +2023-10-12 12:43:28.467934: Epoch 148 +2023-10-12 12:43:28.468044: Current learning rate: 0.00866 +2023-10-12 12:49:11.326048: train_loss -0.7291 +2023-10-12 12:49:11.326190: val_loss -0.7265 +2023-10-12 12:49:11.326285: Pseudo dice [0.967, 0.8824, 0.9088, 0.8706, 0.9179, 0.6528] +2023-10-12 12:49:11.326363: Epoch time: 342.86 s +2023-10-12 12:49:12.544180: +2023-10-12 12:49:12.544367: Epoch 149 +2023-10-12 12:49:12.544530: Current learning rate: 0.00865 +2023-10-12 12:54:55.276865: train_loss -0.7282 +2023-10-12 12:54:55.277011: val_loss -0.6804 +2023-10-12 12:54:55.277105: Pseudo dice [0.9698, 0.879, 0.9067, 0.833, 0.8661, 0.6571] +2023-10-12 12:54:55.277182: Epoch time: 342.73 s +2023-10-12 12:54:58.346410: +2023-10-12 12:54:58.346547: Epoch 150 +2023-10-12 12:54:58.346649: Current learning rate: 0.00864 +2023-10-12 13:00:40.986966: train_loss -0.7322 +2023-10-12 13:00:40.987102: val_loss -0.6658 +2023-10-12 13:00:40.987197: Pseudo dice [0.9674, 0.8726, 0.8996, 0.8716, 0.8974, 0.6285] +2023-10-12 13:00:40.987276: Epoch time: 342.64 s +2023-10-12 13:00:42.205333: +2023-10-12 13:00:42.205453: Epoch 151 +2023-10-12 13:00:42.205565: Current learning rate: 0.00863 +2023-10-12 13:06:24.884889: train_loss -0.729 +2023-10-12 13:06:24.885038: val_loss -0.66 +2023-10-12 13:06:24.885132: Pseudo dice [0.9645, 0.88, 0.9104, 0.8667, 0.9018, 0.6458] +2023-10-12 13:06:24.885212: Epoch time: 342.68 s +2023-10-12 13:06:26.100578: +2023-10-12 13:06:26.100700: Epoch 152 +2023-10-12 13:06:26.100811: Current learning rate: 0.00862 +2023-10-12 13:12:08.857471: train_loss -0.7252 +2023-10-12 13:12:08.857616: val_loss -0.7209 +2023-10-12 13:12:08.857710: Pseudo dice [0.9707, 0.8844, 0.8937, 0.8906, 0.8959, 0.641] +2023-10-12 13:12:08.857790: Epoch time: 342.76 s +2023-10-12 13:12:08.857852: Yayy! New best EMA pseudo Dice: 0.8572 +2023-10-12 13:12:11.895641: +2023-10-12 13:12:11.895744: Epoch 153 +2023-10-12 13:12:11.895854: Current learning rate: 0.00861 +2023-10-12 13:17:54.573727: train_loss -0.7295 +2023-10-12 13:17:54.573881: val_loss -0.6878 +2023-10-12 13:17:54.573983: Pseudo dice [0.9674, 0.8823, 0.8865, 0.8853, 0.8676, 0.627] +2023-10-12 13:17:54.574067: Epoch time: 342.68 s +2023-10-12 13:17:55.812576: +2023-10-12 13:17:55.812678: Epoch 154 +2023-10-12 13:17:55.812788: Current learning rate: 0.0086 +2023-10-12 13:23:38.467927: train_loss -0.7288 +2023-10-12 13:23:38.468085: val_loss -0.7026 +2023-10-12 13:23:38.468189: Pseudo dice [0.9674, 0.8774, 0.8993, 0.8604, 0.8869, 0.5955] +2023-10-12 13:23:38.468273: Epoch time: 342.66 s +2023-10-12 13:23:39.704122: +2023-10-12 13:23:39.704313: Epoch 155 +2023-10-12 13:23:39.704469: Current learning rate: 0.00859 +2023-10-12 13:29:22.340714: train_loss -0.7191 +2023-10-12 13:29:22.340870: val_loss -0.6831 +2023-10-12 13:29:22.340973: Pseudo dice [0.9714, 0.859, 0.9016, 0.8974, 0.876, 0.6507] +2023-10-12 13:29:22.341072: Epoch time: 342.64 s +2023-10-12 13:29:23.594664: +2023-10-12 13:29:23.594774: Epoch 156 +2023-10-12 13:29:23.594877: Current learning rate: 0.00858 +2023-10-12 13:35:06.287747: train_loss -0.7119 +2023-10-12 13:35:06.287890: val_loss -0.6163 +2023-10-12 13:35:06.287984: Pseudo dice [0.9671, 0.8719, 0.884, 0.8748, 0.888, 0.6274] +2023-10-12 13:35:06.288062: Epoch time: 342.69 s +2023-10-12 13:35:07.720810: +2023-10-12 13:35:07.720936: Epoch 157 +2023-10-12 13:35:07.721051: Current learning rate: 0.00858 +2023-10-12 13:40:50.486576: train_loss -0.7142 +2023-10-12 13:40:50.486725: val_loss -0.6692 +2023-10-12 13:40:50.486842: Pseudo dice [0.9672, 0.8726, 0.9037, 0.8767, 0.866, 0.667] +2023-10-12 13:40:50.486918: Epoch time: 342.77 s +2023-10-12 13:40:51.732299: +2023-10-12 13:40:51.732475: Epoch 158 +2023-10-12 13:40:51.732635: Current learning rate: 0.00857 +2023-10-12 13:46:34.546075: train_loss -0.7233 +2023-10-12 13:46:34.546219: val_loss -0.6681 +2023-10-12 13:46:34.546323: Pseudo dice [0.9706, 0.8732, 0.9085, 0.8592, 0.8984, 0.6721] +2023-10-12 13:46:34.546402: Epoch time: 342.81 s +2023-10-12 13:46:35.797614: +2023-10-12 13:46:35.797796: Epoch 159 +2023-10-12 13:46:35.797966: Current learning rate: 0.00856 +2023-10-12 13:52:18.629951: train_loss -0.7429 +2023-10-12 13:52:18.630093: val_loss -0.6758 +2023-10-12 13:52:18.630280: Pseudo dice [0.9698, 0.8782, 0.9182, 0.853, 0.8662, 0.6614] +2023-10-12 13:52:18.630405: Epoch time: 342.83 s +2023-10-12 13:52:19.869171: +2023-10-12 13:52:19.869349: Epoch 160 +2023-10-12 13:52:19.869509: Current learning rate: 0.00855 +2023-10-12 13:58:02.534563: train_loss -0.7489 +2023-10-12 13:58:02.534823: val_loss -0.6522 +2023-10-12 13:58:02.534952: Pseudo dice [0.9678, 0.8805, 0.907, 0.8918, 0.9107, 0.637] +2023-10-12 13:58:02.535036: Epoch time: 342.67 s +2023-10-12 13:58:02.535102: Yayy! New best EMA pseudo Dice: 0.8578 +2023-10-12 13:58:05.445079: +2023-10-12 13:58:05.445340: Epoch 161 +2023-10-12 13:58:05.445493: Current learning rate: 0.00854 +2023-10-12 14:03:48.162120: train_loss -0.7338 +2023-10-12 14:03:48.162257: val_loss -0.668 +2023-10-12 14:03:48.162350: Pseudo dice [0.9666, 0.8775, 0.9073, 0.8625, 0.8701, 0.6085] +2023-10-12 14:03:48.162427: Epoch time: 342.72 s +2023-10-12 14:03:49.402274: +2023-10-12 14:03:49.402451: Epoch 162 +2023-10-12 14:03:49.402808: Current learning rate: 0.00853 +2023-10-12 14:09:32.150535: train_loss -0.737 +2023-10-12 14:09:32.150676: val_loss -0.6953 +2023-10-12 14:09:32.150780: Pseudo dice [0.9597, 0.8645, 0.891, 0.9066, 0.9017, 0.6648] +2023-10-12 14:09:32.150864: Epoch time: 342.75 s +2023-10-12 14:09:33.400113: +2023-10-12 14:09:33.400273: Epoch 163 +2023-10-12 14:09:33.400417: Current learning rate: 0.00852 +2023-10-12 14:15:16.149449: train_loss -0.7352 +2023-10-12 14:15:16.149609: val_loss -0.6739 +2023-10-12 14:15:16.149712: Pseudo dice [0.9712, 0.8755, 0.8869, 0.869, 0.863, 0.6678] +2023-10-12 14:15:16.149796: Epoch time: 342.75 s +2023-10-12 14:15:17.576332: +2023-10-12 14:15:17.576458: Epoch 164 +2023-10-12 14:15:17.576558: Current learning rate: 0.00851 +2023-10-12 14:21:00.427428: train_loss -0.7241 +2023-10-12 14:21:00.427593: val_loss -0.6878 +2023-10-12 14:21:00.427696: Pseudo dice [0.9697, 0.8725, 0.9101, 0.8166, 0.915, 0.6222] +2023-10-12 14:21:00.427782: Epoch time: 342.85 s +2023-10-12 14:21:01.630303: +2023-10-12 14:21:01.630530: Epoch 165 +2023-10-12 14:21:01.630680: Current learning rate: 0.0085 +2023-10-12 14:26:44.347566: train_loss -0.7124 +2023-10-12 14:26:44.347731: val_loss -0.6362 +2023-10-12 14:26:44.347835: Pseudo dice [0.9553, 0.869, 0.8956, 0.9097, 0.8461, 0.6222] +2023-10-12 14:26:44.347919: Epoch time: 342.72 s +2023-10-12 14:26:45.549893: +2023-10-12 14:26:45.549997: Epoch 166 +2023-10-12 14:26:45.550108: Current learning rate: 0.00849 +2023-10-12 14:32:28.184649: train_loss -0.7413 +2023-10-12 14:32:28.184787: val_loss -0.6891 +2023-10-12 15:29:32.414339: Pseudo dice [0.9637, 0.8735, 0.9081, 0.8501, 0.8777, 0.6443] +2023-10-12 15:29:32.414578: Epoch time: 342.64 s +2023-10-12 15:29:33.648384: +2023-10-12 15:29:33.648486: Epoch 167 +2023-10-12 15:29:33.648597: Current learning rate: 0.00848 +2023-10-12 15:35:14.183711: train_loss -0.7291 +2023-10-12 15:35:14.183875: val_loss -0.6885 +2023-10-12 15:35:14.183980: Pseudo dice [0.9707, 0.8815, 0.8981, 0.8739, 0.8956, 0.665] +2023-10-12 15:35:14.184064: Epoch time: 340.54 s +2023-10-12 15:35:15.410174: +2023-10-12 15:35:15.410277: Epoch 168 +2023-10-12 15:35:15.410388: Current learning rate: 0.00847 +2023-10-12 15:40:58.086177: train_loss -0.7113 +2023-10-12 15:40:58.086323: val_loss -0.6846 +2023-10-12 15:40:58.086416: Pseudo dice [0.9721, 0.8777, 0.904, 0.8845, 0.8753, 0.6649] +2023-10-12 15:40:58.086492: Epoch time: 342.68 s +2023-10-12 15:40:59.318331: +2023-10-12 15:40:59.318741: Epoch 169 +2023-10-12 15:40:59.318944: Current learning rate: 0.00847 +2023-10-12 15:46:41.981069: train_loss -0.7185 +2023-10-12 15:46:41.981236: val_loss -0.6838 +2023-10-12 15:46:41.981342: Pseudo dice [0.9701, 0.8791, 0.8917, 0.8949, 0.8856, 0.6115] +2023-10-12 15:46:41.981514: Epoch time: 342.66 s +2023-10-12 15:46:43.382842: +2023-10-12 15:46:43.382970: Epoch 170 +2023-10-12 15:46:43.383069: Current learning rate: 0.00846 +2023-10-12 15:52:26.288741: train_loss -0.7365 +2023-10-12 15:52:26.288890: val_loss -0.684 +2023-10-12 15:52:26.289007: Pseudo dice [0.9672, 0.8698, 0.902, 0.891, 0.8725, 0.575] +2023-10-12 15:52:26.289090: Epoch time: 342.91 s +2023-10-12 15:52:27.513287: +2023-10-12 15:52:27.513415: Epoch 171 +2023-10-12 15:52:27.513514: Current learning rate: 0.00845 +2023-10-12 15:58:10.361173: train_loss -0.7248 +2023-10-12 15:58:10.361330: val_loss -0.7141 +2023-10-12 15:58:10.361431: Pseudo dice [0.9632, 0.8846, 0.9156, 0.8928, 0.9076, 0.6624] +2023-10-12 15:58:10.361514: Epoch time: 342.85 s +2023-10-12 15:58:11.592046: +2023-10-12 15:58:11.592165: Epoch 172 +2023-10-12 15:58:11.592265: Current learning rate: 0.00844 +2023-10-12 16:03:54.560833: train_loss -0.7324 +2023-10-12 16:03:54.560981: val_loss -0.7173 +2023-10-12 16:03:54.561096: Pseudo dice [0.9678, 0.8826, 0.9071, 0.8916, 0.8828, 0.6355] +2023-10-12 16:03:54.561180: Epoch time: 342.97 s +2023-10-12 16:03:54.561246: Yayy! New best EMA pseudo Dice: 0.8579 +2023-10-12 16:03:57.586237: +2023-10-12 16:03:57.586347: Epoch 173 +2023-10-12 16:03:57.586459: Current learning rate: 0.00843 +2023-10-12 16:09:40.238267: train_loss -0.7343 +2023-10-12 16:09:40.238410: val_loss -0.6516 +2023-10-12 16:09:40.238536: Pseudo dice [0.9628, 0.8668, 0.8967, 0.9007, 0.8984, 0.6499] +2023-10-12 16:09:40.238621: Epoch time: 342.65 s +2023-10-12 16:09:40.238777: Yayy! New best EMA pseudo Dice: 0.8584 +2023-10-12 16:09:43.080663: +2023-10-12 16:09:43.080769: Epoch 174 +2023-10-12 16:09:43.080882: Current learning rate: 0.00842 +2023-10-12 16:15:25.931065: train_loss -0.7303 +2023-10-12 16:15:25.931203: val_loss -0.6765 +2023-10-12 16:15:25.931315: Pseudo dice [0.9729, 0.879, 0.9054, 0.8629, 0.8987, 0.6721] +2023-10-12 16:15:25.931402: Epoch time: 342.85 s +2023-10-12 16:15:25.931474: Yayy! New best EMA pseudo Dice: 0.859 +2023-10-12 16:15:28.878965: +2023-10-12 16:15:28.879082: Epoch 175 +2023-10-12 16:15:28.879212: Current learning rate: 0.00841 +2023-10-12 16:21:11.641807: train_loss -0.7273 +2023-10-12 16:21:11.642030: val_loss -0.6809 +2023-10-12 16:21:11.642154: Pseudo dice [0.971, 0.8735, 0.9083, 0.8929, 0.8548, 0.639] +2023-10-12 16:21:11.642250: Epoch time: 342.76 s +2023-10-12 16:21:12.871196: +2023-10-12 16:21:12.871300: Epoch 176 +2023-10-12 16:21:12.871422: Current learning rate: 0.0084 +2023-10-12 16:26:55.550927: train_loss -0.725 +2023-10-12 16:26:55.551076: val_loss -0.6445 +2023-10-12 16:26:55.551192: Pseudo dice [0.9723, 0.877, 0.899, 0.868, 0.8442, 0.6626] +2023-10-12 16:26:55.551275: Epoch time: 342.68 s +2023-10-12 16:26:56.958582: +2023-10-12 16:26:56.958706: Epoch 177 +2023-10-12 16:26:56.958808: Current learning rate: 0.00839 +2023-10-12 16:32:39.779498: train_loss -0.7287 +2023-10-12 16:32:39.779641: val_loss -0.6101 +2023-10-12 16:32:39.779773: Pseudo dice [0.969, 0.8755, 0.8975, 0.9156, 0.8301, 0.6531] +2023-10-12 16:32:39.779867: Epoch time: 342.82 s +2023-10-12 16:32:41.012056: +2023-10-12 16:32:41.012228: Epoch 178 +2023-10-12 16:32:41.012341: Current learning rate: 0.00838 +2023-10-12 16:38:23.837477: train_loss -0.7256 +2023-10-12 16:38:23.837627: val_loss -0.6625 +2023-10-12 16:38:23.837729: Pseudo dice [0.9659, 0.8512, 0.9107, 0.853, 0.8889, 0.6601] +2023-10-12 16:38:23.837812: Epoch time: 342.83 s +2023-10-12 16:38:25.061964: +2023-10-12 16:38:25.062136: Epoch 179 +2023-10-12 16:38:25.062294: Current learning rate: 0.00837 +2023-10-12 16:44:07.881114: train_loss -0.7593 +2023-10-12 16:44:07.881262: val_loss -0.7008 +2023-10-12 16:44:07.881374: Pseudo dice [0.9686, 0.8875, 0.9094, 0.8707, 0.8982, 0.6617] +2023-10-12 16:44:07.881464: Epoch time: 342.82 s +2023-10-12 16:44:09.114801: +2023-10-12 16:44:09.114944: Epoch 180 +2023-10-12 16:44:09.115072: Current learning rate: 0.00836 +2023-10-12 16:49:51.983721: train_loss -0.7288 +2023-10-12 16:49:51.983872: val_loss -0.6964 +2023-10-12 16:49:51.983990: Pseudo dice [0.9634, 0.8814, 0.9018, 0.8801, 0.8554, 0.6708] +2023-10-12 16:49:51.984075: Epoch time: 342.87 s +2023-10-12 16:49:53.219202: +2023-10-12 16:49:53.219429: Epoch 181 +2023-10-12 16:49:53.219590: Current learning rate: 0.00836 +2023-10-12 16:55:36.037226: train_loss -0.7339 +2023-10-12 16:55:36.037390: val_loss -0.6086 +2023-10-12 16:55:36.037493: Pseudo dice [0.9653, 0.8601, 0.8885, 0.8923, 0.8848, 0.5896] +2023-10-12 16:55:36.037578: Epoch time: 342.82 s +2023-10-12 16:55:37.258619: +2023-10-12 16:55:37.258819: Epoch 182 +2023-10-12 16:55:37.258952: Current learning rate: 0.00835 +2023-10-12 17:01:20.206236: train_loss -0.7214 +2023-10-12 17:01:20.206376: val_loss -0.6769 +2023-10-12 17:01:20.206492: Pseudo dice [0.9702, 0.8693, 0.9023, 0.8876, 0.8933, 0.6962] +2023-10-12 17:01:20.206585: Epoch time: 342.95 s +2023-10-12 17:01:21.477357: +2023-10-12 17:01:21.477547: Epoch 183 +2023-10-12 17:01:21.477702: Current learning rate: 0.00834 +2023-10-12 17:07:04.423003: train_loss -0.7328 +2023-10-12 17:07:04.423144: val_loss -0.6343 +2023-10-12 17:07:04.423260: Pseudo dice [0.9703, 0.8729, 0.9025, 0.893, 0.8548, 0.6208] +2023-10-12 17:07:04.423344: Epoch time: 342.95 s +2023-10-12 17:07:05.836004: +2023-10-12 17:07:05.836117: Epoch 184 +2023-10-12 17:07:05.836244: Current learning rate: 0.00833 +2023-10-12 17:12:48.738295: train_loss -0.7575 +2023-10-12 17:12:48.738441: val_loss -0.6703 +2023-10-12 17:12:48.738582: Pseudo dice [0.9716, 0.8867, 0.8916, 0.8945, 0.8906, 0.6495] +2023-10-12 17:12:48.738667: Epoch time: 342.9 s +2023-10-12 17:12:49.968829: +2023-10-12 17:12:49.968974: Epoch 185 +2023-10-12 17:12:49.969073: Current learning rate: 0.00832 +2023-10-12 17:18:32.879580: train_loss -0.734 +2023-10-12 17:18:32.879740: val_loss -0.6906 +2023-10-12 17:18:32.879858: Pseudo dice [0.9695, 0.8708, 0.9025, 0.896, 0.9066, 0.5967] +2023-10-12 17:18:32.879955: Epoch time: 342.91 s +2023-10-12 17:18:34.114790: +2023-10-12 17:18:34.114918: Epoch 186 +2023-10-12 17:18:34.115030: Current learning rate: 0.00831 +2023-10-12 17:24:17.118692: train_loss -0.7458 +2023-10-12 17:24:17.118870: val_loss -0.6833 +2023-10-12 17:24:17.118972: Pseudo dice [0.9678, 0.8684, 0.9104, 0.8443, 0.8739, 0.7059] +2023-10-12 17:24:17.119054: Epoch time: 343.0 s +2023-10-12 17:24:18.349158: +2023-10-12 17:24:18.349381: Epoch 187 +2023-10-12 17:24:18.349552: Current learning rate: 0.0083 +2023-10-12 17:30:01.202580: train_loss -0.7104 +2023-10-12 17:30:01.202743: val_loss -0.7013 +2023-10-12 17:30:01.214925: Pseudo dice [0.9638, 0.8657, 0.9034, 0.8821, 0.8617, 0.6649] +2023-10-12 17:30:01.215024: Epoch time: 342.85 s +2023-10-12 17:30:02.445177: +2023-10-12 17:30:02.445284: Epoch 188 +2023-10-12 17:30:02.445396: Current learning rate: 0.00829 +2023-10-12 17:35:45.217837: train_loss -0.7377 +2023-10-12 17:35:45.217985: val_loss -0.6607 +2023-10-12 17:35:45.218102: Pseudo dice [0.9693, 0.8778, 0.8911, 0.8833, 0.8906, 0.6349] +2023-10-12 17:35:45.218198: Epoch time: 342.77 s +2023-10-12 17:35:46.452993: +2023-10-12 17:35:46.453094: Epoch 189 +2023-10-12 17:35:46.453216: Current learning rate: 0.00828 +2023-10-12 17:41:29.098134: train_loss -0.7264 +2023-10-12 17:41:29.098285: val_loss -0.6929 +2023-10-12 17:41:29.098402: Pseudo dice [0.9592, 0.8694, 0.8993, 0.8793, 0.8764, 0.6111] +2023-10-12 17:41:29.098487: Epoch time: 342.65 s +2023-10-12 17:41:30.338390: +2023-10-12 17:41:30.338518: Epoch 190 +2023-10-12 17:41:30.338620: Current learning rate: 0.00827 +2023-10-12 17:47:13.056570: train_loss -0.7514 +2023-10-12 17:47:13.056819: val_loss -0.7125 +2023-10-12 17:47:13.056958: Pseudo dice [0.9704, 0.8739, 0.9, 0.8848, 0.8752, 0.6712] +2023-10-12 17:47:13.057054: Epoch time: 342.72 s +2023-10-12 17:47:14.459371: +2023-10-12 17:47:14.459575: Epoch 191 +2023-10-12 17:47:14.459749: Current learning rate: 0.00826 +2023-10-12 17:52:57.099810: train_loss -0.7363 +2023-10-12 17:52:57.099960: val_loss -0.6579 +2023-10-12 17:52:57.100076: Pseudo dice [0.9653, 0.866, 0.9014, 0.8995, 0.9052, 0.6518] +2023-10-12 17:52:57.100177: Epoch time: 342.64 s +2023-10-12 17:52:58.342776: +2023-10-12 17:52:58.342921: Epoch 192 +2023-10-12 17:52:58.343034: Current learning rate: 0.00825 +2023-10-12 17:58:41.047063: train_loss -0.7282 +2023-10-12 17:58:41.047212: val_loss -0.6863 +2023-10-12 17:58:41.047328: Pseudo dice [0.9661, 0.8726, 0.9183, 0.8626, 0.8157, 0.6771] +2023-10-12 17:58:41.047413: Epoch time: 342.71 s +2023-10-12 17:58:42.298066: +2023-10-12 17:58:42.298285: Epoch 193 +2023-10-12 17:58:42.298428: Current learning rate: 0.00824 +2023-10-12 18:04:25.071100: train_loss -0.7384 +2023-10-12 18:04:25.071248: val_loss -0.6702 +2023-10-12 18:04:25.071365: Pseudo dice [0.9698, 0.8725, 0.9037, 0.862, 0.9029, 0.6776] +2023-10-12 18:04:25.071455: Epoch time: 342.77 s +2023-10-12 18:04:26.315116: +2023-10-12 18:04:26.315305: Epoch 194 +2023-10-12 18:04:26.315462: Current learning rate: 0.00824 +2023-10-12 18:10:09.016047: train_loss -0.7521 +2023-10-12 18:10:09.016183: val_loss -0.6588 +2023-10-12 18:10:09.016300: Pseudo dice [0.9707, 0.8812, 0.8959, 0.8873, 0.8879, 0.6517] +2023-10-12 18:10:09.016382: Epoch time: 342.7 s +2023-10-12 18:10:09.016448: Yayy! New best EMA pseudo Dice: 0.8591 +2023-10-12 18:10:11.931193: +2023-10-12 18:10:11.931319: Epoch 195 +2023-10-12 18:10:11.931431: Current learning rate: 0.00823 +2023-10-12 18:15:54.710111: train_loss -0.7491 +2023-10-12 18:15:54.710318: val_loss -0.6834 +2023-10-12 18:15:54.710450: Pseudo dice [0.9688, 0.8764, 0.9157, 0.8739, 0.8681, 0.6626] +2023-10-12 18:15:54.710544: Epoch time: 342.78 s +2023-10-12 18:15:54.710612: Yayy! New best EMA pseudo Dice: 0.8593 +2023-10-12 18:15:57.668374: +2023-10-12 18:15:57.668551: Epoch 196 +2023-10-12 18:15:57.668714: Current learning rate: 0.00822 +2023-10-12 18:21:40.655953: train_loss -0.719 +2023-10-12 18:21:40.656105: val_loss -0.695 +2023-10-12 18:21:40.656227: Pseudo dice [0.9685, 0.8803, 0.9044, 0.8799, 0.8943, 0.5803] +2023-10-12 18:21:40.656322: Epoch time: 342.99 s +2023-10-12 18:21:41.913481: +2023-10-12 18:21:41.913583: Epoch 197 +2023-10-12 18:21:41.913695: Current learning rate: 0.00821 +2023-10-12 18:27:24.918529: train_loss -0.7444 +2023-10-12 18:27:24.918686: val_loss -0.6982 +2023-10-12 18:27:24.918790: Pseudo dice [0.9622, 0.8607, 0.9094, 0.8701, 0.8777, 0.6856] +2023-10-12 18:27:24.918925: Epoch time: 343.01 s +2023-10-12 18:27:26.392369: +2023-10-12 18:27:26.392499: Epoch 198 +2023-10-12 18:27:26.392613: Current learning rate: 0.0082 +2023-10-12 18:33:09.479836: train_loss -0.7254 +2023-10-12 18:33:09.479978: val_loss -0.6378 +2023-10-12 18:33:09.480093: Pseudo dice [0.9677, 0.8741, 0.897, 0.8918, 0.8897, 0.6234] +2023-10-12 18:33:09.480182: Epoch time: 343.09 s +2023-10-12 18:33:10.726269: +2023-10-12 18:33:10.726507: Epoch 199 +2023-10-12 18:33:10.726722: Current learning rate: 0.00819 +2023-10-12 18:38:53.614919: train_loss -0.733 +2023-10-12 18:38:53.615065: val_loss -0.6972 +2023-10-12 18:38:53.615182: Pseudo dice [0.9701, 0.8609, 0.914, 0.8785, 0.8737, 0.634] +2023-10-12 18:38:53.615266: Epoch time: 342.89 s +2023-10-12 18:38:56.633123: +2023-10-12 18:38:56.633228: Epoch 200 +2023-10-12 18:38:56.633344: Current learning rate: 0.00818 +2023-10-12 18:44:39.431932: train_loss -0.7487 +2023-10-12 18:44:39.432073: val_loss -0.6805 +2023-10-12 18:44:39.432190: Pseudo dice [0.9218, 0.8782, 0.9042, 0.8898, 0.8663, 0.6265] +2023-10-12 18:44:39.432273: Epoch time: 342.8 s +2023-10-12 18:44:40.668882: +2023-10-12 18:44:40.669007: Epoch 201 +2023-10-12 18:44:40.669106: Current learning rate: 0.00817 +2023-10-12 18:50:23.694151: train_loss -0.7203 +2023-10-12 18:50:23.694290: val_loss -0.6804 +2023-10-12 18:50:23.694407: Pseudo dice [0.9606, 0.8583, 0.9103, 0.8432, 0.8818, 0.6827] +2023-10-12 18:50:23.694490: Epoch time: 343.03 s +2023-10-12 18:50:24.934606: +2023-10-12 18:50:24.934784: Epoch 202 +2023-10-12 18:50:24.934940: Current learning rate: 0.00816 +2023-10-12 18:56:07.830386: train_loss -0.7271 +2023-10-12 18:56:07.830556: val_loss -0.6739 +2023-10-12 18:56:07.830673: Pseudo dice [0.966, 0.8739, 0.9136, 0.9001, 0.8575, 0.6152] +2023-10-12 18:56:07.830770: Epoch time: 342.9 s +2023-10-12 18:56:09.076642: +2023-10-12 18:56:09.076750: Epoch 203 +2023-10-12 18:56:09.076849: Current learning rate: 0.00815 +2023-10-12 19:01:52.058117: train_loss -0.715 +2023-10-12 19:01:52.058267: val_loss -0.7105 +2023-10-12 19:01:52.058383: Pseudo dice [0.9679, 0.8615, 0.9059, 0.8653, 0.8696, 0.6641] +2023-10-12 19:01:52.058465: Epoch time: 342.98 s +2023-10-12 19:01:53.487819: +2023-10-12 19:01:53.487978: Epoch 204 +2023-10-12 19:01:53.488128: Current learning rate: 0.00814 +2023-10-12 19:07:36.548324: train_loss -0.7437 +2023-10-12 19:07:36.548471: val_loss -0.6672 +2023-10-12 19:07:36.548604: Pseudo dice [0.9722, 0.8736, 0.9066, 0.8853, 0.8533, 0.6505] +2023-10-12 19:07:36.548699: Epoch time: 343.06 s +2023-10-12 19:07:37.794764: +2023-10-12 19:07:37.794894: Epoch 205 +2023-10-12 19:07:37.795019: Current learning rate: 0.00813 +2023-10-12 19:13:20.768502: train_loss -0.7578 +2023-10-12 19:13:20.768649: val_loss -0.6592 +2023-10-12 19:13:20.768767: Pseudo dice [0.9709, 0.8758, 0.885, 0.8925, 0.9153, 0.6348] +2023-10-12 19:13:20.768853: Epoch time: 342.97 s +2023-10-12 19:13:21.935221: +2023-10-12 19:13:21.935444: Epoch 206 +2023-10-12 19:13:21.935563: Current learning rate: 0.00813 +2023-10-12 19:19:04.931539: train_loss -0.721 +2023-10-12 19:19:04.931689: val_loss -0.644 +2023-10-12 19:19:04.931813: Pseudo dice [0.9704, 0.8773, 0.8958, 0.8682, 0.8892, 0.6456] +2023-10-12 19:19:04.931916: Epoch time: 343.0 s +2023-10-12 19:19:06.097172: +2023-10-12 19:19:06.097351: Epoch 207 +2023-10-12 19:19:06.097548: Current learning rate: 0.00812 +2023-10-12 19:24:48.991998: train_loss -0.7417 +2023-10-12 19:24:48.992139: val_loss -0.6886 +2023-10-12 19:24:48.992251: Pseudo dice [0.9696, 0.8733, 0.9044, 0.8643, 0.8863, 0.6302] +2023-10-12 19:24:48.992341: Epoch time: 342.9 s +2023-10-12 19:24:50.167568: +2023-10-12 19:24:50.167753: Epoch 208 +2023-10-12 19:24:50.167952: Current learning rate: 0.00811 +2023-10-12 19:30:33.159427: train_loss -0.7222 +2023-10-12 19:30:33.159587: val_loss -0.6779 +2023-10-12 19:30:33.159710: Pseudo dice [0.9687, 0.8819, 0.9, 0.8716, 0.8826, 0.6478] +2023-10-12 19:30:33.159793: Epoch time: 342.99 s +2023-10-12 19:30:34.322567: +2023-10-12 19:30:34.322683: Epoch 209 +2023-10-12 19:30:34.322781: Current learning rate: 0.0081 +2023-10-12 19:36:17.241267: train_loss -0.7571 +2023-10-12 19:36:17.241492: val_loss -0.6673 +2023-10-12 19:36:17.241615: Pseudo dice [0.9698, 0.8772, 0.9061, 0.8427, 0.8661, 0.6016] +2023-10-12 19:36:17.241710: Epoch time: 342.92 s +2023-10-12 19:36:18.403964: +2023-10-12 19:36:18.404065: Epoch 210 +2023-10-12 19:36:18.404186: Current learning rate: 0.00809 +2023-10-12 19:42:01.183485: train_loss -0.727 +2023-10-12 19:42:01.183656: val_loss -0.676 +2023-10-12 19:42:01.183784: Pseudo dice [0.9695, 0.8757, 0.9086, 0.886, 0.9111, 0.6215] +2023-10-12 19:42:01.183867: Epoch time: 342.78 s +2023-10-12 19:42:02.353401: +2023-10-12 19:42:02.353504: Epoch 211 +2023-10-12 19:42:02.353627: Current learning rate: 0.00808 +2023-10-12 19:47:45.182647: train_loss -0.7275 +2023-10-12 19:47:45.182794: val_loss -0.6562 +2023-10-12 19:47:45.182925: Pseudo dice [0.9692, 0.8739, 0.9003, 0.8835, 0.879, 0.6345] +2023-10-12 19:47:45.183014: Epoch time: 342.83 s +2023-10-12 19:47:46.541347: +2023-10-12 19:47:46.541460: Epoch 212 +2023-10-12 19:47:46.541585: Current learning rate: 0.00807 +2023-10-12 19:53:29.385993: train_loss -0.747 +2023-10-12 19:53:29.386149: val_loss -0.65 +2023-10-12 19:53:29.386254: Pseudo dice [0.9646, 0.8684, 0.9022, 0.8836, 0.8825, 0.6853] +2023-10-12 19:53:29.386338: Epoch time: 342.85 s +2023-10-12 19:53:30.554340: +2023-10-12 19:53:30.554549: Epoch 213 +2023-10-12 19:53:30.554715: Current learning rate: 0.00806 +2023-10-12 19:59:13.217834: train_loss -0.7292 +2023-10-12 19:59:13.217977: val_loss -0.6834 +2023-10-12 19:59:13.218093: Pseudo dice [0.9604, 0.8733, 0.9039, 0.8759, 0.8908, 0.6458] +2023-10-12 19:59:13.218178: Epoch time: 342.66 s +2023-10-12 19:59:14.384539: +2023-10-12 19:59:14.384714: Epoch 214 +2023-10-12 19:59:14.384886: Current learning rate: 0.00805 +2023-10-12 20:04:57.171391: train_loss -0.7533 +2023-10-12 20:04:57.171532: val_loss -0.7038 +2023-10-12 20:04:57.171648: Pseudo dice [0.972, 0.8795, 0.9122, 0.8887, 0.8693, 0.6723] +2023-10-12 20:04:57.171737: Epoch time: 342.79 s +2023-10-12 20:04:58.343828: +2023-10-12 20:04:58.344035: Epoch 215 +2023-10-12 20:04:58.344211: Current learning rate: 0.00804 +2023-10-12 20:10:41.252055: train_loss -0.7404 +2023-10-12 20:10:41.252217: val_loss -0.6729 +2023-10-12 20:10:41.252321: Pseudo dice [0.9664, 0.86, 0.9037, 0.9104, 0.8902, 0.6414] +2023-10-12 20:10:41.252404: Epoch time: 342.91 s +2023-10-12 20:10:42.422717: +2023-10-12 20:10:42.422929: Epoch 216 +2023-10-12 20:10:42.423032: Current learning rate: 0.00803 +2023-10-12 20:16:25.336010: train_loss -0.7373 +2023-10-12 20:16:25.336158: val_loss -0.6776 +2023-10-12 20:16:25.336274: Pseudo dice [0.9709, 0.8782, 0.8919, 0.8924, 0.9113, 0.66] +2023-10-12 20:16:25.336421: Epoch time: 342.91 s +2023-10-12 20:16:25.336550: Yayy! New best EMA pseudo Dice: 0.8595 +2023-10-12 20:16:28.267317: +2023-10-12 20:16:28.267506: Epoch 217 +2023-10-12 20:16:28.267647: Current learning rate: 0.00802 +2023-10-12 20:22:10.961828: train_loss -0.7219 +2023-10-12 20:22:10.961968: val_loss -0.6825 +2023-10-12 20:22:10.962084: Pseudo dice [0.9698, 0.8677, 0.9059, 0.8743, 0.8276, 0.7028] +2023-10-12 20:22:10.962168: Epoch time: 342.7 s +2023-10-12 20:22:12.133820: +2023-10-12 20:22:12.134061: Epoch 218 +2023-10-12 20:22:12.134244: Current learning rate: 0.00801 +2023-10-12 20:27:54.905104: train_loss -0.7113 +2023-10-12 20:27:54.905246: val_loss -0.712 +2023-10-12 20:27:54.905380: Pseudo dice [0.9696, 0.8711, 0.9007, 0.9119, 0.9065, 0.6784] +2023-10-12 20:27:54.905476: Epoch time: 342.77 s +2023-10-12 20:27:54.905555: Yayy! New best EMA pseudo Dice: 0.8607 +2023-10-12 20:27:57.966593: +2023-10-12 20:27:57.966795: Epoch 219 +2023-10-12 20:27:57.966933: Current learning rate: 0.00801 +2023-10-12 20:33:40.710116: train_loss -0.7205 +2023-10-12 20:33:40.710277: val_loss -0.7106 +2023-10-12 20:33:40.710403: Pseudo dice [0.9684, 0.8743, 0.9164, 0.8693, 0.9071, 0.6554] +2023-10-12 20:33:40.710506: Epoch time: 342.74 s +2023-10-12 20:33:40.710585: Yayy! New best EMA pseudo Dice: 0.8612 +2023-10-12 20:33:43.695450: +2023-10-12 20:33:43.695560: Epoch 220 +2023-10-12 20:33:43.695671: Current learning rate: 0.008 +2023-10-12 20:39:26.227540: train_loss -0.7464 +2023-10-12 20:39:26.227683: val_loss -0.6936 +2023-10-12 20:39:26.227803: Pseudo dice [0.9667, 0.8716, 0.8977, 0.863, 0.8903, 0.6671] +2023-10-12 20:39:26.227898: Epoch time: 342.53 s +2023-10-12 20:39:27.394360: +2023-10-12 20:39:27.394467: Epoch 221 +2023-10-12 20:39:27.394588: Current learning rate: 0.00799 +2023-10-12 20:45:10.045250: train_loss -0.759 +2023-10-12 20:45:10.045385: val_loss -0.6494 +2023-10-12 20:45:10.045501: Pseudo dice [0.9703, 0.8781, 0.9045, 0.8907, 0.8332, 0.6343] +2023-10-12 20:45:10.045586: Epoch time: 342.65 s +2023-10-12 20:45:11.210010: +2023-10-12 20:45:11.210174: Epoch 222 +2023-10-12 20:45:11.210335: Current learning rate: 0.00798 +2023-10-12 20:50:53.763303: train_loss -0.722 +2023-10-12 20:50:53.763449: val_loss -0.6897 +2023-10-12 20:50:53.763560: Pseudo dice [0.9706, 0.8643, 0.9009, 0.8902, 0.8605, 0.6717] +2023-10-12 20:50:53.763651: Epoch time: 342.55 s +2023-10-12 20:50:54.933569: +2023-10-12 20:50:54.933739: Epoch 223 +2023-10-12 20:50:54.933878: Current learning rate: 0.00797 +2023-10-12 20:56:37.741737: train_loss -0.7268 +2023-10-12 20:56:37.741878: val_loss -0.6855 +2023-10-12 20:56:37.741992: Pseudo dice [0.9668, 0.8671, 0.888, 0.886, 0.8675, 0.7033] +2023-10-12 20:56:37.742081: Epoch time: 342.81 s +2023-10-12 20:56:38.914000: +2023-10-12 20:56:38.914104: Epoch 224 +2023-10-12 20:56:38.914225: Current learning rate: 0.00796 +2023-10-12 21:02:21.623299: train_loss -0.7485 +2023-10-12 21:02:21.623447: val_loss -0.6285 +2023-10-12 21:02:21.623561: Pseudo dice [0.9692, 0.872, 0.9105, 0.8813, 0.8439, 0.6456] +2023-10-12 21:02:21.623661: Epoch time: 342.71 s +2023-10-12 21:02:22.796241: +2023-10-12 21:02:22.796350: Epoch 225 +2023-10-12 21:02:22.796461: Current learning rate: 0.00795 +2023-10-12 21:08:05.617079: train_loss -0.7492 +2023-10-12 21:08:05.617249: val_loss -0.654 +2023-10-12 21:08:05.617373: Pseudo dice [0.9694, 0.8572, 0.9005, 0.8321, 0.8782, 0.6474] +2023-10-12 21:08:05.617469: Epoch time: 342.82 s +2023-10-12 21:08:06.950568: +2023-10-12 21:08:06.950692: Epoch 226 +2023-10-12 21:08:06.950876: Current learning rate: 0.00794 +2023-10-12 21:13:49.715450: train_loss -0.7429 +2023-10-12 21:13:49.715591: val_loss -0.6943 +2023-10-12 21:13:49.715707: Pseudo dice [0.9644, 0.8692, 0.8974, 0.8531, 0.8789, 0.6812] +2023-10-12 21:13:49.715796: Epoch time: 342.77 s +2023-10-12 21:13:50.872522: +2023-10-12 21:13:50.872639: Epoch 227 +2023-10-12 21:13:50.872762: Current learning rate: 0.00793 +2023-10-12 21:19:33.485498: train_loss -0.7127 +2023-10-12 21:19:33.485656: val_loss -0.721 +2023-10-12 21:19:33.485762: Pseudo dice [0.9678, 0.8837, 0.9069, 0.8845, 0.8724, 0.6602] +2023-10-12 21:19:33.485846: Epoch time: 342.61 s +2023-10-12 21:19:34.641048: +2023-10-12 21:19:34.641317: Epoch 228 +2023-10-12 21:19:34.641475: Current learning rate: 0.00792 +2023-10-12 21:25:17.469187: train_loss -0.7454 +2023-10-12 21:25:17.469328: val_loss -0.6982 +2023-10-12 21:25:17.469442: Pseudo dice [0.9633, 0.8773, 0.892, 0.9009, 0.8674, 0.5912] +2023-10-12 21:25:17.469531: Epoch time: 342.83 s +2023-10-12 21:25:18.622904: +2023-10-12 21:25:18.623012: Epoch 229 +2023-10-12 21:25:18.623135: Current learning rate: 0.00791 +2023-10-12 21:31:01.423483: train_loss -0.7404 +2023-10-12 21:31:01.423626: val_loss -0.6525 +2023-10-12 21:31:01.423741: Pseudo dice [0.9693, 0.8622, 0.9101, 0.8769, 0.8827, 0.6515] +2023-10-12 21:31:01.423831: Epoch time: 342.8 s +2023-10-12 21:31:02.582373: +2023-10-12 21:31:02.582480: Epoch 230 +2023-10-12 21:31:02.582601: Current learning rate: 0.0079 +2023-10-12 21:36:45.349306: train_loss -0.7402 +2023-10-12 21:36:45.349454: val_loss -0.6435 +2023-10-12 21:36:45.349573: Pseudo dice [0.9642, 0.8822, 0.8812, 0.8992, 0.8756, 0.653] +2023-10-12 21:36:45.349658: Epoch time: 342.77 s +2023-10-12 21:36:46.505594: +2023-10-12 21:36:46.505697: Epoch 231 +2023-10-12 21:36:46.505821: Current learning rate: 0.00789 +2023-10-12 21:42:29.275914: train_loss -0.7297 +2023-10-12 21:42:29.276073: val_loss -0.6819 +2023-10-12 21:42:29.276258: Pseudo dice [0.9718, 0.8617, 0.8964, 0.8629, 0.8838, 0.6141] +2023-10-12 21:42:29.276366: Epoch time: 342.77 s +2023-10-12 21:42:30.434813: +2023-10-12 21:42:30.434914: Epoch 232 +2023-10-12 21:42:30.435026: Current learning rate: 0.00789 +2023-10-12 21:48:13.285499: train_loss -0.7253 +2023-10-12 21:48:13.285645: val_loss -0.6518 +2023-10-12 21:48:13.285759: Pseudo dice [0.9684, 0.875, 0.8877, 0.8722, 0.8491, 0.6455] +2023-10-12 21:48:13.285844: Epoch time: 342.85 s +2023-10-12 21:48:14.448205: +2023-10-12 21:48:14.448382: Epoch 233 +2023-10-12 21:48:14.448570: Current learning rate: 0.00788 +2023-10-12 21:53:57.248581: train_loss -0.7303 +2023-10-12 21:53:57.248731: val_loss -0.7007 +2023-10-12 21:53:57.248864: Pseudo dice [0.9667, 0.8833, 0.9181, 0.9089, 0.9074, 0.7257] +2023-10-12 21:53:57.248961: Epoch time: 342.8 s +2023-10-12 21:53:58.403820: +2023-10-12 21:53:58.403936: Epoch 234 +2023-10-12 21:53:58.404048: Current learning rate: 0.00787 +2023-10-12 21:59:41.074805: train_loss -0.7417 +2023-10-12 21:59:41.074969: val_loss -0.6877 +2023-10-12 21:59:41.075087: Pseudo dice [0.9688, 0.8775, 0.9057, 0.9024, 0.8671, 0.6722] +2023-10-12 21:59:41.075170: Epoch time: 342.67 s +2023-10-12 21:59:42.232524: +2023-10-12 21:59:42.232652: Epoch 235 +2023-10-12 21:59:42.232774: Current learning rate: 0.00786 +2023-10-12 22:05:24.980326: train_loss -0.7391 +2023-10-12 22:05:24.980489: val_loss -0.6475 +2023-10-12 22:05:24.980595: Pseudo dice [0.9673, 0.8715, 0.8944, 0.8689, 0.8485, 0.6223] +2023-10-12 22:05:24.980680: Epoch time: 342.75 s +2023-10-12 22:05:26.137499: +2023-10-12 22:05:26.137615: Epoch 236 +2023-10-12 22:05:26.137736: Current learning rate: 0.00785 +2023-10-12 22:11:08.918922: train_loss -0.7347 +2023-10-12 22:11:08.919075: val_loss -0.7053 +2023-10-12 22:11:08.919199: Pseudo dice [0.9718, 0.8863, 0.9097, 0.9013, 0.8616, 0.6174] +2023-10-12 22:11:08.919293: Epoch time: 342.78 s +2023-10-12 22:11:10.077113: +2023-10-12 22:11:10.077224: Epoch 237 +2023-10-12 22:11:10.077347: Current learning rate: 0.00784 +2023-10-12 22:16:53.050404: train_loss -0.7463 +2023-10-12 22:16:53.050582: val_loss -0.6774 +2023-10-12 22:16:53.050707: Pseudo dice [0.9735, 0.8688, 0.9046, 0.8712, 0.8638, 0.6718] +2023-10-12 22:16:53.050804: Epoch time: 342.97 s +2023-10-12 22:16:54.215157: +2023-10-12 22:16:54.215385: Epoch 238 +2023-10-12 22:16:54.215590: Current learning rate: 0.00783 +2023-10-12 22:22:37.022554: train_loss -0.7593 +2023-10-12 22:22:37.022705: val_loss -0.6576 +2023-10-12 22:22:37.022842: Pseudo dice [0.9678, 0.8741, 0.9045, 0.9092, 0.8902, 0.6611] +2023-10-12 22:22:37.022931: Epoch time: 342.81 s +2023-10-12 22:22:38.176850: +2023-10-12 22:22:38.177043: Epoch 239 +2023-10-12 22:22:38.177230: Current learning rate: 0.00782 +2023-10-12 22:28:20.957298: train_loss -0.7507 +2023-10-12 22:28:20.957439: val_loss -0.674 +2023-10-12 22:28:20.957554: Pseudo dice [0.9698, 0.8799, 0.9184, 0.8669, 0.8947, 0.6125] +2023-10-12 22:28:20.957639: Epoch time: 342.78 s +2023-10-12 22:28:22.135167: +2023-10-12 22:28:22.135430: Epoch 240 +2023-10-12 22:28:22.135630: Current learning rate: 0.00781 +2023-10-12 22:34:05.004763: train_loss -0.7481 +2023-10-12 22:34:05.004901: val_loss -0.6944 +2023-10-12 22:34:05.005017: Pseudo dice [0.9725, 0.8681, 0.9028, 0.9092, 0.9233, 0.6601] +2023-10-12 22:34:05.005100: Epoch time: 342.87 s +2023-10-12 22:34:06.356483: +2023-10-12 22:34:06.356597: Epoch 241 +2023-10-12 22:34:06.356703: Current learning rate: 0.0078 +2023-10-12 22:39:49.076850: train_loss -0.7244 +2023-10-12 22:39:49.076998: val_loss -0.707 +2023-10-12 22:39:49.077116: Pseudo dice [0.9679, 0.8839, 0.9199, 0.9013, 0.862, 0.6682] +2023-10-12 22:39:49.077201: Epoch time: 342.72 s +2023-10-12 22:39:50.246781: +2023-10-12 22:39:50.246898: Epoch 242 +2023-10-12 22:39:50.247010: Current learning rate: 0.00779 +2023-10-12 22:45:32.961523: train_loss -0.7248 +2023-10-12 22:45:32.961663: val_loss -0.7143 +2023-10-12 22:45:32.961800: Pseudo dice [0.9657, 0.8831, 0.9082, 0.8764, 0.8901, 0.6122] +2023-10-12 22:45:32.961897: Epoch time: 342.72 s +2023-10-12 22:45:34.134473: +2023-10-12 22:45:34.134650: Epoch 243 +2023-10-12 22:45:34.134818: Current learning rate: 0.00778 +2023-10-12 22:51:16.748852: train_loss -0.7333 +2023-10-12 22:51:16.748998: val_loss -0.7133 +2023-10-12 22:51:16.749122: Pseudo dice [0.9719, 0.8874, 0.906, 0.8738, 0.8341, 0.6634] +2023-10-12 22:51:16.749217: Epoch time: 342.62 s +2023-10-12 22:51:17.920521: +2023-10-12 22:51:17.920706: Epoch 244 +2023-10-12 22:51:17.920892: Current learning rate: 0.00777 +2023-10-12 22:57:00.627321: train_loss -0.7215 +2023-10-12 22:57:00.627470: val_loss -0.675 +2023-10-12 22:57:00.627604: Pseudo dice [0.9658, 0.8799, 0.9031, 0.8573, 0.8949, 0.6007] +2023-10-12 22:57:00.627700: Epoch time: 342.71 s +2023-10-12 22:57:01.810069: +2023-10-12 22:57:01.810174: Epoch 245 +2023-10-12 22:57:01.810295: Current learning rate: 0.00777 +2023-10-12 23:02:44.624632: train_loss -0.7516 +2023-10-12 23:02:44.624782: val_loss -0.6647 +2023-10-12 23:02:44.624899: Pseudo dice [0.9713, 0.8748, 0.8972, 0.8809, 0.8949, 0.5993] +2023-10-12 23:02:44.624983: Epoch time: 342.82 s +2023-10-12 23:02:45.798906: +2023-10-12 23:02:45.799025: Epoch 246 +2023-10-12 23:02:45.799124: Current learning rate: 0.00776 +2023-10-12 23:08:28.454139: train_loss -0.7313 +2023-10-12 23:08:28.454294: val_loss -0.6794 +2023-10-12 23:08:28.454396: Pseudo dice [0.969, 0.8652, 0.9051, 0.8817, 0.8561, 0.6363] +2023-10-12 23:08:28.454489: Epoch time: 342.66 s +2023-10-12 23:08:29.629055: +2023-10-12 23:08:29.629228: Epoch 247 +2023-10-12 23:08:29.629410: Current learning rate: 0.00775 +2023-10-12 23:14:12.463871: train_loss -0.7473 +2023-10-12 23:14:12.464021: val_loss -0.6933 +2023-10-12 23:14:12.464137: Pseudo dice [0.9683, 0.8811, 0.9049, 0.8784, 0.8703, 0.6491] +2023-10-12 23:14:12.464220: Epoch time: 342.84 s +2023-10-12 23:14:13.817331: +2023-10-12 23:14:13.817439: Epoch 248 +2023-10-12 23:14:13.817554: Current learning rate: 0.00774 +2023-10-12 23:19:56.496991: train_loss -0.7327 +2023-10-12 23:19:56.497132: val_loss -0.6865 +2023-10-12 23:19:56.497241: Pseudo dice [0.9721, 0.8816, 0.9078, 0.8904, 0.8858, 0.6714] +2023-10-12 23:19:56.497328: Epoch time: 342.68 s +2023-10-12 23:19:57.673666: +2023-10-12 23:19:57.673865: Epoch 249 +2023-10-12 23:19:57.674041: Current learning rate: 0.00773 +2023-10-12 23:25:40.230918: train_loss -0.7506 +2023-10-12 23:25:40.231061: val_loss -0.6042 +2023-10-12 23:25:40.231192: Pseudo dice [0.9718, 0.8764, 0.9106, 0.8326, 0.8631, 0.6605] +2023-10-12 23:25:40.231287: Epoch time: 342.56 s +2023-10-12 23:25:43.124978: +2023-10-12 23:25:43.125215: Epoch 250 +2023-10-12 23:25:43.125373: Current learning rate: 0.00772 +2023-10-12 23:31:25.829841: train_loss -0.7537 +2023-10-12 23:31:25.829984: val_loss -0.6902 +2023-10-12 23:31:25.830094: Pseudo dice [0.9707, 0.8861, 0.9078, 0.883, 0.9075, 0.714] +2023-10-12 23:31:25.830184: Epoch time: 342.71 s +2023-10-12 23:31:27.009812: +2023-10-12 23:31:27.010027: Epoch 251 +2023-10-12 23:31:27.010207: Current learning rate: 0.00771 +2023-10-12 23:37:09.878119: train_loss -0.7337 +2023-10-12 23:37:09.878273: val_loss -0.6976 +2023-10-12 23:37:09.878395: Pseudo dice [0.9708, 0.8689, 0.9065, 0.8726, 0.904, 0.6857] +2023-10-12 23:37:09.878492: Epoch time: 342.87 s +2023-10-12 23:37:11.060745: +2023-10-12 23:37:11.060962: Epoch 252 +2023-10-12 23:37:11.061111: Current learning rate: 0.0077 +2023-10-12 23:42:53.884339: train_loss -0.7436 +2023-10-12 23:42:53.884486: val_loss -0.6347 +2023-10-12 23:42:53.884597: Pseudo dice [0.9734, 0.8759, 0.9109, 0.9025, 0.8514, 0.6233] +2023-10-12 23:42:53.884688: Epoch time: 342.82 s +2023-10-12 23:42:55.059313: +2023-10-12 23:42:55.059509: Epoch 253 +2023-10-12 23:42:55.059697: Current learning rate: 0.00769 +2023-10-12 23:48:37.615715: train_loss -0.756 +2023-10-12 23:48:37.615862: val_loss -0.6332 +2023-10-12 23:48:37.615976: Pseudo dice [0.9692, 0.8607, 0.8814, 0.8679, 0.8964, 0.6535] +2023-10-12 23:48:37.616059: Epoch time: 342.56 s +2023-10-12 23:48:38.792114: +2023-10-12 23:48:38.792363: Epoch 254 +2023-10-12 23:48:38.792566: Current learning rate: 0.00768 +2023-10-12 23:54:21.512851: train_loss -0.7399 +2023-10-12 23:54:21.512998: val_loss -0.693 +2023-10-12 23:54:21.513113: Pseudo dice [0.967, 0.8816, 0.9111, 0.8871, 0.8884, 0.6253] +2023-10-12 23:54:21.513206: Epoch time: 342.72 s +2023-10-12 23:54:22.875559: +2023-10-12 23:54:22.875859: Epoch 255 +2023-10-12 23:54:22.876079: Current learning rate: 0.00767 +2023-10-13 00:00:05.573528: train_loss -0.7579 +2023-10-13 00:00:05.573683: val_loss -0.6641 +2023-10-13 00:00:05.573798: Pseudo dice [0.9698, 0.8693, 0.8992, 0.8625, 0.8609, 0.6638] +2023-10-13 00:00:05.573880: Epoch time: 342.7 s +2023-10-13 00:00:06.750563: +2023-10-13 00:00:06.750767: Epoch 256 +2023-10-13 00:00:06.750926: Current learning rate: 0.00766 +2023-10-13 00:05:49.646620: train_loss -0.7539 +2023-10-13 00:05:49.646770: val_loss -0.7013 +2023-10-13 00:05:49.646878: Pseudo dice [0.9713, 0.8763, 0.8989, 0.9036, 0.9059, 0.6644] +2023-10-13 00:05:49.646962: Epoch time: 342.9 s +2023-10-13 00:05:50.823166: +2023-10-13 00:05:50.823363: Epoch 257 +2023-10-13 00:05:50.823499: Current learning rate: 0.00765 +2023-10-13 00:11:33.365096: train_loss -0.7708 +2023-10-13 00:11:33.365241: val_loss -0.6669 +2023-10-13 00:11:33.365358: Pseudo dice [0.9688, 0.8656, 0.9052, 0.8792, 0.8679, 0.6918] +2023-10-13 00:11:33.365444: Epoch time: 342.54 s +2023-10-13 00:11:34.551708: +2023-10-13 00:11:34.551819: Epoch 258 +2023-10-13 00:11:34.551930: Current learning rate: 0.00764 +2023-10-13 00:17:17.308586: train_loss -0.7344 +2023-10-13 00:17:17.308752: val_loss -0.6761 +2023-10-13 00:17:17.308855: Pseudo dice [0.9724, 0.8763, 0.8862, 0.9128, 0.8964, 0.668] +2023-10-13 00:17:17.308948: Epoch time: 342.76 s +2023-10-13 00:17:17.309016: Yayy! New best EMA pseudo Dice: 0.8616 +2023-10-13 00:17:20.287906: +2023-10-13 00:17:20.288017: Epoch 259 +2023-10-13 00:17:20.288130: Current learning rate: 0.00764 +2023-10-13 00:23:03.082340: train_loss -0.7598 +2023-10-13 00:23:03.082488: val_loss -0.6911 +2023-10-13 00:23:03.082637: Pseudo dice [0.9727, 0.8769, 0.8984, 0.887, 0.8479, 0.6623] +2023-10-13 00:23:03.082736: Epoch time: 342.8 s +2023-10-13 00:23:04.264346: +2023-10-13 00:23:04.264522: Epoch 260 +2023-10-13 00:23:04.264727: Current learning rate: 0.00763 +2023-10-13 00:28:47.149173: train_loss -0.7174 +2023-10-13 00:28:47.149310: val_loss -0.6972 +2023-10-13 00:28:47.149424: Pseudo dice [0.9686, 0.8529, 0.9113, 0.8334, 0.8656, 0.6339] +2023-10-13 00:28:47.149513: Epoch time: 342.89 s +2023-10-13 00:28:48.330021: +2023-10-13 00:28:48.330126: Epoch 261 +2023-10-13 00:28:48.330249: Current learning rate: 0.00762 +2023-10-13 00:34:31.180980: train_loss -0.75 +2023-10-13 00:34:31.181177: val_loss -0.6736 +2023-10-13 00:34:31.181282: Pseudo dice [0.9726, 0.8857, 0.8999, 0.8881, 0.9093, 0.6222] +2023-10-13 00:34:31.181366: Epoch time: 342.85 s +2023-10-13 00:34:32.540030: +2023-10-13 00:34:32.540138: Epoch 262 +2023-10-13 00:34:32.540256: Current learning rate: 0.00761 +2023-10-13 00:40:15.268363: train_loss -0.7679 +2023-10-13 00:40:15.268514: val_loss -0.6968 +2023-10-13 00:40:15.268630: Pseudo dice [0.9717, 0.8727, 0.9177, 0.9067, 0.8674, 0.6309] +2023-10-13 00:40:15.268715: Epoch time: 342.73 s +2023-10-13 00:40:16.445977: +2023-10-13 00:40:16.446153: Epoch 263 +2023-10-13 00:40:16.446325: Current learning rate: 0.0076 +2023-10-13 00:45:59.280673: train_loss -0.7574 +2023-10-13 00:45:59.280822: val_loss -0.6772 +2023-10-13 00:45:59.280936: Pseudo dice [0.9699, 0.8772, 0.9078, 0.8936, 0.8959, 0.6522] +2023-10-13 00:45:59.281027: Epoch time: 342.84 s +2023-10-13 00:46:00.466591: +2023-10-13 00:46:00.466792: Epoch 264 +2023-10-13 00:46:00.466987: Current learning rate: 0.00759 +2023-10-13 00:51:43.408388: train_loss -0.7598 +2023-10-13 00:51:43.408625: val_loss -0.7467 +2023-10-13 00:51:43.408729: Pseudo dice [0.973, 0.8676, 0.9126, 0.8819, 0.9162, 0.6664] +2023-10-13 00:51:43.408814: Epoch time: 342.94 s +2023-10-13 00:51:44.595400: +2023-10-13 00:51:44.595556: Epoch 265 +2023-10-13 00:51:44.595736: Current learning rate: 0.00758 +2023-10-13 00:57:27.297900: train_loss -0.7435 +2023-10-13 00:57:27.298048: val_loss -0.6875 +2023-10-13 00:57:27.298159: Pseudo dice [0.9717, 0.8762, 0.9032, 0.9072, 0.8998, 0.6542] +2023-10-13 00:57:27.298247: Epoch time: 342.7 s +2023-10-13 00:57:27.298320: Yayy! New best EMA pseudo Dice: 0.8622 +2023-10-13 00:57:30.165656: +2023-10-13 00:57:30.165761: Epoch 266 +2023-10-13 00:57:30.165873: Current learning rate: 0.00757 +2023-10-13 01:03:13.051394: train_loss -0.7639 +2023-10-13 01:03:13.051544: val_loss -0.6457 +2023-10-13 01:03:13.051661: Pseudo dice [0.9711, 0.8778, 0.9042, 0.8412, 0.8553, 0.6497] +2023-10-13 01:03:13.051744: Epoch time: 342.89 s +2023-10-13 01:03:14.283727: +2023-10-13 01:03:14.283917: Epoch 267 +2023-10-13 01:03:14.284082: Current learning rate: 0.00756 +2023-10-13 01:08:57.176868: train_loss -0.7395 +2023-10-13 01:08:57.177010: val_loss -0.6973 +2023-10-13 01:08:57.177123: Pseudo dice [0.9722, 0.8826, 0.8933, 0.8915, 0.9039, 0.714] +2023-10-13 01:08:57.177211: Epoch time: 342.89 s +2023-10-13 01:08:57.177283: Yayy! New best EMA pseudo Dice: 0.8625 +2023-10-13 01:09:00.175925: +2023-10-13 01:09:00.176113: Epoch 268 +2023-10-13 01:09:00.176245: Current learning rate: 0.00755 +2023-10-13 01:14:43.129569: train_loss -0.7454 +2023-10-13 01:14:43.129730: val_loss -0.6269 +2023-10-13 01:14:43.129833: Pseudo dice [0.9713, 0.8824, 0.91, 0.8952, 0.8748, 0.6981] +2023-10-13 01:14:43.129917: Epoch time: 342.95 s +2023-10-13 01:14:43.129984: Yayy! New best EMA pseudo Dice: 0.8635 +2023-10-13 01:14:46.249269: +2023-10-13 01:14:46.249463: Epoch 269 +2023-10-13 01:14:46.249633: Current learning rate: 0.00754 +2023-10-13 01:20:28.966278: train_loss -0.7597 +2023-10-13 01:20:28.966419: val_loss -0.6737 +2023-10-13 01:20:28.966563: Pseudo dice [0.9722, 0.882, 0.904, 0.8774, 0.913, 0.6273] +2023-10-13 01:20:28.966660: Epoch time: 342.72 s +2023-10-13 01:20:30.149321: +2023-10-13 01:20:30.149514: Epoch 270 +2023-10-13 01:20:30.149667: Current learning rate: 0.00753 +2023-10-13 01:26:12.882092: train_loss -0.7479 +2023-10-13 01:26:12.882256: val_loss -0.7171 +2023-10-13 01:26:12.882362: Pseudo dice [0.9588, 0.8761, 0.9085, 0.8903, 0.9074, 0.6672] +2023-10-13 01:26:12.882450: Epoch time: 342.73 s +2023-10-13 01:26:12.882526: Yayy! New best EMA pseudo Dice: 0.8638 +2023-10-13 01:26:15.815374: +2023-10-13 01:26:15.815503: Epoch 271 +2023-10-13 01:26:15.815617: Current learning rate: 0.00752 +2023-10-13 01:31:58.649895: train_loss -0.7321 +2023-10-13 01:31:58.650056: val_loss -0.6339 +2023-10-13 01:31:58.650157: Pseudo dice [0.9559, 0.8696, 0.8911, 0.8939, 0.9066, 0.6278] +2023-10-13 01:31:58.650246: Epoch time: 342.84 s +2023-10-13 01:31:59.842769: +2023-10-13 01:31:59.842889: Epoch 272 +2023-10-13 01:31:59.843006: Current learning rate: 0.00751 +2023-10-13 01:37:42.635398: train_loss -0.7296 +2023-10-13 01:37:42.635549: val_loss -0.6821 +2023-10-13 01:37:42.635665: Pseudo dice [0.9724, 0.8705, 0.8881, 0.8409, 0.8765, 0.7175] +2023-10-13 01:37:42.635750: Epoch time: 342.79 s +2023-10-13 01:37:43.823385: +2023-10-13 01:37:43.823557: Epoch 273 +2023-10-13 01:37:43.823673: Current learning rate: 0.00751 +2023-10-13 01:43:26.599983: train_loss -0.7413 +2023-10-13 01:43:26.600141: val_loss -0.6505 +2023-10-13 01:43:26.600266: Pseudo dice [0.967, 0.8676, 0.9117, 0.9127, 0.898, 0.6678] +2023-10-13 01:43:26.600361: Epoch time: 342.78 s +2023-10-13 01:43:27.777097: +2023-10-13 01:43:27.777200: Epoch 274 +2023-10-13 01:43:27.777325: Current learning rate: 0.0075 +2023-10-13 01:49:10.532862: train_loss -0.7425 +2023-10-13 01:49:10.533020: val_loss -0.6607 +2023-10-13 01:49:10.533123: Pseudo dice [0.972, 0.8673, 0.9019, 0.9024, 0.8679, 0.6063] +2023-10-13 01:49:10.533206: Epoch time: 342.76 s +2023-10-13 01:49:11.714001: +2023-10-13 01:49:11.714158: Epoch 275 +2023-10-13 01:49:11.714259: Current learning rate: 0.00749 +2023-10-13 01:54:54.382353: train_loss -0.777 +2023-10-13 01:54:54.382529: val_loss -0.676 +2023-10-13 01:54:54.382654: Pseudo dice [0.972, 0.8747, 0.9051, 0.8423, 0.8824, 0.6562] +2023-10-13 01:54:54.382750: Epoch time: 342.67 s +2023-10-13 01:54:55.742477: +2023-10-13 01:54:55.742605: Epoch 276 +2023-10-13 01:54:55.742718: Current learning rate: 0.00748 +2023-10-13 02:00:38.601842: train_loss -0.7297 +2023-10-13 02:00:38.602007: val_loss -0.6989 +2023-10-13 02:00:38.602112: Pseudo dice [0.9732, 0.8778, 0.907, 0.8546, 0.8787, 0.6548] +2023-10-13 02:00:38.602196: Epoch time: 342.86 s +2023-10-13 02:00:39.788367: +2023-10-13 02:00:39.788586: Epoch 277 +2023-10-13 02:00:39.788747: Current learning rate: 0.00747 +2023-10-13 02:06:22.732376: train_loss -0.7139 +2023-10-13 02:06:22.732536: val_loss -0.6549 +2023-10-13 02:06:22.732659: Pseudo dice [0.974, 0.8627, 0.8964, 0.8262, 0.8475, 0.6334] +2023-10-13 02:06:22.732755: Epoch time: 342.94 s +2023-10-13 02:06:23.913621: +2023-10-13 02:06:23.913753: Epoch 278 +2023-10-13 02:06:23.913880: Current learning rate: 0.00746 +2023-10-13 02:12:06.746331: train_loss -0.7224 +2023-10-13 02:12:06.746489: val_loss -0.6672 +2023-10-13 02:12:06.746601: Pseudo dice [0.9691, 0.8669, 0.8997, 0.8676, 0.8667, 0.6701] +2023-10-13 02:12:06.746682: Epoch time: 342.83 s +2023-10-13 02:12:07.953043: +2023-10-13 02:12:07.953204: Epoch 279 +2023-10-13 02:12:07.953376: Current learning rate: 0.00745 +2023-10-13 02:17:50.768334: train_loss -0.7546 +2023-10-13 02:17:50.768484: val_loss -0.7089 +2023-10-13 02:17:50.768596: Pseudo dice [0.9663, 0.8774, 0.9064, 0.8941, 0.858, 0.7179] +2023-10-13 02:17:50.768685: Epoch time: 342.82 s +2023-10-13 02:17:51.948952: +2023-10-13 02:17:51.949065: Epoch 280 +2023-10-13 02:17:51.949188: Current learning rate: 0.00744 +2023-10-13 02:23:34.574049: train_loss -0.7664 +2023-10-13 02:23:34.574208: val_loss -0.6626 +2023-10-13 02:23:34.574325: Pseudo dice [0.9703, 0.8695, 0.8952, 0.8658, 0.8657, 0.6341] +2023-10-13 02:23:34.574408: Epoch time: 342.63 s +2023-10-13 02:23:35.758125: +2023-10-13 02:23:35.758229: Epoch 281 +2023-10-13 02:23:35.758341: Current learning rate: 0.00743 +2023-10-13 02:29:18.544606: train_loss -0.7546 +2023-10-13 02:29:18.544753: val_loss -0.7332 +2023-10-13 02:29:18.544871: Pseudo dice [0.9693, 0.8695, 0.9119, 0.8424, 0.9011, 0.6939] +2023-10-13 02:29:18.544968: Epoch time: 342.79 s +2023-10-13 02:29:19.729994: +2023-10-13 02:29:19.730093: Epoch 282 +2023-10-13 02:29:19.730204: Current learning rate: 0.00742 +2023-10-13 02:35:02.503694: train_loss -0.7481 +2023-10-13 02:35:02.503840: val_loss -0.6222 +2023-10-13 02:35:02.503942: Pseudo dice [0.9662, 0.869, 0.8925, 0.8576, 0.8611, 0.6693] +2023-10-13 02:35:02.504026: Epoch time: 342.77 s +2023-10-13 02:35:03.687072: +2023-10-13 02:35:03.687258: Epoch 283 +2023-10-13 02:35:03.687398: Current learning rate: 0.00741 +2023-10-13 02:40:46.631087: train_loss -0.7476 +2023-10-13 02:40:46.631236: val_loss -0.7213 +2023-10-13 02:40:46.631353: Pseudo dice [0.9704, 0.8772, 0.9129, 0.8357, 0.8907, 0.6042] +2023-10-13 02:40:46.631436: Epoch time: 342.94 s +2023-10-13 02:40:48.002883: +2023-10-13 02:40:48.002995: Epoch 284 +2023-10-13 02:40:48.003110: Current learning rate: 0.0074 +2023-10-13 02:46:30.975469: train_loss -0.7509 +2023-10-13 02:46:30.975621: val_loss -0.6183 +2023-10-13 02:46:30.975736: Pseudo dice [0.9622, 0.8741, 0.8791, 0.8897, 0.8927, 0.703] +2023-10-13 02:46:30.975825: Epoch time: 342.97 s +2023-10-13 02:46:32.154802: +2023-10-13 02:46:32.154953: Epoch 285 +2023-10-13 02:46:32.155053: Current learning rate: 0.00739 +2023-10-13 02:52:14.994334: train_loss -0.7404 +2023-10-13 02:52:14.994507: val_loss -0.6915 +2023-10-13 02:52:14.994612: Pseudo dice [0.9697, 0.8737, 0.9195, 0.8584, 0.8862, 0.6477] +2023-10-13 02:52:14.994697: Epoch time: 342.84 s +2023-10-13 02:52:16.189037: +2023-10-13 02:52:16.189155: Epoch 286 +2023-10-13 02:52:16.189267: Current learning rate: 0.00738 +2023-10-13 02:57:58.928765: train_loss -0.7355 +2023-10-13 02:57:58.928912: val_loss -0.7118 +2023-10-13 02:57:58.929011: Pseudo dice [0.9725, 0.884, 0.894, 0.8855, 0.899, 0.6725] +2023-10-13 02:57:58.929094: Epoch time: 342.74 s +2023-10-13 02:58:00.136651: +2023-10-13 02:58:00.136766: Epoch 287 +2023-10-13 02:58:00.136878: Current learning rate: 0.00738 +2023-10-13 03:03:42.877740: train_loss -0.7531 +2023-10-13 03:03:42.877967: val_loss -0.6603 +2023-10-13 03:03:42.878101: Pseudo dice [0.9683, 0.867, 0.9035, 0.8179, 0.8582, 0.6901] +2023-10-13 03:03:42.878185: Epoch time: 342.74 s +2023-10-13 03:03:44.076132: +2023-10-13 03:03:44.076237: Epoch 288 +2023-10-13 03:03:44.076349: Current learning rate: 0.00737 +2023-10-13 03:09:26.790481: train_loss -0.7303 +2023-10-13 03:09:26.790653: val_loss -0.6712 +2023-10-13 03:09:26.790755: Pseudo dice [0.9622, 0.8689, 0.9007, 0.9144, 0.8994, 0.5951] +2023-10-13 03:09:26.790839: Epoch time: 342.72 s +2023-10-13 03:09:28.001872: +2023-10-13 03:09:28.002049: Epoch 289 +2023-10-13 03:09:28.002211: Current learning rate: 0.00736 +2023-10-13 03:15:10.536130: train_loss -0.7487 +2023-10-13 03:15:10.536295: val_loss -0.7248 +2023-10-13 03:15:10.536400: Pseudo dice [0.9693, 0.8796, 0.9057, 0.869, 0.9095, 0.655] +2023-10-13 03:15:10.536484: Epoch time: 342.53 s +2023-10-13 03:15:11.744608: +2023-10-13 03:15:11.744711: Epoch 290 +2023-10-13 03:15:11.744823: Current learning rate: 0.00735 +2023-10-13 03:20:54.497553: train_loss -0.7225 +2023-10-13 03:20:54.497700: val_loss -0.6697 +2023-10-13 03:20:54.497793: Pseudo dice [0.9709, 0.8643, 0.9135, 0.8856, 0.8718, 0.6565] +2023-10-13 03:20:54.497871: Epoch time: 342.75 s +2023-10-13 03:20:55.876689: +2023-10-13 03:20:55.876883: Epoch 291 +2023-10-13 03:20:55.877025: Current learning rate: 0.00734 +2023-10-13 03:26:38.523057: train_loss -0.7373 +2023-10-13 03:26:38.523239: val_loss -0.6649 +2023-10-13 03:26:38.523346: Pseudo dice [0.9594, 0.865, 0.913, 0.8955, 0.8739, 0.5843] +2023-10-13 03:26:38.523442: Epoch time: 342.65 s +2023-10-13 03:26:39.725153: +2023-10-13 03:26:39.725284: Epoch 292 +2023-10-13 03:26:39.725384: Current learning rate: 0.00733 +2023-10-13 03:32:22.587011: train_loss -0.7484 +2023-10-13 03:32:22.587154: val_loss -0.6855 +2023-10-13 03:32:22.587270: Pseudo dice [0.9708, 0.8651, 0.8868, 0.8577, 0.8768, 0.6551] +2023-10-13 03:32:22.587354: Epoch time: 342.86 s +2023-10-13 03:32:23.781646: +2023-10-13 03:32:23.781829: Epoch 293 +2023-10-13 03:32:23.782001: Current learning rate: 0.00732 +2023-10-13 03:38:06.601422: train_loss -0.7435 +2023-10-13 03:38:06.601596: val_loss -0.6979 +2023-10-13 03:38:06.601710: Pseudo dice [0.9724, 0.8673, 0.9107, 0.8895, 0.9178, 0.6147] +2023-10-13 03:38:06.601794: Epoch time: 342.82 s +2023-10-13 03:38:07.796560: +2023-10-13 03:38:07.796673: Epoch 294 +2023-10-13 03:38:07.796785: Current learning rate: 0.00731 +2023-10-13 03:43:50.527402: train_loss -0.7564 +2023-10-13 03:43:50.527551: val_loss -0.642 +2023-10-13 03:43:50.527669: Pseudo dice [0.968, 0.8627, 0.9025, 0.8405, 0.8976, 0.6416] +2023-10-13 03:43:50.527754: Epoch time: 342.73 s +2023-10-13 03:43:51.724544: +2023-10-13 03:43:51.724734: Epoch 295 +2023-10-13 03:43:51.724871: Current learning rate: 0.0073 +2023-10-13 03:49:34.467982: train_loss -0.7369 +2023-10-13 03:49:34.468144: val_loss -0.6601 +2023-10-13 03:49:34.468247: Pseudo dice [0.9696, 0.866, 0.9038, 0.8956, 0.8847, 0.6333] +2023-10-13 03:49:34.468332: Epoch time: 342.74 s +2023-10-13 03:49:35.675151: +2023-10-13 03:49:35.675380: Epoch 296 +2023-10-13 03:49:35.675539: Current learning rate: 0.00729 +2023-10-13 03:55:18.244521: train_loss -0.7306 +2023-10-13 03:55:18.244672: val_loss -0.6755 +2023-10-13 03:55:18.244776: Pseudo dice [0.972, 0.8751, 0.9093, 0.8861, 0.8893, 0.6317] +2023-10-13 03:55:18.244859: Epoch time: 342.57 s +2023-10-13 03:55:19.440390: +2023-10-13 03:55:19.440557: Epoch 297 +2023-10-13 03:55:19.440731: Current learning rate: 0.00728 +2023-10-13 04:01:02.018935: train_loss -0.7373 +2023-10-13 04:01:02.019101: val_loss -0.6936 +2023-10-13 04:01:02.019204: Pseudo dice [0.9724, 0.8702, 0.9076, 0.8556, 0.8508, 0.6769] +2023-10-13 04:01:02.019287: Epoch time: 342.58 s +2023-10-13 04:01:03.409831: +2023-10-13 04:01:03.410059: Epoch 298 +2023-10-13 04:01:03.410206: Current learning rate: 0.00727 +2023-10-13 04:06:46.094219: train_loss -0.7044 +2023-10-13 04:06:46.094377: val_loss -0.6615 +2023-10-13 04:06:46.094478: Pseudo dice [0.9676, 0.8534, 0.9074, 0.89, 0.8782, 0.6106] +2023-10-13 04:06:46.094570: Epoch time: 342.69 s +2023-10-13 04:06:47.294024: +2023-10-13 04:06:47.294142: Epoch 299 +2023-10-13 04:06:47.294257: Current learning rate: 0.00726 +2023-10-13 04:12:29.887120: train_loss -0.7168 +2023-10-13 04:12:29.887260: val_loss -0.6642 +2023-10-13 04:12:29.887379: Pseudo dice [0.9714, 0.8809, 0.8858, 0.8844, 0.8957, 0.702] +2023-10-13 04:12:29.887463: Epoch time: 342.59 s +2023-10-13 04:12:32.840348: +2023-10-13 04:12:32.840478: Epoch 300 +2023-10-13 04:12:32.840579: Current learning rate: 0.00725 +2023-10-13 04:18:15.555564: train_loss -0.736 +2023-10-13 04:18:15.555787: val_loss -0.6958 +2023-10-13 04:18:15.555890: Pseudo dice [0.968, 0.8852, 0.9063, 0.898, 0.8905, 0.6821] +2023-10-13 04:18:15.555975: Epoch time: 342.72 s +2023-10-13 04:18:16.760713: +2023-10-13 04:18:16.760893: Epoch 301 +2023-10-13 04:18:16.761051: Current learning rate: 0.00724 +2023-10-13 04:23:59.386629: train_loss -0.7514 +2023-10-13 04:23:59.386784: val_loss -0.6683 +2023-10-13 04:23:59.386887: Pseudo dice [0.9721, 0.8688, 0.9118, 0.882, 0.8396, 0.6624] +2023-10-13 04:23:59.386971: Epoch time: 342.63 s +2023-10-13 04:24:00.585486: +2023-10-13 04:24:00.585673: Epoch 302 +2023-10-13 04:24:00.585826: Current learning rate: 0.00724 +2023-10-13 04:29:43.338140: train_loss -0.7459 +2023-10-13 04:29:43.338288: val_loss -0.7177 +2023-10-13 04:29:43.338404: Pseudo dice [0.9694, 0.8788, 0.9154, 0.8929, 0.8822, 0.6571] +2023-10-13 04:29:43.338489: Epoch time: 342.75 s +2023-10-13 04:29:44.545983: +2023-10-13 04:29:44.546092: Epoch 303 +2023-10-13 04:29:44.546206: Current learning rate: 0.00723 +2023-10-13 04:35:27.365859: train_loss -0.7343 +2023-10-13 04:35:27.366002: val_loss -0.6662 +2023-10-13 04:35:27.366120: Pseudo dice [0.969, 0.8723, 0.9081, 0.8843, 0.8761, 0.6255] +2023-10-13 04:35:27.366205: Epoch time: 342.82 s +2023-10-13 04:35:28.568687: +2023-10-13 04:35:28.568792: Epoch 304 +2023-10-13 04:35:28.568904: Current learning rate: 0.00722 +2023-10-13 04:41:11.342215: train_loss -0.7317 +2023-10-13 04:41:11.342416: val_loss -0.6395 +2023-10-13 04:41:11.342571: Pseudo dice [0.9684, 0.8801, 0.909, 0.8772, 0.8459, 0.6482] +2023-10-13 04:41:11.342687: Epoch time: 342.77 s +2023-10-13 04:41:12.755240: +2023-10-13 04:41:12.755354: Epoch 305 +2023-10-13 04:41:12.755466: Current learning rate: 0.00721 +2023-10-13 04:46:55.476404: train_loss -0.7372 +2023-10-13 04:46:55.476553: val_loss -0.6771 +2023-10-13 04:46:55.476671: Pseudo dice [0.9736, 0.8757, 0.8849, 0.8911, 0.8651, 0.6529] +2023-10-13 04:46:55.476755: Epoch time: 342.72 s +2023-10-13 04:46:56.682459: +2023-10-13 04:46:56.682595: Epoch 306 +2023-10-13 04:46:56.682696: Current learning rate: 0.0072 +2023-10-13 04:52:39.330860: train_loss -0.7339 +2023-10-13 04:52:39.331017: val_loss -0.6782 +2023-10-13 04:52:39.331121: Pseudo dice [0.9716, 0.8796, 0.8993, 0.8478, 0.8978, 0.6088] +2023-10-13 04:52:39.331203: Epoch time: 342.65 s +2023-10-13 04:52:40.540936: +2023-10-13 04:52:40.541053: Epoch 307 +2023-10-13 04:52:40.541164: Current learning rate: 0.00719 +2023-10-13 04:58:23.366292: train_loss -0.7647 +2023-10-13 04:58:23.366457: val_loss -0.6454 +2023-10-13 04:58:23.366576: Pseudo dice [0.975, 0.8868, 0.8872, 0.8903, 0.8918, 0.6988] +2023-10-13 04:58:23.366660: Epoch time: 342.83 s +2023-10-13 04:58:24.568142: +2023-10-13 04:58:24.568244: Epoch 308 +2023-10-13 04:58:24.568356: Current learning rate: 0.00718 +2023-10-13 05:04:07.349599: train_loss -0.7698 +2023-10-13 05:04:07.349736: val_loss -0.6742 +2023-10-13 05:04:07.349833: Pseudo dice [0.9711, 0.8871, 0.8996, 0.9083, 0.901, 0.6063] +2023-10-13 05:04:07.349911: Epoch time: 342.78 s +2023-10-13 05:04:08.553393: +2023-10-13 05:04:08.553544: Epoch 309 +2023-10-13 05:04:08.553724: Current learning rate: 0.00717 +2023-10-13 05:09:51.107177: train_loss -0.7533 +2023-10-13 05:09:51.107364: val_loss -0.627 +2023-10-13 05:09:51.107466: Pseudo dice [0.9673, 0.8505, 0.8934, 0.8742, 0.8992, 0.6228] +2023-10-13 05:09:51.107550: Epoch time: 342.55 s +2023-10-13 05:09:52.307305: +2023-10-13 05:09:52.307490: Epoch 310 +2023-10-13 05:09:52.307671: Current learning rate: 0.00716 +2023-10-13 05:15:34.970317: train_loss -0.739 +2023-10-13 05:15:34.970460: val_loss -0.7365 +2023-10-13 05:15:34.970597: Pseudo dice [0.9708, 0.8755, 0.9219, 0.8848, 0.8953, 0.6741] +2023-10-13 05:15:34.970683: Epoch time: 342.66 s +2023-10-13 05:15:36.178558: +2023-10-13 05:15:36.178667: Epoch 311 +2023-10-13 05:15:36.178853: Current learning rate: 0.00715 +2023-10-13 05:21:18.822112: train_loss -0.7424 +2023-10-13 05:21:18.822255: val_loss -0.6626 +2023-10-13 05:21:18.822379: Pseudo dice [0.9696, 0.8735, 0.9169, 0.8503, 0.8544, 0.6783] +2023-10-13 05:21:18.822463: Epoch time: 342.64 s +2023-10-13 05:21:20.034964: +2023-10-13 05:21:20.035133: Epoch 312 +2023-10-13 05:21:20.035299: Current learning rate: 0.00714 +2023-10-13 05:27:02.765357: train_loss -0.7469 +2023-10-13 05:27:02.765526: val_loss -0.6932 +2023-10-13 05:27:02.765630: Pseudo dice [0.9732, 0.8846, 0.9009, 0.8641, 0.8633, 0.6637] +2023-10-13 05:27:02.765716: Epoch time: 342.73 s +2023-10-13 05:27:04.163299: +2023-10-13 05:27:04.163426: Epoch 313 +2023-10-13 05:27:04.163528: Current learning rate: 0.00713 +2023-10-13 05:32:46.799208: train_loss -0.7145 +2023-10-13 05:32:46.799362: val_loss -0.6661 +2023-10-13 05:32:46.799466: Pseudo dice [0.9668, 0.8614, 0.9055, 0.8504, 0.8948, 0.5942] +2023-10-13 05:32:46.799551: Epoch time: 342.64 s +2023-10-13 05:32:48.004393: +2023-10-13 05:32:48.004503: Epoch 314 +2023-10-13 05:32:48.004614: Current learning rate: 0.00712 +2023-10-13 05:38:30.735227: train_loss -0.7348 +2023-10-13 05:38:30.735370: val_loss -0.6993 +2023-10-13 05:38:30.735490: Pseudo dice [0.9702, 0.8654, 0.9083, 0.8481, 0.8651, 0.6796] +2023-10-13 05:38:30.735574: Epoch time: 342.73 s +2023-10-13 05:38:31.939982: +2023-10-13 05:38:31.940156: Epoch 315 +2023-10-13 05:38:31.940309: Current learning rate: 0.00711 +2023-10-13 05:44:14.646338: train_loss -0.7405 +2023-10-13 05:44:14.646600: val_loss -0.6921 +2023-10-13 05:44:14.646764: Pseudo dice [0.9676, 0.8694, 0.9037, 0.8219, 0.8897, 0.6466] +2023-10-13 05:44:14.646850: Epoch time: 342.71 s +2023-10-13 05:44:15.849050: +2023-10-13 05:44:15.849157: Epoch 316 +2023-10-13 05:44:15.849266: Current learning rate: 0.0071 +2023-10-13 05:49:58.545790: train_loss -0.7622 +2023-10-13 05:49:58.545940: val_loss -0.6677 +2023-10-13 05:49:58.546061: Pseudo dice [0.9737, 0.8838, 0.8991, 0.8999, 0.8865, 0.6827] +2023-10-13 05:49:58.546147: Epoch time: 342.7 s +2023-10-13 05:49:59.755638: +2023-10-13 05:49:59.755827: Epoch 317 +2023-10-13 05:49:59.755972: Current learning rate: 0.0071 +2023-10-13 05:55:42.524919: train_loss -0.7428 +2023-10-13 05:55:42.525082: val_loss -0.659 +2023-10-13 05:55:42.525186: Pseudo dice [0.9715, 0.8766, 0.9131, 0.8375, 0.9054, 0.6465] +2023-10-13 05:55:42.525271: Epoch time: 342.77 s +2023-10-13 05:55:43.730741: +2023-10-13 05:55:43.730881: Epoch 318 +2023-10-13 05:55:43.730981: Current learning rate: 0.00709 +2023-10-13 06:01:26.425971: train_loss -0.7435 +2023-10-13 06:01:26.426108: val_loss -0.7165 +2023-10-13 06:01:26.426226: Pseudo dice [0.9727, 0.8776, 0.8978, 0.8974, 0.8891, 0.6678] +2023-10-13 06:01:26.426309: Epoch time: 342.7 s +2023-10-13 06:01:27.628901: +2023-10-13 06:01:27.629004: Epoch 319 +2023-10-13 06:01:27.629117: Current learning rate: 0.00708 +2023-10-13 06:07:10.422142: train_loss -0.7453 +2023-10-13 06:07:10.422301: val_loss -0.6773 +2023-10-13 06:07:10.422404: Pseudo dice [0.9724, 0.884, 0.9166, 0.8516, 0.8891, 0.7025] +2023-10-13 06:07:10.422487: Epoch time: 342.79 s +2023-10-13 06:07:11.808942: +2023-10-13 06:07:11.809127: Epoch 320 +2023-10-13 06:07:11.809268: Current learning rate: 0.00707 +2023-10-13 06:12:54.670733: train_loss -0.7603 +2023-10-13 06:12:54.670895: val_loss -0.6918 +2023-10-13 06:12:54.671011: Pseudo dice [0.9714, 0.871, 0.908, 0.8755, 0.9062, 0.6898] +2023-10-13 06:12:54.671095: Epoch time: 342.86 s +2023-10-13 06:12:55.874753: +2023-10-13 06:12:55.874868: Epoch 321 +2023-10-13 06:12:55.874969: Current learning rate: 0.00706 +2023-10-13 06:18:38.645413: train_loss -0.7411 +2023-10-13 06:18:38.645571: val_loss -0.7028 +2023-10-13 06:18:38.645686: Pseudo dice [0.9682, 0.8841, 0.9007, 0.9163, 0.9058, 0.6581] +2023-10-13 06:18:38.645771: Epoch time: 342.77 s +2023-10-13 06:18:39.848779: +2023-10-13 06:18:39.848888: Epoch 322 +2023-10-13 06:18:39.849000: Current learning rate: 0.00705 +2023-10-13 06:24:22.522674: train_loss -0.7702 +2023-10-13 06:24:22.522829: val_loss -0.6853 +2023-10-13 06:24:22.522964: Pseudo dice [0.9706, 0.8774, 0.8974, 0.901, 0.8915, 0.6307] +2023-10-13 06:24:22.523048: Epoch time: 342.67 s +2023-10-13 06:24:23.747129: +2023-10-13 06:24:23.747258: Epoch 323 +2023-10-13 06:24:23.747373: Current learning rate: 0.00704 +2023-10-13 06:30:06.532027: train_loss -0.7473 +2023-10-13 06:30:06.532194: val_loss -0.7104 +2023-10-13 06:30:06.532299: Pseudo dice [0.9716, 0.8767, 0.9149, 0.8962, 0.9253, 0.6749] +2023-10-13 06:30:06.532383: Epoch time: 342.79 s +2023-10-13 06:30:07.736594: +2023-10-13 06:30:07.736712: Epoch 324 +2023-10-13 06:30:07.736811: Current learning rate: 0.00703 +2023-10-13 06:35:50.440258: train_loss -0.7711 +2023-10-13 06:35:50.440407: val_loss -0.7084 +2023-10-13 06:35:50.440542: Pseudo dice [0.9711, 0.8687, 0.9167, 0.8593, 0.8494, 0.6559] +2023-10-13 06:35:50.440629: Epoch time: 342.7 s +2023-10-13 06:35:51.692569: +2023-10-13 06:35:51.692738: Epoch 325 +2023-10-13 06:35:51.692901: Current learning rate: 0.00702 +2023-10-13 06:41:34.480199: train_loss -0.7747 +2023-10-13 06:41:34.480350: val_loss -0.709 +2023-10-13 06:41:34.480468: Pseudo dice [0.9713, 0.8847, 0.9044, 0.872, 0.9038, 0.6898] +2023-10-13 06:41:34.480565: Epoch time: 342.79 s +2023-10-13 06:41:35.689030: +2023-10-13 06:41:35.689148: Epoch 326 +2023-10-13 06:41:35.689247: Current learning rate: 0.00701 +2023-10-13 06:47:18.504876: train_loss -0.7537 +2023-10-13 06:47:18.505029: val_loss -0.6641 +2023-10-13 06:47:18.505133: Pseudo dice [0.9705, 0.8805, 0.9102, 0.8657, 0.8557, 0.672] +2023-10-13 06:47:18.505217: Epoch time: 342.82 s +2023-10-13 06:47:19.883931: +2023-10-13 06:47:19.884111: Epoch 327 +2023-10-13 06:47:19.884259: Current learning rate: 0.007 +2023-10-13 06:53:02.675400: train_loss -0.7526 +2023-10-13 06:53:02.675563: val_loss -0.6604 +2023-10-13 06:53:02.675686: Pseudo dice [0.9694, 0.8752, 0.8926, 0.9037, 0.8738, 0.6891] +2023-10-13 06:53:02.675782: Epoch time: 342.79 s +2023-10-13 06:53:03.876927: +2023-10-13 06:53:03.877036: Epoch 328 +2023-10-13 06:53:03.877142: Current learning rate: 0.00699 +2023-10-13 06:58:46.760841: train_loss -0.7385 +2023-10-13 06:58:46.760991: val_loss -0.6955 +2023-10-13 06:58:46.761111: Pseudo dice [0.9722, 0.8694, 0.9149, 0.857, 0.915, 0.652] +2023-10-13 06:58:46.761197: Epoch time: 342.88 s +2023-10-13 06:58:47.967103: +2023-10-13 06:58:47.967212: Epoch 329 +2023-10-13 06:58:47.967324: Current learning rate: 0.00698 +2023-10-13 07:04:30.720583: train_loss -0.7413 +2023-10-13 07:04:30.720810: val_loss -0.6373 +2023-10-13 07:04:30.720933: Pseudo dice [0.9728, 0.8769, 0.899, 0.8467, 0.8912, 0.6311] +2023-10-13 07:04:30.721030: Epoch time: 342.75 s +2023-10-13 07:04:31.924802: +2023-10-13 07:04:31.924931: Epoch 330 +2023-10-13 07:04:31.925043: Current learning rate: 0.00697 +2023-10-13 07:10:14.823430: train_loss -0.7659 +2023-10-13 07:10:14.823596: val_loss -0.6923 +2023-10-13 07:10:14.823720: Pseudo dice [0.973, 0.8811, 0.9068, 0.9025, 0.8592, 0.6844] +2023-10-13 07:10:14.823817: Epoch time: 342.9 s +2023-10-13 07:10:16.029216: +2023-10-13 07:10:16.029392: Epoch 331 +2023-10-13 07:10:16.029563: Current learning rate: 0.00696 +2023-10-13 07:15:58.898686: train_loss -0.7668 +2023-10-13 07:15:58.898837: val_loss -0.713 +2023-10-13 07:15:58.898959: Pseudo dice [0.9667, 0.8675, 0.9103, 0.903, 0.8823, 0.6161] +2023-10-13 07:15:58.899055: Epoch time: 342.87 s +2023-10-13 07:16:00.108662: +2023-10-13 07:16:00.108869: Epoch 332 +2023-10-13 07:16:00.109032: Current learning rate: 0.00696 +2023-10-13 07:21:42.926078: train_loss -0.7625 +2023-10-13 07:21:42.926235: val_loss -0.6631 +2023-10-13 07:21:42.926338: Pseudo dice [0.9722, 0.8893, 0.9089, 0.8621, 0.918, 0.6621] +2023-10-13 07:21:42.926423: Epoch time: 342.82 s +2023-10-13 07:21:44.139149: +2023-10-13 07:21:44.139273: Epoch 333 +2023-10-13 07:21:44.139376: Current learning rate: 0.00695 +2023-10-13 07:27:27.021345: train_loss -0.7398 +2023-10-13 07:27:27.021510: val_loss -0.6936 +2023-10-13 07:27:27.021626: Pseudo dice [0.9713, 0.8756, 0.9026, 0.8511, 0.8997, 0.6672] +2023-10-13 07:27:27.021710: Epoch time: 342.88 s +2023-10-13 07:27:28.403214: +2023-10-13 07:27:28.403325: Epoch 334 +2023-10-13 07:27:28.403437: Current learning rate: 0.00694 +2023-10-13 07:33:11.083445: train_loss -0.749 +2023-10-13 07:33:11.083593: val_loss -0.734 +2023-10-13 07:33:11.083719: Pseudo dice [0.9714, 0.8765, 0.9216, 0.8906, 0.8931, 0.6579] +2023-10-13 07:33:11.083814: Epoch time: 342.68 s +2023-10-13 07:33:12.305363: +2023-10-13 07:33:12.305588: Epoch 335 +2023-10-13 07:33:12.305778: Current learning rate: 0.00693 +2023-10-13 07:38:55.027869: train_loss -0.7535 +2023-10-13 07:38:55.028013: val_loss -0.6334 +2023-10-13 07:38:55.028132: Pseudo dice [0.9708, 0.8743, 0.8994, 0.8455, 0.8779, 0.6546] +2023-10-13 07:38:55.028216: Epoch time: 342.72 s +2023-10-13 07:38:56.251559: +2023-10-13 07:38:56.251761: Epoch 336 +2023-10-13 07:38:56.251919: Current learning rate: 0.00692 +2023-10-13 07:44:39.074110: train_loss -0.7554 +2023-10-13 07:44:39.074258: val_loss -0.6323 +2023-10-13 07:44:39.074368: Pseudo dice [0.971, 0.8622, 0.8987, 0.8798, 0.9046, 0.6835] +2023-10-13 07:44:39.074455: Epoch time: 342.82 s +2023-10-13 07:44:40.301116: +2023-10-13 07:44:40.301301: Epoch 337 +2023-10-13 07:44:40.301448: Current learning rate: 0.00691 +2023-10-13 07:50:23.136615: train_loss -0.753 +2023-10-13 07:50:23.136765: val_loss -0.6796 +2023-10-13 07:50:23.136881: Pseudo dice [0.9736, 0.8669, 0.9147, 0.8583, 0.8795, 0.6699] +2023-10-13 07:50:23.136966: Epoch time: 342.84 s +2023-10-13 07:50:24.358132: +2023-10-13 07:50:24.358323: Epoch 338 +2023-10-13 07:50:24.358484: Current learning rate: 0.0069 +2023-10-13 07:56:07.156213: train_loss -0.7591 +2023-10-13 07:56:07.156357: val_loss -0.6902 +2023-10-13 07:56:07.156477: Pseudo dice [0.9737, 0.8826, 0.9017, 0.8882, 0.8836, 0.7129] +2023-10-13 07:56:07.156568: Epoch time: 342.8 s +2023-10-13 07:56:08.378700: +2023-10-13 07:56:08.378810: Epoch 339 +2023-10-13 07:56:08.378939: Current learning rate: 0.00689 +2023-10-13 08:01:50.888207: train_loss -0.7566 +2023-10-13 08:01:50.888373: val_loss -0.685 +2023-10-13 08:01:50.888509: Pseudo dice [0.9725, 0.8738, 0.9126, 0.8977, 0.9243, 0.6536] +2023-10-13 08:01:50.888591: Epoch time: 342.51 s +2023-10-13 08:01:50.888658: Yayy! New best EMA pseudo Dice: 0.8646 +2023-10-13 08:01:53.759227: +2023-10-13 08:01:53.759410: Epoch 340 +2023-10-13 08:01:53.759567: Current learning rate: 0.00688 +2023-10-13 08:07:36.578556: train_loss -0.7548 +2023-10-13 08:07:36.578713: val_loss -0.6637 +2023-10-13 08:07:36.578815: Pseudo dice [0.9723, 0.8649, 0.9076, 0.8931, 0.8827, 0.6534] +2023-10-13 08:07:36.578899: Epoch time: 342.82 s +2023-10-13 08:07:37.978545: +2023-10-13 08:07:37.978741: Epoch 341 +2023-10-13 08:07:37.978909: Current learning rate: 0.00687 +2023-10-13 08:13:20.731017: train_loss -0.7358 +2023-10-13 08:13:20.731158: val_loss -0.6664 +2023-10-13 08:13:20.731272: Pseudo dice [0.9674, 0.8653, 0.8886, 0.8913, 0.9163, 0.693] +2023-10-13 08:13:20.731363: Epoch time: 342.75 s +2023-10-13 08:13:20.731435: Yayy! New best EMA pseudo Dice: 0.865 +2023-10-13 08:13:23.587703: +2023-10-13 08:13:23.587888: Epoch 342 +2023-10-13 08:13:23.588025: Current learning rate: 0.00686 +2023-10-13 08:19:06.395730: train_loss -0.7368 +2023-10-13 08:19:06.395891: val_loss -0.7099 +2023-10-13 08:19:06.395996: Pseudo dice [0.9624, 0.8793, 0.9142, 0.8844, 0.8645, 0.6145] +2023-10-13 08:19:06.396080: Epoch time: 342.81 s +2023-10-13 08:19:07.622273: +2023-10-13 08:19:07.622466: Epoch 343 +2023-10-13 08:19:07.622766: Current learning rate: 0.00685 +2023-10-13 08:24:50.519843: train_loss -0.719 +2023-10-13 08:24:50.519991: val_loss -0.7061 +2023-10-13 08:24:50.520103: Pseudo dice [0.9717, 0.8806, 0.9017, 0.8699, 0.8994, 0.6685] +2023-10-13 08:24:50.520192: Epoch time: 342.9 s +2023-10-13 08:24:51.745181: +2023-10-13 08:24:51.745287: Epoch 344 +2023-10-13 08:24:51.745399: Current learning rate: 0.00684 +2023-10-13 08:30:34.607709: train_loss -0.7325 +2023-10-13 08:30:34.607844: val_loss -0.7241 +2023-10-13 08:30:34.607958: Pseudo dice [0.9687, 0.8705, 0.9088, 0.9158, 0.8505, 0.6806] +2023-10-13 08:30:34.608042: Epoch time: 342.86 s +2023-10-13 08:30:35.828925: +2023-10-13 08:30:35.829032: Epoch 345 +2023-10-13 08:30:35.829144: Current learning rate: 0.00683 +2023-10-13 08:36:18.751522: train_loss -0.7372 +2023-10-13 08:36:18.751680: val_loss -0.645 +2023-10-13 08:36:18.751803: Pseudo dice [0.9713, 0.8629, 0.8963, 0.8197, 0.812, 0.6796] +2023-10-13 08:36:18.751899: Epoch time: 342.92 s +2023-10-13 08:36:19.975366: +2023-10-13 08:36:19.975542: Epoch 346 +2023-10-13 08:36:19.975731: Current learning rate: 0.00682 +2023-10-13 08:42:02.818171: train_loss -0.7617 +2023-10-13 08:42:02.818320: val_loss -0.6875 +2023-10-13 08:42:02.818438: Pseudo dice [0.9697, 0.8688, 0.9127, 0.8996, 0.8678, 0.6461] +2023-10-13 08:42:02.818541: Epoch time: 342.84 s +2023-10-13 08:42:04.210964: +2023-10-13 08:42:04.211075: Epoch 347 +2023-10-13 08:42:04.211187: Current learning rate: 0.00681 +2023-10-13 08:47:46.793569: train_loss -0.7623 +2023-10-13 08:47:46.793715: val_loss -0.6815 +2023-10-13 08:47:46.793857: Pseudo dice [0.9721, 0.8821, 0.8886, 0.8914, 0.8962, 0.6808] +2023-10-13 08:47:46.793954: Epoch time: 342.58 s +2023-10-13 08:47:48.017481: +2023-10-13 08:47:48.017597: Epoch 348 +2023-10-13 08:47:48.017720: Current learning rate: 0.0068 +2023-10-13 08:53:30.848833: train_loss -0.7563 +2023-10-13 08:53:30.848997: val_loss -0.6532 +2023-10-13 08:53:30.849121: Pseudo dice [0.9681, 0.872, 0.9094, 0.8439, 0.902, 0.6815] +2023-10-13 08:53:30.849216: Epoch time: 342.83 s +2023-10-13 08:53:32.088167: +2023-10-13 08:53:32.088367: Epoch 349 +2023-10-13 08:53:32.088507: Current learning rate: 0.0068 +2023-10-13 08:59:14.870995: train_loss -0.7626 +2023-10-13 08:59:14.871145: val_loss -0.6412 +2023-10-13 08:59:14.871260: Pseudo dice [0.9725, 0.8733, 0.8961, 0.9071, 0.8992, 0.631] +2023-10-13 08:59:14.871351: Epoch time: 342.78 s +2023-10-13 08:59:17.821899: +2023-10-13 08:59:17.822125: Epoch 350 +2023-10-13 08:59:17.822244: Current learning rate: 0.00679 +2023-10-13 09:05:00.703714: train_loss -0.7379 +2023-10-13 09:05:00.703862: val_loss -0.6495 +2023-10-13 09:05:00.703979: Pseudo dice [0.9663, 0.8838, 0.8965, 0.8572, 0.892, 0.668] +2023-10-13 09:05:00.704063: Epoch time: 342.88 s +2023-10-13 09:05:01.926329: +2023-10-13 09:05:01.926435: Epoch 351 +2023-10-13 09:05:01.926552: Current learning rate: 0.00678 +2023-10-13 09:10:44.849550: train_loss -0.7464 +2023-10-13 09:10:44.849712: val_loss -0.6288 +2023-10-13 09:10:44.849816: Pseudo dice [0.9735, 0.8844, 0.8872, 0.7979, 0.8902, 0.7047] +2023-10-13 09:10:44.849901: Epoch time: 342.92 s +2023-10-13 09:10:46.073590: +2023-10-13 09:10:46.073782: Epoch 352 +2023-10-13 09:10:46.073946: Current learning rate: 0.00677 +2023-10-13 09:16:28.773375: train_loss -0.7291 +2023-10-13 09:16:28.773539: val_loss -0.7238 +2023-10-13 09:16:28.773643: Pseudo dice [0.9748, 0.8841, 0.9079, 0.9127, 0.9098, 0.6588] +2023-10-13 09:16:28.773727: Epoch time: 342.7 s +2023-10-13 09:16:30.000275: +2023-10-13 09:16:30.000375: Epoch 353 +2023-10-13 09:16:30.000488: Current learning rate: 0.00676 +2023-10-13 09:22:12.835063: train_loss -0.7341 +2023-10-13 09:22:12.835224: val_loss -0.6651 +2023-10-13 09:22:12.835326: Pseudo dice [0.9689, 0.8745, 0.9043, 0.8879, 0.8586, 0.645] +2023-10-13 09:22:12.835409: Epoch time: 342.84 s +2023-10-13 09:22:14.249982: +2023-10-13 09:22:14.250162: Epoch 354 +2023-10-13 09:22:14.250326: Current learning rate: 0.00675 +2023-10-13 09:27:56.951790: train_loss -0.7457 +2023-10-13 09:27:56.951931: val_loss -0.7114 +2023-10-13 09:27:56.952044: Pseudo dice [0.9714, 0.8804, 0.9035, 0.8498, 0.9165, 0.6566] +2023-10-13 09:27:56.952132: Epoch time: 342.7 s +2023-10-13 09:27:58.180476: +2023-10-13 09:27:58.180664: Epoch 355 +2023-10-13 09:27:58.180825: Current learning rate: 0.00674 +2023-10-13 09:33:41.083723: train_loss -0.7374 +2023-10-13 09:33:41.083873: val_loss -0.7173 +2023-10-13 09:33:41.083991: Pseudo dice [0.9719, 0.8646, 0.907, 0.9007, 0.8978, 0.6816] +2023-10-13 09:33:41.084075: Epoch time: 342.9 s +2023-10-13 09:33:42.304666: +2023-10-13 09:33:42.304782: Epoch 356 +2023-10-13 09:33:42.304898: Current learning rate: 0.00673 +2023-10-13 09:39:25.171038: train_loss -0.7465 +2023-10-13 09:39:25.171180: val_loss -0.6502 +2023-10-13 09:39:25.171298: Pseudo dice [0.97, 0.8772, 0.8836, 0.828, 0.8594, 0.694] +2023-10-13 09:39:25.171381: Epoch time: 342.87 s +2023-10-13 09:39:26.397131: +2023-10-13 09:39:26.397243: Epoch 357 +2023-10-13 09:39:26.397366: Current learning rate: 0.00672 +2023-10-13 09:45:09.353044: train_loss -0.7621 +2023-10-13 09:45:09.353192: val_loss -0.679 +2023-10-13 09:45:09.353302: Pseudo dice [0.9682, 0.8732, 0.9042, 0.9113, 0.9097, 0.6302] +2023-10-13 09:45:09.353391: Epoch time: 342.96 s +2023-10-13 09:45:10.573556: +2023-10-13 09:45:10.573673: Epoch 358 +2023-10-13 09:45:10.573799: Current learning rate: 0.00671 +2023-10-13 09:50:53.417343: train_loss -0.7353 +2023-10-13 09:50:53.417527: val_loss -0.6621 +2023-10-13 09:50:53.417646: Pseudo dice [0.9721, 0.879, 0.9131, 0.8437, 0.9175, 0.6781] +2023-10-13 09:50:53.417730: Epoch time: 342.84 s +2023-10-13 09:50:54.643150: +2023-10-13 09:50:54.643253: Epoch 359 +2023-10-13 09:50:54.643364: Current learning rate: 0.0067 +2023-10-13 09:56:37.436021: train_loss -0.736 +2023-10-13 09:56:37.436242: val_loss -0.6367 +2023-10-13 09:56:37.436364: Pseudo dice [0.9672, 0.8654, 0.9077, 0.9024, 0.8793, 0.6135] +2023-10-13 09:56:37.436460: Epoch time: 342.79 s +2023-10-13 09:56:38.664633: +2023-10-13 09:56:38.664739: Epoch 360 +2023-10-13 09:56:38.664907: Current learning rate: 0.00669 +2023-10-13 10:02:21.492487: train_loss -0.7609 +2023-10-13 10:02:21.492675: val_loss -0.7082 +2023-10-13 10:02:21.492809: Pseudo dice [0.9724, 0.8813, 0.9146, 0.8457, 0.8928, 0.6547] +2023-10-13 10:02:21.492904: Epoch time: 342.83 s +2023-10-13 10:02:22.906086: +2023-10-13 10:02:22.906280: Epoch 361 +2023-10-13 10:02:22.906416: Current learning rate: 0.00668 +2023-10-13 10:08:05.626436: train_loss -0.7355 +2023-10-13 10:08:05.626613: val_loss -0.6951 +2023-10-13 10:08:05.626737: Pseudo dice [0.969, 0.8858, 0.904, 0.8668, 0.9091, 0.7035] +2023-10-13 10:08:05.626848: Epoch time: 342.72 s +2023-10-13 10:08:06.857670: +2023-10-13 10:08:06.857877: Epoch 362 +2023-10-13 10:08:06.858057: Current learning rate: 0.00667 +2023-10-13 10:13:49.670724: train_loss -0.7642 +2023-10-13 10:13:49.670910: val_loss -0.6718 +2023-10-13 10:13:49.671093: Pseudo dice [0.9708, 0.8831, 0.8875, 0.865, 0.878, 0.6643] +2023-10-13 10:13:49.671190: Epoch time: 342.81 s +2023-10-13 10:13:50.911498: +2023-10-13 10:13:50.911702: Epoch 363 +2023-10-13 10:13:50.911888: Current learning rate: 0.00666 +2023-10-13 10:19:33.623294: train_loss -0.73 +2023-10-13 10:19:33.623444: val_loss -0.6587 +2023-10-13 10:19:33.623557: Pseudo dice [0.9682, 0.86, 0.8963, 0.8616, 0.8648, 0.6337] +2023-10-13 10:19:33.623645: Epoch time: 342.71 s +2023-10-13 10:19:34.853762: +2023-10-13 10:19:34.854069: Epoch 364 +2023-10-13 10:19:34.854307: Current learning rate: 0.00665 +2023-10-13 10:25:17.645166: train_loss -0.7228 +2023-10-13 10:25:17.645328: val_loss -0.625 +2023-10-13 10:25:17.645451: Pseudo dice [0.9685, 0.876, 0.8907, 0.8932, 0.9253, 0.6701] +2023-10-13 10:25:17.645547: Epoch time: 342.79 s +2023-10-13 10:25:18.879182: +2023-10-13 10:25:18.879290: Epoch 365 +2023-10-13 10:25:18.879414: Current learning rate: 0.00665 +2023-10-13 10:31:01.593807: train_loss -0.7617 +2023-10-13 10:31:01.593957: val_loss -0.6498 +2023-10-13 10:31:01.594075: Pseudo dice [0.9723, 0.8722, 0.8961, 0.8729, 0.8453, 0.6812] +2023-10-13 10:31:01.594160: Epoch time: 342.72 s +2023-10-13 10:31:02.835676: +2023-10-13 10:31:02.835782: Epoch 366 +2023-10-13 10:31:02.835894: Current learning rate: 0.00664 +2023-10-13 10:36:45.743730: train_loss -0.7569 +2023-10-13 10:36:45.743867: val_loss -0.6839 +2023-10-13 10:36:45.744000: Pseudo dice [0.97, 0.8904, 0.9052, 0.8681, 0.878, 0.6393] +2023-10-13 10:36:45.744097: Epoch time: 342.91 s +2023-10-13 10:36:46.983547: +2023-10-13 10:36:46.983735: Epoch 367 +2023-10-13 10:36:46.983885: Current learning rate: 0.00663 +2023-10-13 10:42:29.860356: train_loss -0.7543 +2023-10-13 10:42:29.860499: val_loss -0.6626 +2023-10-13 10:42:29.860613: Pseudo dice [0.9671, 0.8815, 0.9085, 0.8819, 0.901, 0.6692] +2023-10-13 10:42:29.860697: Epoch time: 342.88 s +2023-10-13 10:42:31.281739: +2023-10-13 10:42:31.281854: Epoch 368 +2023-10-13 10:42:31.281964: Current learning rate: 0.00662 +2023-10-13 10:48:14.051269: train_loss -0.7502 +2023-10-13 10:48:14.051418: val_loss -0.7007 +2023-10-13 10:48:14.051535: Pseudo dice [0.97, 0.8768, 0.8956, 0.9001, 0.9111, 0.6758] +2023-10-13 10:48:14.051630: Epoch time: 342.77 s +2023-10-13 10:48:15.278221: +2023-10-13 10:48:15.278436: Epoch 369 +2023-10-13 10:48:15.278720: Current learning rate: 0.00661 +2023-10-13 10:53:58.198438: train_loss -0.7591 +2023-10-13 10:53:58.198605: val_loss -0.6769 +2023-10-13 10:53:58.198709: Pseudo dice [0.9711, 0.8846, 0.904, 0.9105, 0.8936, 0.6326] +2023-10-13 10:53:58.198793: Epoch time: 342.92 s +2023-10-13 10:53:59.424129: +2023-10-13 10:53:59.424317: Epoch 370 +2023-10-13 10:53:59.424467: Current learning rate: 0.0066 +2023-10-13 10:59:42.269051: train_loss -0.7857 +2023-10-13 10:59:42.269222: val_loss -0.6615 +2023-10-13 10:59:42.269357: Pseudo dice [0.972, 0.8782, 0.8893, 0.8865, 0.8814, 0.7246] +2023-10-13 10:59:42.269453: Epoch time: 342.85 s +2023-10-13 10:59:43.515711: +2023-10-13 10:59:43.515920: Epoch 371 +2023-10-13 10:59:43.516091: Current learning rate: 0.00659 +2023-10-13 11:05:26.315012: train_loss -0.7534 +2023-10-13 11:05:26.315163: val_loss -0.6789 +2023-10-13 11:05:26.315279: Pseudo dice [0.9719, 0.8797, 0.906, 0.8699, 0.9019, 0.6657] +2023-10-13 11:05:26.315363: Epoch time: 342.8 s +2023-10-13 11:05:27.542766: +2023-10-13 11:05:27.542961: Epoch 372 +2023-10-13 11:05:27.543124: Current learning rate: 0.00658 +2023-10-13 11:11:10.310141: train_loss -0.7607 +2023-10-13 11:11:10.310343: val_loss -0.6823 +2023-10-13 11:11:10.310492: Pseudo dice [0.9699, 0.883, 0.902, 0.852, 0.9086, 0.6282] +2023-10-13 11:11:10.310644: Epoch time: 342.77 s +2023-10-13 11:11:11.551667: +2023-10-13 11:11:11.551787: Epoch 373 +2023-10-13 11:11:11.551887: Current learning rate: 0.00657 +2023-10-13 11:16:54.456971: train_loss -0.7754 +2023-10-13 11:16:54.457132: val_loss -0.674 +2023-10-13 11:16:54.457269: Pseudo dice [0.9695, 0.8764, 0.9144, 0.8716, 0.8903, 0.6937] +2023-10-13 11:16:54.457375: Epoch time: 342.91 s +2023-10-13 11:16:55.705124: +2023-10-13 11:16:55.705228: Epoch 374 +2023-10-13 11:16:55.705339: Current learning rate: 0.00656 +2023-10-13 11:22:38.463376: train_loss -0.7764 +2023-10-13 11:22:38.463516: val_loss -0.6566 +2023-10-13 11:22:38.463632: Pseudo dice [0.9721, 0.8669, 0.9077, 0.8744, 0.9123, 0.6751] +2023-10-13 11:22:38.463717: Epoch time: 342.76 s +2023-10-13 11:22:39.870850: +2023-10-13 11:22:39.870960: Epoch 375 +2023-10-13 11:22:39.871075: Current learning rate: 0.00655 +2023-10-13 11:28:22.674581: train_loss -0.7466 +2023-10-13 11:28:22.674728: val_loss -0.6696 +2023-10-13 11:28:22.674837: Pseudo dice [0.9672, 0.8796, 0.9069, 0.8454, 0.8807, 0.6659] +2023-10-13 11:28:22.674921: Epoch time: 342.8 s +2023-10-13 11:28:23.905317: +2023-10-13 11:28:23.905434: Epoch 376 +2023-10-13 11:28:23.905543: Current learning rate: 0.00654 +2023-10-13 11:34:06.451731: train_loss -0.7616 +2023-10-13 11:34:06.451875: val_loss -0.6488 +2023-10-13 11:34:06.452008: Pseudo dice [0.9698, 0.8743, 0.8952, 0.8788, 0.8773, 0.6594] +2023-10-13 11:34:06.452103: Epoch time: 342.55 s +2023-10-13 11:34:07.683011: +2023-10-13 11:34:07.683279: Epoch 377 +2023-10-13 11:34:07.683488: Current learning rate: 0.00653 +2023-10-13 11:39:50.297529: train_loss -0.7606 +2023-10-13 11:39:50.297697: val_loss -0.6786 +2023-10-13 11:39:50.297801: Pseudo dice [0.9685, 0.8811, 0.9142, 0.8953, 0.8846, 0.65] +2023-10-13 11:39:50.297885: Epoch time: 342.62 s +2023-10-13 11:39:51.523275: +2023-10-13 11:39:51.523509: Epoch 378 +2023-10-13 11:39:51.523612: Current learning rate: 0.00652 +2023-10-13 11:45:34.181892: train_loss -0.7456 +2023-10-13 11:45:34.182048: val_loss -0.6642 +2023-10-13 11:45:34.182171: Pseudo dice [0.9679, 0.8873, 0.9009, 0.8528, 0.8643, 0.6757] +2023-10-13 11:45:34.182266: Epoch time: 342.66 s +2023-10-13 11:45:35.407383: +2023-10-13 11:45:35.407599: Epoch 379 +2023-10-13 11:45:35.407703: Current learning rate: 0.00651 +2023-10-13 11:51:18.228338: train_loss -0.7232 +2023-10-13 11:51:18.228493: val_loss -0.6278 +2023-10-13 11:51:18.228595: Pseudo dice [0.9674, 0.8553, 0.8918, 0.8803, 0.8856, 0.6181] +2023-10-13 11:51:18.228679: Epoch time: 342.82 s +2023-10-13 11:51:19.465345: +2023-10-13 11:51:19.465507: Epoch 380 +2023-10-13 11:51:19.465676: Current learning rate: 0.0065 +2023-10-13 11:57:02.320043: train_loss -0.7308 +2023-10-13 11:57:02.320190: val_loss -0.6755 +2023-10-13 11:57:02.320304: Pseudo dice [0.962, 0.8689, 0.9041, 0.9162, 0.8704, 0.6313] +2023-10-13 11:57:02.320395: Epoch time: 342.86 s +2023-10-13 11:57:03.554756: +2023-10-13 11:57:03.554942: Epoch 381 +2023-10-13 11:57:03.555102: Current learning rate: 0.00649 +2023-10-13 12:02:46.290900: train_loss -0.7269 +2023-10-13 12:02:46.291070: val_loss -0.6793 +2023-10-13 12:02:46.291173: Pseudo dice [0.9687, 0.8678, 0.9048, 0.8913, 0.888, 0.628] +2023-10-13 12:02:46.291257: Epoch time: 342.74 s +2023-10-13 12:02:47.738464: +2023-10-13 12:02:47.738601: Epoch 382 +2023-10-13 12:02:47.738703: Current learning rate: 0.00648 +2023-10-13 12:08:30.418318: train_loss -0.7422 +2023-10-13 12:08:30.418461: val_loss -0.6916 +2023-10-13 12:08:30.418588: Pseudo dice [0.965, 0.8682, 0.9022, 0.8265, 0.9008, 0.6145] +2023-10-13 12:08:30.418675: Epoch time: 342.68 s +2023-10-13 12:08:31.673867: +2023-10-13 12:08:31.673989: Epoch 383 +2023-10-13 12:08:31.674112: Current learning rate: 0.00648 +2023-10-13 12:14:14.289157: train_loss -0.7363 +2023-10-13 12:14:14.289294: val_loss -0.6626 +2023-10-13 12:14:14.289411: Pseudo dice [0.9693, 0.8594, 0.8872, 0.8761, 0.8774, 0.6359] +2023-10-13 12:14:14.289495: Epoch time: 342.62 s +2023-10-13 12:14:15.539223: +2023-10-13 12:14:15.539419: Epoch 384 +2023-10-13 12:14:15.539569: Current learning rate: 0.00647 +2023-10-13 12:19:58.509317: train_loss -0.7557 +2023-10-13 12:19:58.509482: val_loss -0.6911 +2023-10-13 12:19:58.509585: Pseudo dice [0.9725, 0.8655, 0.9037, 0.8542, 0.8794, 0.6584] +2023-10-13 12:19:58.509669: Epoch time: 342.97 s +2023-10-13 12:19:59.752321: +2023-10-13 12:19:59.752510: Epoch 385 +2023-10-13 12:19:59.752650: Current learning rate: 0.00646 +2023-10-13 12:25:42.565106: train_loss -0.7602 +2023-10-13 12:25:42.565254: val_loss -0.6788 +2023-10-13 12:25:42.565369: Pseudo dice [0.9697, 0.8746, 0.9149, 0.8946, 0.9059, 0.6151] +2023-10-13 12:25:42.565451: Epoch time: 342.81 s +2023-10-13 12:25:43.809049: +2023-10-13 12:25:43.809233: Epoch 386 +2023-10-13 12:25:43.809394: Current learning rate: 0.00645 +2023-10-13 12:31:26.623498: train_loss -0.7562 +2023-10-13 12:31:26.623664: val_loss -0.6699 +2023-10-13 12:31:26.623767: Pseudo dice [0.972, 0.8831, 0.9124, 0.8915, 0.8766, 0.6746] +2023-10-13 12:31:26.623851: Epoch time: 342.82 s +2023-10-13 12:31:27.867989: +2023-10-13 12:31:27.868093: Epoch 387 +2023-10-13 12:31:27.868204: Current learning rate: 0.00644 +2023-10-13 12:37:10.768650: train_loss -0.7546 +2023-10-13 12:37:10.768793: val_loss -0.7091 +2023-10-13 12:37:10.768920: Pseudo dice [0.9729, 0.8697, 0.8916, 0.8987, 0.8828, 0.6666] +2023-10-13 12:37:10.769003: Epoch time: 342.9 s +2023-10-13 12:37:12.016447: +2023-10-13 12:37:12.016616: Epoch 388 +2023-10-13 12:37:12.016768: Current learning rate: 0.00643 +2023-10-13 12:42:54.677792: train_loss -0.7573 +2023-10-13 12:42:54.677926: val_loss -0.7103 +2023-10-13 12:42:54.678045: Pseudo dice [0.9724, 0.8747, 0.9202, 0.9089, 0.8825, 0.6337] +2023-10-13 12:42:54.678130: Epoch time: 342.66 s +2023-10-13 12:42:56.110414: +2023-10-13 12:42:56.110641: Epoch 389 +2023-10-13 12:42:56.110762: Current learning rate: 0.00642 +2023-10-13 12:48:38.686204: train_loss -0.7459 +2023-10-13 12:48:38.686366: val_loss -0.7102 +2023-10-13 12:48:38.686489: Pseudo dice [0.9705, 0.8652, 0.9164, 0.8813, 0.8946, 0.6942] +2023-10-13 12:48:38.686593: Epoch time: 342.58 s +2023-10-13 12:48:39.943308: +2023-10-13 12:48:39.943503: Epoch 390 +2023-10-13 12:48:39.943681: Current learning rate: 0.00641 +2023-10-13 12:54:22.584442: train_loss -0.7443 +2023-10-13 12:54:22.584591: val_loss -0.6742 +2023-10-13 12:54:22.584709: Pseudo dice [0.9723, 0.8761, 0.9028, 0.851, 0.8527, 0.695] +2023-10-13 12:54:22.584793: Epoch time: 342.64 s +2023-10-13 12:54:23.835523: +2023-10-13 12:54:23.835636: Epoch 391 +2023-10-13 12:54:23.835747: Current learning rate: 0.0064 +2023-10-13 13:00:06.607536: train_loss -0.742 +2023-10-13 13:00:06.607684: val_loss -0.7048 +2023-10-13 13:00:06.607799: Pseudo dice [0.9708, 0.8621, 0.9199, 0.916, 0.866, 0.6378] +2023-10-13 13:00:06.607882: Epoch time: 342.77 s +2023-10-13 13:00:07.862641: +2023-10-13 13:00:07.862763: Epoch 392 +2023-10-13 13:00:07.862881: Current learning rate: 0.00639 +2023-10-13 13:05:50.420409: train_loss -0.7715 +2023-10-13 13:05:50.420549: val_loss -0.6885 +2023-10-13 13:05:50.420667: Pseudo dice [0.9675, 0.8768, 0.9109, 0.8479, 0.8951, 0.6581] +2023-10-13 13:05:50.420759: Epoch time: 342.56 s +2023-10-13 13:05:51.666748: +2023-10-13 13:05:51.666934: Epoch 393 +2023-10-13 13:05:51.667100: Current learning rate: 0.00638 +2023-10-13 13:11:34.427865: train_loss -0.729 +2023-10-13 13:11:34.428021: val_loss -0.6881 +2023-10-13 13:11:34.428125: Pseudo dice [0.9704, 0.8805, 0.9044, 0.8819, 0.8957, 0.6637] +2023-10-13 13:11:34.428208: Epoch time: 342.76 s +2023-10-13 13:11:35.674872: +2023-10-13 13:11:35.674978: Epoch 394 +2023-10-13 13:11:35.675089: Current learning rate: 0.00637 +2023-10-13 13:17:18.327990: train_loss -0.7686 +2023-10-13 13:17:18.328137: val_loss -0.6414 +2023-10-13 13:17:18.328255: Pseudo dice [0.9695, 0.8785, 0.904, 0.8722, 0.8945, 0.7043] +2023-10-13 13:17:18.328341: Epoch time: 342.65 s +2023-10-13 13:17:19.590950: +2023-10-13 13:17:19.591052: Epoch 395 +2023-10-13 13:17:19.591163: Current learning rate: 0.00636 +2023-10-13 13:23:02.260173: train_loss -0.7483 +2023-10-13 13:23:02.260355: val_loss -0.7063 +2023-10-13 13:23:02.260469: Pseudo dice [0.9705, 0.871, 0.9137, 0.9007, 0.8698, 0.6509] +2023-10-13 13:23:02.260558: Epoch time: 342.67 s +2023-10-13 13:23:03.507803: +2023-10-13 13:23:03.508046: Epoch 396 +2023-10-13 13:23:03.508235: Current learning rate: 0.00635 +2023-10-13 13:28:46.147573: train_loss -0.7259 +2023-10-13 13:28:46.147714: val_loss -0.6801 +2023-10-13 13:28:46.147827: Pseudo dice [0.9711, 0.8735, 0.9064, 0.8521, 0.9022, 0.6234] +2023-10-13 13:28:46.147915: Epoch time: 342.64 s +2023-10-13 13:28:47.393064: +2023-10-13 13:28:47.393289: Epoch 397 +2023-10-13 13:28:47.393400: Current learning rate: 0.00634 +2023-10-13 13:34:29.984709: train_loss -0.7551 +2023-10-13 13:34:29.984851: val_loss -0.6447 +2023-10-13 13:34:29.984962: Pseudo dice [0.9742, 0.8856, 0.8951, 0.882, 0.9052, 0.6664] +2023-10-13 13:34:29.985051: Epoch time: 342.59 s +2023-10-13 13:34:31.231583: +2023-10-13 13:34:31.231791: Epoch 398 +2023-10-13 13:34:31.231943: Current learning rate: 0.00633 +2023-10-13 13:40:13.994481: train_loss -0.7592 +2023-10-13 13:40:13.994658: val_loss -0.6394 +2023-10-13 13:40:13.994778: Pseudo dice [0.9709, 0.8651, 0.8999, 0.8999, 0.8937, 0.6361] +2023-10-13 13:40:13.994875: Epoch time: 342.76 s +2023-10-13 13:40:15.254033: +2023-10-13 13:40:15.254143: Epoch 399 +2023-10-13 13:40:15.254255: Current learning rate: 0.00632 +2023-10-13 13:45:58.157813: train_loss -0.7399 +2023-10-13 13:45:58.157954: val_loss -0.6704 +2023-10-13 13:45:58.158082: Pseudo dice [0.9712, 0.8686, 0.9052, 0.8617, 0.9115, 0.6525] +2023-10-13 13:45:58.158168: Epoch time: 342.9 s +2023-10-13 13:46:01.277431: +2023-10-13 13:46:01.277548: Epoch 400 +2023-10-13 13:46:01.277647: Current learning rate: 0.00631 +2023-10-13 13:51:43.936476: train_loss -0.755 +2023-10-13 13:51:43.936633: val_loss -0.6576 +2023-10-13 13:51:43.936769: Pseudo dice [0.972, 0.8694, 0.8926, 0.8839, 0.8729, 0.6522] +2023-10-13 13:51:43.936876: Epoch time: 342.66 s +2023-10-13 13:51:45.215942: +2023-10-13 13:51:45.216043: Epoch 401 +2023-10-13 13:51:45.216155: Current learning rate: 0.0063 +2023-10-13 13:57:27.964722: train_loss -0.7388 +2023-10-13 13:57:27.964861: val_loss -0.6852 +2023-10-13 13:57:27.964978: Pseudo dice [0.967, 0.8762, 0.9024, 0.8708, 0.901, 0.6127] +2023-10-13 13:57:27.965074: Epoch time: 342.75 s +2023-10-13 13:57:29.385847: +2023-10-13 13:57:29.386070: Epoch 402 +2023-10-13 13:57:29.386194: Current learning rate: 0.0063 +2023-10-13 14:03:12.040042: train_loss -0.7658 +2023-10-13 14:03:12.040190: val_loss -0.6281 +2023-10-13 14:03:12.040305: Pseudo dice [0.9726, 0.8654, 0.8863, 0.8374, 0.8885, 0.6144] +2023-10-13 14:03:12.040389: Epoch time: 342.65 s +2023-10-13 14:03:13.297179: +2023-10-13 14:03:13.297292: Epoch 403 +2023-10-13 14:03:13.297414: Current learning rate: 0.00629 +2023-10-13 14:08:55.974898: train_loss -0.7867 +2023-10-13 14:08:55.975067: val_loss -0.6667 +2023-10-13 14:08:55.975191: Pseudo dice [0.9716, 0.8807, 0.8968, 0.8509, 0.9009, 0.6348] +2023-10-13 14:08:55.975286: Epoch time: 342.68 s +2023-10-13 14:08:57.222117: +2023-10-13 14:08:57.222410: Epoch 404 +2023-10-13 14:08:57.222611: Current learning rate: 0.00628 +2023-10-13 14:14:40.005064: train_loss -0.7664 +2023-10-13 14:14:40.005205: val_loss -0.6633 +2023-10-13 14:14:40.005333: Pseudo dice [0.9688, 0.8687, 0.9032, 0.8997, 0.8741, 0.6302] +2023-10-13 14:14:40.005418: Epoch time: 342.78 s +2023-10-13 14:14:41.250716: +2023-10-13 14:14:41.250832: Epoch 405 +2023-10-13 14:14:41.250934: Current learning rate: 0.00627 +2023-10-13 14:20:24.001407: train_loss -0.7428 +2023-10-13 14:20:24.001607: val_loss -0.6756 +2023-10-13 14:20:24.001721: Pseudo dice [0.973, 0.8636, 0.9081, 0.805, 0.8655, 0.6881] +2023-10-13 14:20:24.001811: Epoch time: 342.75 s +2023-10-13 14:20:25.247285: +2023-10-13 14:20:25.247483: Epoch 406 +2023-10-13 14:20:25.247638: Current learning rate: 0.00626 +2023-10-13 14:26:07.983951: train_loss -0.7498 +2023-10-13 14:26:07.984094: val_loss -0.7148 +2023-10-13 14:26:07.984207: Pseudo dice [0.9728, 0.8779, 0.9109, 0.8712, 0.912, 0.6918] +2023-10-13 14:26:07.984297: Epoch time: 342.74 s +2023-10-13 14:26:09.228207: +2023-10-13 14:26:09.228380: Epoch 407 +2023-10-13 14:26:09.228526: Current learning rate: 0.00625 +2023-10-13 14:31:51.952741: train_loss -0.7538 +2023-10-13 14:31:51.952882: val_loss -0.6324 +2023-10-13 14:31:51.953007: Pseudo dice [0.9692, 0.8621, 0.9109, 0.8598, 0.8962, 0.6201] +2023-10-13 14:31:51.953091: Epoch time: 342.73 s +2023-10-13 14:31:53.216341: +2023-10-13 14:31:53.216521: Epoch 408 +2023-10-13 14:31:53.216698: Current learning rate: 0.00624 +2023-10-13 14:37:35.985859: train_loss -0.7346 +2023-10-13 14:37:35.986026: val_loss -0.7107 +2023-10-13 14:37:35.986128: Pseudo dice [0.9702, 0.881, 0.8964, 0.8651, 0.8921, 0.6962] +2023-10-13 14:37:35.986223: Epoch time: 342.77 s +2023-10-13 14:37:37.411854: +2023-10-13 14:37:37.412193: Epoch 409 +2023-10-13 14:37:37.412359: Current learning rate: 0.00623 +2023-10-13 14:43:20.164193: train_loss -0.7708 +2023-10-13 14:43:20.164340: val_loss -0.6804 +2023-10-13 14:43:20.164443: Pseudo dice [0.9716, 0.8673, 0.9081, 0.8753, 0.8788, 0.6651] +2023-10-13 14:43:20.164526: Epoch time: 342.75 s +2023-10-13 14:43:21.406313: +2023-10-13 14:43:21.406502: Epoch 410 +2023-10-13 14:43:21.406615: Current learning rate: 0.00622 +2023-10-13 14:49:04.295796: train_loss -0.7445 +2023-10-13 14:49:04.295943: val_loss -0.6915 +2023-10-13 14:49:04.296063: Pseudo dice [0.9731, 0.8786, 0.9074, 0.8799, 0.8823, 0.656] +2023-10-13 14:49:04.296146: Epoch time: 342.89 s +2023-10-13 14:49:05.460550: +2023-10-13 14:49:05.460657: Epoch 411 +2023-10-13 14:49:05.460771: Current learning rate: 0.00621 +2023-10-13 14:54:48.076678: train_loss -0.7716 +2023-10-13 14:54:48.076844: val_loss -0.6707 +2023-10-13 14:54:48.076948: Pseudo dice [0.9745, 0.8783, 0.9123, 0.9078, 0.8781, 0.6448] +2023-10-13 14:54:48.077034: Epoch time: 342.62 s +2023-10-13 14:54:49.243381: +2023-10-13 14:54:49.243556: Epoch 412 +2023-10-13 14:54:49.243716: Current learning rate: 0.0062 +2023-10-13 15:00:31.975599: train_loss -0.7491 +2023-10-13 15:00:31.975738: val_loss -0.6451 +2023-10-13 15:00:31.975862: Pseudo dice [0.9697, 0.873, 0.9096, 0.8632, 0.8885, 0.6248] +2023-10-13 15:00:31.975945: Epoch time: 342.73 s +2023-10-13 15:00:33.148459: +2023-10-13 15:00:33.148565: Epoch 413 +2023-10-13 15:00:33.148675: Current learning rate: 0.00619 +2023-10-13 15:06:15.969322: train_loss -0.7485 +2023-10-13 15:06:15.969558: val_loss -0.6824 +2023-10-13 15:06:15.969682: Pseudo dice [0.9668, 0.869, 0.9079, 0.8888, 0.8871, 0.6042] +2023-10-13 15:06:15.969793: Epoch time: 342.82 s +2023-10-13 15:06:17.136460: +2023-10-13 15:06:17.136641: Epoch 414 +2023-10-13 15:06:17.136833: Current learning rate: 0.00618 +2023-10-13 15:11:59.986307: train_loss -0.7497 +2023-10-13 15:11:59.986465: val_loss -0.6673 +2023-10-13 15:11:59.986597: Pseudo dice [0.9682, 0.8773, 0.8901, 0.9108, 0.9116, 0.6329] +2023-10-13 15:11:59.986693: Epoch time: 342.85 s +2023-10-13 15:12:01.156659: +2023-10-13 15:12:01.156836: Epoch 415 +2023-10-13 15:12:01.156980: Current learning rate: 0.00617 +2023-10-13 15:17:43.966484: train_loss -0.7298 +2023-10-13 15:17:43.966656: val_loss -0.6995 +2023-10-13 15:17:43.966758: Pseudo dice [0.9738, 0.8718, 0.9008, 0.8666, 0.8699, 0.6308] +2023-10-13 15:17:43.966853: Epoch time: 342.81 s +2023-10-13 15:17:45.318417: +2023-10-13 15:17:45.318549: Epoch 416 +2023-10-13 15:17:45.318660: Current learning rate: 0.00616 +2023-10-13 15:23:28.197032: train_loss -0.7345 +2023-10-13 15:23:28.197174: val_loss -0.7366 +2023-10-13 15:23:28.197290: Pseudo dice [0.9725, 0.876, 0.9048, 0.8991, 0.9297, 0.6231] +2023-10-13 15:23:28.197384: Epoch time: 342.88 s +2023-10-13 15:23:29.370346: +2023-10-13 15:23:29.370537: Epoch 417 +2023-10-13 15:23:29.370691: Current learning rate: 0.00615 +2023-10-13 15:29:12.248099: train_loss -0.7433 +2023-10-13 15:29:12.248266: val_loss -0.7002 +2023-10-13 15:29:12.248370: Pseudo dice [0.972, 0.8681, 0.9009, 0.8825, 0.8366, 0.6716] +2023-10-13 15:29:12.248452: Epoch time: 342.88 s +2023-10-13 15:29:13.420288: +2023-10-13 15:29:13.420393: Epoch 418 +2023-10-13 15:29:13.420505: Current learning rate: 0.00614 +2023-10-13 15:34:56.174927: train_loss -0.7607 +2023-10-13 15:34:56.175067: val_loss -0.6807 +2023-10-13 15:34:56.175179: Pseudo dice [0.9714, 0.8685, 0.9065, 0.8956, 0.8833, 0.6615] +2023-10-13 15:34:56.175269: Epoch time: 342.76 s +2023-10-13 15:34:57.347947: +2023-10-13 15:34:57.348145: Epoch 419 +2023-10-13 15:34:57.348334: Current learning rate: 0.00613 +2023-10-13 15:40:40.071080: train_loss -0.7613 +2023-10-13 15:40:40.071230: val_loss -0.6919 +2023-10-13 15:40:40.071347: Pseudo dice [0.972, 0.8728, 0.8988, 0.9101, 0.8967, 0.6794] +2023-10-13 15:40:40.071442: Epoch time: 342.72 s +2023-10-13 15:40:41.250981: +2023-10-13 15:40:41.251085: Epoch 420 +2023-10-13 15:40:41.251196: Current learning rate: 0.00612 +2023-10-13 15:46:24.057930: train_loss -0.7545 +2023-10-13 15:46:24.058085: val_loss -0.6872 +2023-10-13 15:46:24.058189: Pseudo dice [0.9741, 0.8789, 0.9045, 0.8499, 0.8642, 0.6477] +2023-10-13 15:46:24.058273: Epoch time: 342.81 s +2023-10-13 15:46:25.253415: +2023-10-13 15:46:25.253575: Epoch 421 +2023-10-13 15:46:25.253745: Current learning rate: 0.00612 +2023-10-13 15:52:08.052398: train_loss -0.7261 +2023-10-13 15:52:08.052547: val_loss -0.6757 +2023-10-13 15:52:08.052664: Pseudo dice [0.9715, 0.8703, 0.9008, 0.9019, 0.9167, 0.6897] +2023-10-13 15:52:08.052748: Epoch time: 342.8 s +2023-10-13 15:52:09.225643: +2023-10-13 15:52:09.225841: Epoch 422 +2023-10-13 15:52:09.226028: Current learning rate: 0.00611 +2023-10-13 15:57:51.930464: train_loss -0.7582 +2023-10-13 15:57:51.930621: val_loss -0.707 +2023-10-13 15:57:51.930722: Pseudo dice [0.9711, 0.8721, 0.9215, 0.8923, 0.892, 0.6681] +2023-10-13 15:57:51.930804: Epoch time: 342.71 s +2023-10-13 15:57:53.287061: +2023-10-13 15:57:53.287248: Epoch 423 +2023-10-13 15:57:53.287450: Current learning rate: 0.0061 +2023-10-13 16:03:35.955413: train_loss -0.7723 +2023-10-13 16:03:35.955556: val_loss -0.6496 +2023-10-13 16:03:35.955671: Pseudo dice [0.9735, 0.877, 0.8937, 0.9184, 0.8933, 0.7041] +2023-10-13 16:03:35.955761: Epoch time: 342.67 s +2023-10-13 16:03:37.133963: +2023-10-13 16:03:37.134152: Epoch 424 +2023-10-13 16:03:37.134305: Current learning rate: 0.00609 +2023-10-13 16:09:19.964826: train_loss -0.7681 +2023-10-13 16:09:19.964990: val_loss -0.6877 +2023-10-13 16:09:19.965094: Pseudo dice [0.9701, 0.868, 0.8989, 0.8599, 0.8669, 0.6391] +2023-10-13 16:09:19.965178: Epoch time: 342.83 s +2023-10-13 16:09:21.135122: +2023-10-13 16:09:21.135229: Epoch 425 +2023-10-13 16:09:21.135340: Current learning rate: 0.00608 +2023-10-13 16:15:03.882759: train_loss -0.7473 +2023-10-13 16:15:03.882925: val_loss -0.669 +2023-10-13 16:15:03.883054: Pseudo dice [0.9685, 0.8795, 0.8991, 0.8611, 0.8749, 0.7028] +2023-10-13 16:15:03.883148: Epoch time: 342.75 s +2023-10-13 16:15:05.059929: +2023-10-13 16:15:05.060053: Epoch 426 +2023-10-13 16:15:05.060165: Current learning rate: 0.00607 +2023-10-13 16:20:47.885742: train_loss -0.7493 +2023-10-13 16:20:47.885908: val_loss -0.6674 +2023-10-13 16:20:47.886011: Pseudo dice [0.9732, 0.8814, 0.9005, 0.8782, 0.901, 0.6804] +2023-10-13 16:20:47.886109: Epoch time: 342.83 s +2023-10-13 16:20:49.057310: +2023-10-13 16:20:49.057413: Epoch 427 +2023-10-13 16:20:49.057534: Current learning rate: 0.00606 +2023-10-13 16:26:31.784241: train_loss -0.7607 +2023-10-13 16:26:31.784377: val_loss -0.6651 +2023-10-13 16:26:31.784494: Pseudo dice [0.9726, 0.8743, 0.8941, 0.84, 0.8802, 0.664] +2023-10-13 16:26:31.784577: Epoch time: 342.73 s +2023-10-13 16:26:32.965942: +2023-10-13 16:26:32.966044: Epoch 428 +2023-10-13 16:26:32.966155: Current learning rate: 0.00605 +2023-10-13 16:32:15.715254: train_loss -0.7443 +2023-10-13 16:32:15.715415: val_loss -0.6677 +2023-10-13 16:32:15.715518: Pseudo dice [0.9735, 0.8778, 0.9053, 0.8547, 0.8924, 0.6611] +2023-10-13 16:32:15.715601: Epoch time: 342.75 s +2023-10-13 16:32:16.886466: +2023-10-13 16:32:16.886681: Epoch 429 +2023-10-13 16:32:16.886813: Current learning rate: 0.00604 +2023-10-13 16:37:59.592430: train_loss -0.7545 +2023-10-13 16:37:59.592670: val_loss -0.6988 +2023-10-13 16:37:59.592825: Pseudo dice [0.9711, 0.8695, 0.9189, 0.8286, 0.8931, 0.6093] +2023-10-13 16:37:59.592922: Epoch time: 342.71 s +2023-10-13 16:38:00.946768: +2023-10-13 16:38:00.946994: Epoch 430 +2023-10-13 16:38:00.947109: Current learning rate: 0.00603 +2023-10-13 16:43:43.808078: train_loss -0.7701 +2023-10-13 16:43:43.808226: val_loss -0.6743 +2023-10-13 16:43:43.808335: Pseudo dice [0.9736, 0.873, 0.9072, 0.8529, 0.8837, 0.6836] +2023-10-13 16:43:43.808443: Epoch time: 342.86 s +2023-10-13 16:43:44.988612: +2023-10-13 16:43:44.988728: Epoch 431 +2023-10-13 16:43:44.988839: Current learning rate: 0.00602 +2023-10-13 16:49:27.797865: train_loss -0.7335 +2023-10-13 16:49:27.798011: val_loss -0.7101 +2023-10-13 16:49:27.798137: Pseudo dice [0.9642, 0.873, 0.9064, 0.8745, 0.8839, 0.6515] +2023-10-13 16:49:27.798219: Epoch time: 342.81 s +2023-10-13 16:49:28.973822: +2023-10-13 16:49:28.974009: Epoch 432 +2023-10-13 16:49:28.974174: Current learning rate: 0.00601 +2023-10-13 16:55:11.869621: train_loss -0.7401 +2023-10-13 16:55:11.869769: val_loss -0.7228 +2023-10-13 16:55:11.869895: Pseudo dice [0.9729, 0.8782, 0.9109, 0.925, 0.8884, 0.6679] +2023-10-13 16:55:11.869990: Epoch time: 342.9 s +2023-10-13 16:55:13.045411: +2023-10-13 16:55:13.045605: Epoch 433 +2023-10-13 16:55:13.045800: Current learning rate: 0.006 +2023-10-13 17:00:55.898628: train_loss -0.7747 +2023-10-13 17:00:55.898777: val_loss -0.6868 +2023-10-13 17:00:55.898909: Pseudo dice [0.9726, 0.8711, 0.9088, 0.9074, 0.8575, 0.6399] +2023-10-13 17:00:55.898999: Epoch time: 342.85 s +2023-10-13 17:00:57.071712: +2023-10-13 17:00:57.071828: Epoch 434 +2023-10-13 17:00:57.071942: Current learning rate: 0.00599 +2023-10-13 17:06:39.969159: train_loss -0.7426 +2023-10-13 17:06:39.969315: val_loss -0.6345 +2023-10-13 17:06:39.969418: Pseudo dice [0.9719, 0.8592, 0.8933, 0.868, 0.8688, 0.6199] +2023-10-13 17:06:39.969501: Epoch time: 342.9 s +2023-10-13 17:06:41.140115: +2023-10-13 17:06:41.140305: Epoch 435 +2023-10-13 17:06:41.140452: Current learning rate: 0.00598 +2023-10-13 17:12:23.886024: train_loss -0.762 +2023-10-13 17:12:23.886189: val_loss -0.6759 +2023-10-13 17:12:23.886293: Pseudo dice [0.9738, 0.8831, 0.8963, 0.856, 0.902, 0.6371] +2023-10-13 17:12:23.886377: Epoch time: 342.75 s +2023-10-13 17:12:25.060276: +2023-10-13 17:12:25.060377: Epoch 436 +2023-10-13 17:12:25.060488: Current learning rate: 0.00597 +2023-10-13 17:18:07.857692: train_loss -0.7535 +2023-10-13 17:18:07.857839: val_loss -0.7187 +2023-10-13 17:18:07.857957: Pseudo dice [0.9743, 0.8824, 0.9035, 0.8876, 0.8956, 0.6568] +2023-10-13 17:18:07.858041: Epoch time: 342.8 s +2023-10-13 17:18:09.028383: +2023-10-13 17:18:09.028500: Epoch 437 +2023-10-13 17:18:09.028599: Current learning rate: 0.00596 +2023-10-13 17:23:51.819252: train_loss -0.7376 +2023-10-13 17:23:51.819398: val_loss -0.7031 +2023-10-13 17:23:51.819514: Pseudo dice [0.9658, 0.8701, 0.907, 0.8811, 0.9248, 0.7042] +2023-10-13 17:23:51.819596: Epoch time: 342.79 s +2023-10-13 17:23:53.182742: +2023-10-13 17:23:53.182955: Epoch 438 +2023-10-13 17:23:53.183058: Current learning rate: 0.00595 +2023-10-13 17:29:35.927444: train_loss -0.7302 +2023-10-13 17:29:35.927587: val_loss -0.6663 +2023-10-13 17:29:35.927711: Pseudo dice [0.9667, 0.8863, 0.881, 0.9118, 0.8714, 0.6386] +2023-10-13 17:29:35.927801: Epoch time: 342.75 s +2023-10-13 17:29:37.096911: +2023-10-13 17:29:37.097029: Epoch 439 +2023-10-13 17:29:37.097151: Current learning rate: 0.00594 +2023-10-13 17:35:19.830430: train_loss -0.7412 +2023-10-13 17:35:19.830601: val_loss -0.6855 +2023-10-13 17:35:19.830706: Pseudo dice [0.9707, 0.8867, 0.9158, 0.8838, 0.895, 0.6457] +2023-10-13 17:35:19.830799: Epoch time: 342.73 s +2023-10-13 17:35:21.002531: +2023-10-13 17:35:21.002655: Epoch 440 +2023-10-13 17:35:21.002758: Current learning rate: 0.00593 +2023-10-13 17:41:03.793797: train_loss -0.7609 +2023-10-13 17:41:03.793942: val_loss -0.6712 +2023-10-13 17:41:03.794059: Pseudo dice [0.9721, 0.8752, 0.8936, 0.8756, 0.9225, 0.6912] +2023-10-13 17:41:03.794142: Epoch time: 342.79 s +2023-10-13 17:41:04.964851: +2023-10-13 17:41:04.965032: Epoch 441 +2023-10-13 17:41:04.965202: Current learning rate: 0.00592 +2023-10-13 17:46:47.729725: train_loss -0.7625 +2023-10-13 17:46:47.729876: val_loss -0.6793 +2023-10-13 17:46:47.729994: Pseudo dice [0.9712, 0.8723, 0.9031, 0.8908, 0.8912, 0.5794] +2023-10-13 17:46:47.730077: Epoch time: 342.77 s +2023-10-13 17:46:48.903386: +2023-10-13 17:46:48.903491: Epoch 442 +2023-10-13 17:46:48.903601: Current learning rate: 0.00592 +2023-10-13 17:52:31.704711: train_loss -0.7356 +2023-10-13 17:52:31.704853: val_loss -0.6639 +2023-10-13 17:52:31.704969: Pseudo dice [0.9698, 0.8626, 0.9134, 0.8074, 0.888, 0.6407] +2023-10-13 17:52:31.705051: Epoch time: 342.8 s +2023-10-13 17:52:32.874976: +2023-10-13 17:52:32.875191: Epoch 443 +2023-10-13 17:52:32.875294: Current learning rate: 0.00591 +2023-10-13 17:58:15.753271: train_loss -0.7616 +2023-10-13 17:58:15.753438: val_loss -0.6925 +2023-10-13 17:58:15.753592: Pseudo dice [0.9739, 0.8744, 0.9116, 0.8998, 0.8724, 0.695] +2023-10-13 17:58:15.753681: Epoch time: 342.88 s +2023-10-13 17:58:16.970631: +2023-10-13 17:58:16.970751: Epoch 444 +2023-10-13 17:58:16.970875: Current learning rate: 0.0059 +2023-10-13 18:03:59.703957: train_loss -0.7636 +2023-10-13 18:03:59.704101: val_loss -0.6847 +2023-10-13 18:03:59.704218: Pseudo dice [0.9685, 0.881, 0.9022, 0.86, 0.8862, 0.6563] +2023-10-13 18:03:59.704300: Epoch time: 342.73 s +2023-10-13 18:04:01.048369: +2023-10-13 18:04:01.048558: Epoch 445 +2023-10-13 18:04:01.048755: Current learning rate: 0.00589 +2023-10-13 18:09:43.771955: train_loss -0.7492 +2023-10-13 18:09:43.772103: val_loss -0.718 +2023-10-13 18:09:43.772228: Pseudo dice [0.9714, 0.8562, 0.9161, 0.8783, 0.877, 0.5928] +2023-10-13 18:09:43.772313: Epoch time: 342.72 s +2023-10-13 18:09:44.945360: +2023-10-13 18:09:44.945503: Epoch 446 +2023-10-13 18:09:44.945620: Current learning rate: 0.00588 +2023-10-13 18:15:27.694736: train_loss -0.7396 +2023-10-13 18:15:27.694901: val_loss -0.641 +2023-10-13 18:15:27.695011: Pseudo dice [0.9665, 0.8747, 0.8941, 0.8646, 0.8954, 0.6029] +2023-10-13 18:15:27.695101: Epoch time: 342.75 s +2023-10-13 18:15:28.855468: +2023-10-13 18:15:28.855695: Epoch 447 +2023-10-13 18:15:28.855853: Current learning rate: 0.00587 +2023-10-13 18:21:11.615659: train_loss -0.7497 +2023-10-13 18:21:11.615804: val_loss -0.6943 +2023-10-13 18:21:11.615919: Pseudo dice [0.9711, 0.8651, 0.9118, 0.8665, 0.8606, 0.6264] +2023-10-13 18:21:11.616008: Epoch time: 342.76 s +2023-10-13 18:21:12.778821: +2023-10-13 18:21:12.778952: Epoch 448 +2023-10-13 18:21:12.779075: Current learning rate: 0.00586 +2023-10-13 18:26:55.530255: train_loss -0.7447 +2023-10-13 18:26:55.530404: val_loss -0.7066 +2023-10-13 18:26:55.530544: Pseudo dice [0.9704, 0.8727, 0.909, 0.8799, 0.8848, 0.6557] +2023-10-13 18:26:55.530642: Epoch time: 342.75 s +2023-10-13 18:26:56.693327: +2023-10-13 18:26:56.693487: Epoch 449 +2023-10-13 18:26:56.693654: Current learning rate: 0.00585 +2023-10-13 18:32:39.448441: train_loss -0.7495 +2023-10-13 18:32:39.448600: val_loss -0.657 +2023-10-13 18:32:39.448726: Pseudo dice [0.9699, 0.8645, 0.9037, 0.8562, 0.8572, 0.6291] +2023-10-13 18:32:39.448826: Epoch time: 342.76 s +2023-10-13 18:32:42.440600: +2023-10-13 18:32:42.440784: Epoch 450 +2023-10-13 18:32:42.440928: Current learning rate: 0.00584 +2023-10-13 18:38:25.046258: train_loss -0.7579 +2023-10-13 18:38:25.046408: val_loss -0.6639 +2023-10-13 18:38:25.046534: Pseudo dice [0.9722, 0.8805, 0.9161, 0.8917, 0.8876, 0.6673] +2023-10-13 18:38:25.046621: Epoch time: 342.61 s +2023-10-13 18:38:26.208077: +2023-10-13 18:38:26.208263: Epoch 451 +2023-10-13 18:38:26.208423: Current learning rate: 0.00583 +2023-10-13 18:44:08.909490: train_loss -0.7751 +2023-10-13 18:44:08.909630: val_loss -0.6768 +2023-10-13 18:44:08.909748: Pseudo dice [0.9715, 0.8777, 0.9101, 0.8502, 0.8797, 0.621] +2023-10-13 18:44:08.909833: Epoch time: 342.7 s +2023-10-13 18:44:10.249463: +2023-10-13 18:44:10.249584: Epoch 452 +2023-10-13 18:44:10.249699: Current learning rate: 0.00582 +2023-10-13 18:49:52.915139: train_loss -0.7486 +2023-10-13 18:49:52.915301: val_loss -0.7172 +2023-10-13 18:49:52.915406: Pseudo dice [0.9645, 0.8796, 0.8872, 0.8877, 0.8947, 0.6715] +2023-10-13 18:49:52.915500: Epoch time: 342.67 s +2023-10-13 18:49:54.076556: +2023-10-13 18:49:54.076818: Epoch 453 +2023-10-13 18:49:54.076920: Current learning rate: 0.00581 +2023-10-13 18:55:36.958684: train_loss -0.7558 +2023-10-13 18:55:36.958827: val_loss -0.7048 +2023-10-13 18:55:36.958930: Pseudo dice [0.9717, 0.8765, 0.9147, 0.8948, 0.8786, 0.6475] +2023-10-13 18:55:36.959020: Epoch time: 342.88 s +2023-10-13 18:55:38.119600: +2023-10-13 18:55:38.119715: Epoch 454 +2023-10-13 18:55:38.119827: Current learning rate: 0.0058 +2023-10-13 19:01:20.933187: train_loss -0.748 +2023-10-13 19:01:20.933328: val_loss -0.6289 +2023-10-13 19:01:20.933423: Pseudo dice [0.97, 0.8813, 0.8867, 0.8645, 0.8991, 0.6903] +2023-10-13 19:01:20.933503: Epoch time: 342.81 s +2023-10-13 19:01:22.091761: +2023-10-13 19:01:22.091951: Epoch 455 +2023-10-13 19:01:22.092113: Current learning rate: 0.00579 +2023-10-13 19:07:04.794304: train_loss -0.7546 +2023-10-13 19:07:04.794450: val_loss -0.666 +2023-10-13 19:07:04.794575: Pseudo dice [0.9676, 0.8698, 0.9092, 0.8601, 0.8842, 0.6724] +2023-10-13 19:07:04.794659: Epoch time: 342.7 s +2023-10-13 19:07:05.953352: +2023-10-13 19:07:05.953534: Epoch 456 +2023-10-13 19:07:05.953698: Current learning rate: 0.00578 +2023-10-13 19:12:48.610779: train_loss -0.7761 +2023-10-13 19:12:48.610946: val_loss -0.7298 +2023-10-13 19:12:48.611063: Pseudo dice [0.97, 0.8721, 0.9066, 0.8569, 0.9234, 0.6628] +2023-10-13 19:12:48.611147: Epoch time: 342.66 s +2023-10-13 19:12:49.772390: +2023-10-13 19:12:49.772576: Epoch 457 +2023-10-13 19:12:49.772738: Current learning rate: 0.00577 +2023-10-13 19:18:32.609794: train_loss -0.7732 +2023-10-13 19:18:32.609944: val_loss -0.6519 +2023-10-13 19:18:32.610059: Pseudo dice [0.972, 0.8812, 0.9206, 0.8889, 0.8861, 0.6825] +2023-10-13 19:18:32.610142: Epoch time: 342.84 s +2023-10-13 19:18:33.774019: +2023-10-13 19:18:33.774194: Epoch 458 +2023-10-13 19:18:33.774367: Current learning rate: 0.00576 +2023-10-13 19:24:16.603532: train_loss -0.7591 +2023-10-13 19:24:16.603678: val_loss -0.6789 +2023-10-13 19:24:16.603794: Pseudo dice [0.9712, 0.8814, 0.8979, 0.9075, 0.9187, 0.7064] +2023-10-13 19:24:16.603877: Epoch time: 342.83 s +2023-10-13 19:24:17.762957: +2023-10-13 19:24:17.763190: Epoch 459 +2023-10-13 19:24:17.763353: Current learning rate: 0.00575 +2023-10-13 19:30:00.482012: train_loss -0.7721 +2023-10-13 19:30:00.482173: val_loss -0.6753 +2023-10-13 19:30:00.482275: Pseudo dice [0.972, 0.879, 0.9036, 0.8577, 0.9135, 0.6368] +2023-10-13 19:30:00.482358: Epoch time: 342.72 s +2023-10-13 19:30:01.833045: +2023-10-13 19:30:01.833174: Epoch 460 +2023-10-13 19:30:01.833277: Current learning rate: 0.00574 +2023-10-13 19:35:44.535309: train_loss -0.7418 +2023-10-13 19:35:44.535475: val_loss -0.6078 +2023-10-13 19:35:44.535578: Pseudo dice [0.9742, 0.8755, 0.8937, 0.8513, 0.8862, 0.637] +2023-10-13 19:35:44.535661: Epoch time: 342.7 s +2023-10-13 19:35:45.696593: +2023-10-13 19:35:45.696810: Epoch 461 +2023-10-13 19:35:45.696944: Current learning rate: 0.00573 +2023-10-13 19:41:28.547642: train_loss -0.7554 +2023-10-13 19:41:28.547792: val_loss -0.6928 +2023-10-13 19:41:28.547888: Pseudo dice [0.971, 0.8896, 0.9141, 0.9076, 0.8718, 0.6924] +2023-10-13 19:41:28.547968: Epoch time: 342.85 s +2023-10-13 19:41:29.705675: +2023-10-13 19:41:29.705791: Epoch 462 +2023-10-13 19:41:29.705903: Current learning rate: 0.00572 +2023-10-13 19:47:12.449514: train_loss -0.748 +2023-10-13 19:47:12.449673: val_loss -0.6745 +2023-10-13 19:47:12.449776: Pseudo dice [0.9722, 0.8698, 0.9154, 0.8507, 0.9007, 0.6325] +2023-10-13 19:47:12.449861: Epoch time: 342.74 s +2023-10-13 19:47:13.607915: +2023-10-13 19:47:13.608028: Epoch 463 +2023-10-13 19:47:13.608144: Current learning rate: 0.00571 +2023-10-13 19:52:56.289461: train_loss -0.7545 +2023-10-13 19:52:56.289626: val_loss -0.7028 +2023-10-13 19:52:56.289727: Pseudo dice [0.9749, 0.8767, 0.912, 0.8487, 0.8794, 0.6761] +2023-10-13 19:52:56.289812: Epoch time: 342.68 s +2023-10-13 19:52:57.450308: +2023-10-13 19:52:57.450432: Epoch 464 +2023-10-13 19:52:57.450555: Current learning rate: 0.0057 +2023-10-13 19:58:40.196074: train_loss -0.7779 +2023-10-13 19:58:40.196231: val_loss -0.6248 +2023-10-13 19:58:40.196333: Pseudo dice [0.9694, 0.8819, 0.9015, 0.8697, 0.9099, 0.6983] +2023-10-13 19:58:40.196417: Epoch time: 342.75 s +2023-10-13 19:58:41.359899: +2023-10-13 19:58:41.360005: Epoch 465 +2023-10-13 19:58:41.360115: Current learning rate: 0.0057 +2023-10-13 20:04:23.969077: train_loss -0.7588 +2023-10-13 20:04:23.969218: val_loss -0.7017 +2023-10-13 20:04:23.969334: Pseudo dice [0.971, 0.8681, 0.9137, 0.9086, 0.8925, 0.6924] +2023-10-13 20:04:23.969418: Epoch time: 342.61 s +2023-10-13 20:04:25.129140: +2023-10-13 20:04:25.129261: Epoch 466 +2023-10-13 20:04:25.129360: Current learning rate: 0.00569 +2023-10-13 20:10:07.914138: train_loss -0.7658 +2023-10-13 20:10:07.914296: val_loss -0.6489 +2023-10-13 20:10:07.914398: Pseudo dice [0.9703, 0.8743, 0.9117, 0.8401, 0.8564, 0.6811] +2023-10-13 20:10:07.914482: Epoch time: 342.79 s +2023-10-13 20:10:09.074698: +2023-10-13 20:10:09.074803: Epoch 467 +2023-10-13 20:10:09.074913: Current learning rate: 0.00568 +2023-10-13 20:15:51.578480: train_loss -0.7725 +2023-10-13 20:15:51.578667: val_loss -0.6811 +2023-10-13 20:15:51.578769: Pseudo dice [0.9706, 0.8852, 0.913, 0.921, 0.9011, 0.6703] +2023-10-13 20:15:51.578851: Epoch time: 342.5 s +2023-10-13 20:15:52.735017: +2023-10-13 20:15:52.735136: Epoch 468 +2023-10-13 20:15:52.735251: Current learning rate: 0.00567 +2023-10-13 20:21:35.416423: train_loss -0.7626 +2023-10-13 20:21:35.416635: val_loss -0.701 +2023-10-13 20:21:35.416776: Pseudo dice [0.973, 0.8809, 0.9086, 0.8909, 0.8903, 0.6712] +2023-10-13 20:21:35.416905: Epoch time: 342.68 s +2023-10-13 20:21:35.416973: Yayy! New best EMA pseudo Dice: 0.8654 +2023-10-13 20:21:38.550738: +2023-10-13 20:21:38.550939: Epoch 469 +2023-10-13 20:21:38.551105: Current learning rate: 0.00566 +2023-10-13 20:27:21.251365: train_loss -0.7573 +2023-10-13 20:27:21.251513: val_loss -0.6736 +2023-10-13 20:27:21.251629: Pseudo dice [0.9662, 0.8797, 0.8966, 0.8711, 0.8947, 0.6931] +2023-10-13 20:27:21.251712: Epoch time: 342.7 s +2023-10-13 20:27:21.251779: Yayy! New best EMA pseudo Dice: 0.8655 +2023-10-13 20:27:24.257592: +2023-10-13 20:27:24.257707: Epoch 470 +2023-10-13 20:27:24.257821: Current learning rate: 0.00565 +2023-10-13 20:33:07.044611: train_loss -0.7644 +2023-10-13 20:33:07.044759: val_loss -0.6748 +2023-10-13 20:33:07.044876: Pseudo dice [0.9699, 0.874, 0.9141, 0.8602, 0.8863, 0.626] +2023-10-13 20:33:07.044961: Epoch time: 342.79 s +2023-10-13 20:33:08.216286: +2023-10-13 20:33:08.216450: Epoch 471 +2023-10-13 20:33:08.216576: Current learning rate: 0.00564 +2023-10-13 20:38:50.903836: train_loss -0.7577 +2023-10-13 20:38:50.903978: val_loss -0.6858 +2023-10-13 20:38:50.904093: Pseudo dice [0.9719, 0.8721, 0.906, 0.9011, 0.889, 0.6231] +2023-10-13 20:38:50.904176: Epoch time: 342.69 s +2023-10-13 20:38:52.073691: +2023-10-13 20:38:52.073796: Epoch 472 +2023-10-13 20:38:52.073898: Current learning rate: 0.00563 +2023-10-13 20:44:34.673557: train_loss -0.7508 +2023-10-13 20:44:34.673724: val_loss -0.6694 +2023-10-13 20:44:34.673828: Pseudo dice [0.9724, 0.8739, 0.9054, 0.8958, 0.8488, 0.6848] +2023-10-13 20:44:34.673910: Epoch time: 342.6 s +2023-10-13 20:44:35.835400: +2023-10-13 20:44:35.835660: Epoch 473 +2023-10-13 20:44:35.835839: Current learning rate: 0.00562 +2023-10-13 20:50:18.566574: train_loss -0.7557 +2023-10-13 20:50:18.566727: val_loss -0.6882 +2023-10-13 20:50:18.566829: Pseudo dice [0.9734, 0.8821, 0.9057, 0.8689, 0.8819, 0.6507] +2023-10-13 20:50:18.566919: Epoch time: 342.73 s +2023-10-13 20:50:19.890629: +2023-10-13 20:50:19.890841: Epoch 474 +2023-10-13 20:50:19.891009: Current learning rate: 0.00561 +2023-10-13 20:56:02.653807: train_loss -0.7542 +2023-10-13 20:56:02.653968: val_loss -0.6779 +2023-10-13 20:56:02.654072: Pseudo dice [0.9651, 0.8878, 0.9043, 0.8997, 0.9014, 0.7205] +2023-10-13 20:56:02.654158: Epoch time: 342.76 s +2023-10-13 20:56:03.814491: +2023-10-13 20:56:03.814634: Epoch 475 +2023-10-13 20:56:03.814736: Current learning rate: 0.0056 +2023-10-13 21:01:46.559974: train_loss -0.765 +2023-10-13 21:01:46.560142: val_loss -0.6898 +2023-10-13 21:01:46.560247: Pseudo dice [0.9702, 0.8744, 0.8973, 0.8718, 0.8581, 0.6256] +2023-10-13 21:01:46.560332: Epoch time: 342.75 s +2023-10-13 21:01:47.721451: +2023-10-13 21:01:47.721568: Epoch 476 +2023-10-13 21:01:47.721681: Current learning rate: 0.00559 +2023-10-13 21:07:30.418255: train_loss -0.7545 +2023-10-13 21:07:30.418397: val_loss -0.6938 +2023-10-13 21:07:30.418521: Pseudo dice [0.9695, 0.8742, 0.9083, 0.9077, 0.8955, 0.652] +2023-10-13 21:07:30.418609: Epoch time: 342.7 s +2023-10-13 21:07:31.580715: +2023-10-13 21:07:31.580885: Epoch 477 +2023-10-13 21:07:31.581058: Current learning rate: 0.00558 +2023-10-13 21:13:14.212479: train_loss -0.7431 +2023-10-13 21:13:14.212637: val_loss -0.6795 +2023-10-13 21:13:14.212791: Pseudo dice [0.9678, 0.8667, 0.8977, 0.8889, 0.901, 0.6212] +2023-10-13 21:13:14.212939: Epoch time: 342.63 s +2023-10-13 21:13:15.393621: +2023-10-13 21:13:15.393734: Epoch 478 +2023-10-13 21:13:15.393846: Current learning rate: 0.00557 +2023-10-13 21:18:58.102442: train_loss -0.7506 +2023-10-13 21:18:58.102617: val_loss -0.6869 +2023-10-13 21:18:58.102720: Pseudo dice [0.974, 0.8824, 0.9148, 0.8766, 0.8823, 0.7076] +2023-10-13 21:18:58.102803: Epoch time: 342.71 s +2023-10-13 21:18:59.285739: +2023-10-13 21:18:59.285846: Epoch 479 +2023-10-13 21:18:59.285956: Current learning rate: 0.00556 +2023-10-13 21:24:42.061664: train_loss -0.7648 +2023-10-13 21:24:42.061812: val_loss -0.6705 +2023-10-13 21:24:42.061929: Pseudo dice [0.9723, 0.8782, 0.907, 0.8468, 0.9188, 0.6476] +2023-10-13 21:24:42.062011: Epoch time: 342.78 s +2023-10-13 21:24:43.240377: +2023-10-13 21:24:43.240480: Epoch 480 +2023-10-13 21:24:43.240592: Current learning rate: 0.00555 +2023-10-13 21:30:26.114118: train_loss -0.7685 +2023-10-13 21:30:26.114263: val_loss -0.7279 +2023-10-13 21:30:26.114378: Pseudo dice [0.9736, 0.8856, 0.9145, 0.8838, 0.9068, 0.6979] +2023-10-13 21:30:26.114463: Epoch time: 342.87 s +2023-10-13 21:30:27.298173: +2023-10-13 21:30:27.298275: Epoch 481 +2023-10-13 21:30:27.298388: Current learning rate: 0.00554 +2023-10-13 21:36:10.124116: train_loss -0.7644 +2023-10-13 21:36:10.124267: val_loss -0.6641 +2023-10-13 21:36:10.124383: Pseudo dice [0.9726, 0.8851, 0.8848, 0.9298, 0.8775, 0.6806] +2023-10-13 21:36:10.124467: Epoch time: 342.83 s +2023-10-13 21:36:10.124532: Yayy! New best EMA pseudo Dice: 0.8661 +2023-10-13 21:36:13.334249: +2023-10-13 21:36:13.334475: Epoch 482 +2023-10-13 21:36:13.334589: Current learning rate: 0.00553 +2023-10-13 21:41:56.032331: train_loss -0.7431 +2023-10-13 21:41:56.032482: val_loss -0.6813 +2023-10-13 21:41:56.032579: Pseudo dice [0.9724, 0.8756, 0.9121, 0.8197, 0.8978, 0.6563] +2023-10-13 21:41:56.032657: Epoch time: 342.7 s +2023-10-13 21:41:57.219925: +2023-10-13 21:41:57.220038: Epoch 483 +2023-10-13 21:41:57.220150: Current learning rate: 0.00552 +2023-10-13 21:47:39.919025: train_loss -0.758 +2023-10-13 21:47:39.919188: val_loss -0.6802 +2023-10-13 21:47:39.919290: Pseudo dice [0.9655, 0.8868, 0.906, 0.8915, 0.8629, 0.6595] +2023-10-13 21:47:39.919374: Epoch time: 342.7 s +2023-10-13 21:47:41.102314: +2023-10-13 21:47:41.102428: Epoch 484 +2023-10-13 21:47:41.102549: Current learning rate: 0.00551 +2023-10-13 21:53:23.844656: train_loss -0.7449 +2023-10-13 21:53:23.844855: val_loss -0.6696 +2023-10-13 21:53:23.844958: Pseudo dice [0.9726, 0.8756, 0.9043, 0.8985, 0.8691, 0.6464] +2023-10-13 21:53:23.845042: Epoch time: 342.74 s +2023-10-13 21:53:25.021140: +2023-10-13 21:53:25.021313: Epoch 485 +2023-10-13 21:53:25.021472: Current learning rate: 0.0055 +2023-10-13 21:59:07.710898: train_loss -0.7488 +2023-10-13 21:59:07.711045: val_loss -0.6893 +2023-10-13 21:59:07.711160: Pseudo dice [0.9724, 0.8702, 0.9026, 0.8931, 0.879, 0.634] +2023-10-13 21:59:07.711244: Epoch time: 342.69 s +2023-10-13 21:59:08.895779: +2023-10-13 21:59:08.895951: Epoch 486 +2023-10-13 21:59:08.896113: Current learning rate: 0.00549 +2023-10-13 22:04:51.660993: train_loss -0.7676 +2023-10-13 22:04:51.661154: val_loss -0.6475 +2023-10-13 22:04:51.661267: Pseudo dice [0.9709, 0.8709, 0.8996, 0.884, 0.8804, 0.6183] +2023-10-13 22:04:51.661360: Epoch time: 342.77 s +2023-10-13 22:04:52.864772: +2023-10-13 22:04:52.864874: Epoch 487 +2023-10-13 22:04:52.864984: Current learning rate: 0.00548 +2023-10-13 22:10:35.540339: train_loss -0.7576 +2023-10-13 22:10:35.540563: val_loss -0.6497 +2023-10-13 22:10:35.540668: Pseudo dice [0.9741, 0.8889, 0.9033, 0.8672, 0.8771, 0.6665] +2023-10-13 22:10:35.540752: Epoch time: 342.68 s +2023-10-13 22:10:36.726859: +2023-10-13 22:10:36.727039: Epoch 488 +2023-10-13 22:10:36.727161: Current learning rate: 0.00547 +2023-10-13 22:16:19.534940: train_loss -0.7615 +2023-10-13 22:16:19.535086: val_loss -0.6575 +2023-10-13 22:16:19.535203: Pseudo dice [0.9667, 0.8767, 0.9079, 0.8491, 0.8916, 0.6484] +2023-10-13 22:16:19.535286: Epoch time: 342.81 s +2023-10-13 22:16:20.896125: +2023-10-13 22:16:20.896323: Epoch 489 +2023-10-13 22:16:20.896467: Current learning rate: 0.00546 +2023-10-13 22:22:03.693961: train_loss -0.7422 +2023-10-13 22:22:03.694103: val_loss -0.7094 +2023-10-13 22:22:03.694219: Pseudo dice [0.9711, 0.863, 0.9232, 0.8875, 0.892, 0.6524] +2023-10-13 22:22:03.694304: Epoch time: 342.8 s +2023-10-13 22:22:04.877347: +2023-10-13 22:22:04.877567: Epoch 490 +2023-10-13 22:22:04.877707: Current learning rate: 0.00546 +2023-10-13 22:27:47.635006: train_loss -0.7624 +2023-10-13 22:27:47.635154: val_loss -0.6698 +2023-10-13 22:27:47.635269: Pseudo dice [0.9733, 0.8751, 0.8996, 0.8598, 0.8866, 0.6615] +2023-10-13 22:27:47.635352: Epoch time: 342.76 s +2023-10-13 22:27:48.818633: +2023-10-13 22:27:48.818755: Epoch 491 +2023-10-13 22:27:48.818854: Current learning rate: 0.00545 +2023-10-13 22:33:31.671255: train_loss -0.7586 +2023-10-13 22:33:31.671398: val_loss -0.6342 +2023-10-13 22:33:31.671517: Pseudo dice [0.9736, 0.8698, 0.9083, 0.8827, 0.8882, 0.6614] +2023-10-13 22:33:31.671602: Epoch time: 342.85 s +2023-10-13 22:33:32.851060: +2023-10-13 22:33:32.851192: Epoch 492 +2023-10-13 22:33:32.851303: Current learning rate: 0.00544 +2023-10-13 22:39:15.655610: train_loss -0.755 +2023-10-13 22:39:15.655751: val_loss -0.6262 +2023-10-13 22:39:15.655866: Pseudo dice [0.9703, 0.8549, 0.9065, 0.8618, 0.8827, 0.6285] +2023-10-13 22:39:15.655950: Epoch time: 342.81 s +2023-10-13 22:39:16.834273: +2023-10-13 22:39:16.834431: Epoch 493 +2023-10-13 22:39:16.834603: Current learning rate: 0.00543 +2023-10-13 22:44:59.593207: train_loss -0.7597 +2023-10-13 22:44:59.593379: val_loss -0.6624 +2023-10-13 22:44:59.593484: Pseudo dice [0.9719, 0.8689, 0.9185, 0.8671, 0.8137, 0.6263] +2023-10-13 22:44:59.593567: Epoch time: 342.76 s +2023-10-13 22:45:00.796358: +2023-10-13 22:45:00.796461: Epoch 494 +2023-10-13 22:45:00.796570: Current learning rate: 0.00542 +2023-10-13 22:50:43.559069: train_loss -0.7681 +2023-10-13 22:50:43.559232: val_loss -0.694 +2023-10-13 22:50:43.559345: Pseudo dice [0.973, 0.8726, 0.9118, 0.8987, 0.8604, 0.6806] +2023-10-13 22:50:43.559514: Epoch time: 342.76 s +2023-10-13 22:50:44.745672: +2023-10-13 22:50:44.745773: Epoch 495 +2023-10-13 22:50:44.745884: Current learning rate: 0.00541 +2023-10-13 22:56:27.595353: train_loss -0.7563 +2023-10-13 22:56:27.595503: val_loss -0.6664 +2023-10-13 22:56:27.595618: Pseudo dice [0.9729, 0.868, 0.9118, 0.8326, 0.8699, 0.7097] +2023-10-13 22:56:27.595701: Epoch time: 342.85 s +2023-10-13 22:56:28.956664: +2023-10-13 22:56:28.956848: Epoch 496 +2023-10-13 22:56:28.957009: Current learning rate: 0.0054 +2023-10-13 23:02:11.758677: train_loss -0.7645 +2023-10-13 23:02:11.758836: val_loss -0.6693 +2023-10-13 23:02:11.758941: Pseudo dice [0.9682, 0.8668, 0.9126, 0.8758, 0.897, 0.6535] +2023-10-13 23:02:11.759025: Epoch time: 342.8 s +2023-10-13 23:02:12.942362: +2023-10-13 23:02:12.942598: Epoch 497 +2023-10-13 23:02:12.942722: Current learning rate: 0.00539 +2023-10-13 23:07:55.777285: train_loss -0.7387 +2023-10-13 23:07:55.777441: val_loss -0.6693 +2023-10-13 23:07:55.777545: Pseudo dice [0.9708, 0.874, 0.877, 0.9082, 0.8657, 0.6637] +2023-10-13 23:07:55.777629: Epoch time: 342.84 s +2023-10-13 23:07:57.002566: +2023-10-13 23:07:57.002682: Epoch 498 +2023-10-13 23:07:57.002796: Current learning rate: 0.00538 +2023-10-13 23:13:39.831550: train_loss -0.7755 +2023-10-13 23:13:39.831701: val_loss -0.6851 +2023-10-13 23:13:39.831802: Pseudo dice [0.9739, 0.8813, 0.9115, 0.9206, 0.9018, 0.6563] +2023-10-13 23:13:39.831884: Epoch time: 342.83 s +2023-10-13 23:13:41.011805: +2023-10-13 23:13:41.011917: Epoch 499 +2023-10-13 23:13:41.012030: Current learning rate: 0.00537 +2023-10-13 23:19:23.738166: train_loss -0.7497 +2023-10-13 23:19:23.738324: val_loss -0.6858 +2023-10-13 23:19:23.738428: Pseudo dice [0.9706, 0.8674, 0.9086, 0.9048, 0.8632, 0.6149] +2023-10-13 23:19:23.738519: Epoch time: 342.73 s +2023-10-13 23:19:26.760052: +2023-10-13 23:19:26.760223: Epoch 500 +2023-10-13 23:19:26.760374: Current learning rate: 0.00536 +2023-10-13 23:25:09.597438: train_loss -0.7762 +2023-10-13 23:25:09.597573: val_loss -0.6941 +2023-10-13 23:25:09.597689: Pseudo dice [0.9748, 0.8709, 0.9051, 0.8744, 0.8879, 0.5889] +2023-10-13 23:25:09.597773: Epoch time: 342.84 s +2023-10-13 23:25:10.779368: +2023-10-13 23:25:10.779555: Epoch 501 +2023-10-13 23:25:10.779696: Current learning rate: 0.00535 +2023-10-13 23:30:53.492405: train_loss -0.7521 +2023-10-13 23:30:53.492553: val_loss -0.6774 +2023-10-13 23:30:53.492654: Pseudo dice [0.9729, 0.8775, 0.9089, 0.8872, 0.8752, 0.6523] +2023-10-13 23:30:53.492739: Epoch time: 342.71 s +2023-10-13 23:30:54.679928: +2023-10-13 23:30:54.680030: Epoch 502 +2023-10-13 23:30:54.680144: Current learning rate: 0.00534 +2023-10-13 23:36:37.482852: train_loss -0.7741 +2023-10-13 23:36:37.483117: val_loss -0.7046 +2023-10-13 23:36:37.483241: Pseudo dice [0.9728, 0.8684, 0.8812, 0.916, 0.879, 0.649] +2023-10-13 23:36:37.483325: Epoch time: 342.8 s +2023-10-13 23:36:38.665983: +2023-10-13 23:36:38.666084: Epoch 503 +2023-10-13 23:36:38.666193: Current learning rate: 0.00533 +2023-10-13 23:42:21.564930: train_loss -0.7545 +2023-10-13 23:42:21.565095: val_loss -0.7014 +2023-10-13 23:42:21.565195: Pseudo dice [0.9704, 0.8746, 0.8881, 0.903, 0.9076, 0.6925] +2023-10-13 23:42:21.565278: Epoch time: 342.9 s +2023-10-13 23:42:22.929875: +2023-10-13 23:42:22.930065: Epoch 504 +2023-10-13 23:42:22.930216: Current learning rate: 0.00532 +2023-10-13 23:48:05.849986: train_loss -0.7513 +2023-10-13 23:48:05.850137: val_loss -0.6635 +2023-10-13 23:48:05.850255: Pseudo dice [0.9716, 0.8761, 0.9013, 0.8685, 0.8574, 0.6334] +2023-10-13 23:48:05.850340: Epoch time: 342.92 s +2023-10-13 23:48:07.036578: +2023-10-13 23:48:07.036695: Epoch 505 +2023-10-13 23:48:07.036805: Current learning rate: 0.00531 +2023-10-13 23:53:49.834350: train_loss -0.7671 +2023-10-13 23:53:49.834490: val_loss -0.6881 +2023-10-13 23:53:49.834619: Pseudo dice [0.9709, 0.8764, 0.9013, 0.8776, 0.9049, 0.6744] +2023-10-13 23:53:49.834721: Epoch time: 342.8 s +2023-10-13 23:53:51.016369: +2023-10-13 23:53:51.016569: Epoch 506 +2023-10-13 23:53:51.016719: Current learning rate: 0.0053 +2023-10-13 23:59:33.863346: train_loss -0.7616 +2023-10-13 23:59:33.863503: val_loss -0.6504 +2023-10-13 23:59:33.863610: Pseudo dice [0.9705, 0.8774, 0.893, 0.8922, 0.903, 0.6441] +2023-10-13 23:59:33.863694: Epoch time: 342.85 s +2023-10-13 23:59:35.065684: +2023-10-13 23:59:35.065877: Epoch 507 +2023-10-13 23:59:35.066017: Current learning rate: 0.00529 +2023-10-14 00:05:17.714968: train_loss -0.7826 +2023-10-14 00:05:17.715108: val_loss -0.6706 +2023-10-14 00:05:17.715224: Pseudo dice [0.9719, 0.8682, 0.9051, 0.8186, 0.8847, 0.6094] +2023-10-14 00:05:17.715308: Epoch time: 342.65 s +2023-10-14 00:05:18.895402: +2023-10-14 00:05:18.895510: Epoch 508 +2023-10-14 00:05:18.895621: Current learning rate: 0.00528 +2023-10-14 00:11:01.678665: train_loss -0.7613 +2023-10-14 00:11:01.678916: val_loss -0.6766 +2023-10-14 00:11:01.679021: Pseudo dice [0.9726, 0.8787, 0.8838, 0.8587, 0.8774, 0.6546] +2023-10-14 00:11:01.679105: Epoch time: 342.78 s +2023-10-14 00:11:02.878380: +2023-10-14 00:11:02.878582: Epoch 509 +2023-10-14 00:11:02.878746: Current learning rate: 0.00527 +2023-10-14 00:16:45.608742: train_loss -0.7662 +2023-10-14 00:16:45.608891: val_loss -0.6378 +2023-10-14 00:16:45.609008: Pseudo dice [0.9702, 0.8729, 0.9015, 0.8956, 0.8968, 0.6812] +2023-10-14 00:16:45.609093: Epoch time: 342.73 s +2023-10-14 00:16:46.788916: +2023-10-14 00:16:46.789093: Epoch 510 +2023-10-14 00:16:46.789234: Current learning rate: 0.00526 +2023-10-14 00:22:29.509088: train_loss -0.7542 +2023-10-14 00:22:29.509248: val_loss -0.6201 +2023-10-14 00:22:29.509350: Pseudo dice [0.9661, 0.8718, 0.8798, 0.9165, 0.9053, 0.6205] +2023-10-14 00:22:29.509432: Epoch time: 342.72 s +2023-10-14 00:22:30.863433: +2023-10-14 00:22:30.863565: Epoch 511 +2023-10-14 00:22:30.863678: Current learning rate: 0.00525 +2023-10-14 00:28:13.498229: train_loss -0.76 +2023-10-14 00:28:13.498438: val_loss -0.6476 +2023-10-14 00:28:13.498549: Pseudo dice [0.9632, 0.8769, 0.9018, 0.8442, 0.9033, 0.6513] +2023-10-14 00:28:13.498638: Epoch time: 342.64 s +2023-10-14 00:28:14.676231: +2023-10-14 00:28:14.676351: Epoch 512 +2023-10-14 00:28:14.676467: Current learning rate: 0.00524 +2023-10-14 00:33:57.339422: train_loss -0.7796 +2023-10-14 00:33:57.339571: val_loss -0.7136 +2023-10-14 00:33:57.339689: Pseudo dice [0.9635, 0.87, 0.9143, 0.8404, 0.8724, 0.6572] +2023-10-14 00:33:57.339772: Epoch time: 342.66 s +2023-10-14 00:33:58.545179: +2023-10-14 00:33:58.545379: Epoch 513 +2023-10-14 00:33:58.545519: Current learning rate: 0.00523 +2023-10-14 00:39:41.224281: train_loss -0.7343 +2023-10-14 00:39:41.224448: val_loss -0.6954 +2023-10-14 00:39:41.224551: Pseudo dice [0.9661, 0.8698, 0.8939, 0.8987, 0.8703, 0.628] +2023-10-14 00:39:41.224635: Epoch time: 342.68 s +2023-10-14 00:39:42.412284: +2023-10-14 00:39:42.412398: Epoch 514 +2023-10-14 00:39:42.412512: Current learning rate: 0.00522 +2023-10-14 00:45:25.100104: train_loss -0.735 +2023-10-14 00:45:25.100254: val_loss -0.6623 +2023-10-14 00:45:25.100372: Pseudo dice [0.9693, 0.8661, 0.91, 0.8641, 0.9006, 0.6693] +2023-10-14 00:45:25.100456: Epoch time: 342.69 s +2023-10-14 00:45:26.296932: +2023-10-14 00:45:26.297125: Epoch 515 +2023-10-14 00:45:26.297274: Current learning rate: 0.00521 +2023-10-14 00:51:09.031526: train_loss -0.7589 +2023-10-14 00:51:09.031677: val_loss -0.6954 +2023-10-14 00:51:09.031795: Pseudo dice [0.971, 0.8829, 0.903, 0.8564, 0.9092, 0.6916] +2023-10-14 00:51:09.031879: Epoch time: 342.74 s +2023-10-14 00:51:10.219244: +2023-10-14 00:51:10.219350: Epoch 516 +2023-10-14 00:51:10.219462: Current learning rate: 0.0052 +2023-10-14 00:56:52.943144: train_loss -0.7592 +2023-10-14 00:56:52.943335: val_loss -0.6974 +2023-10-14 00:56:52.943439: Pseudo dice [0.9728, 0.8753, 0.9102, 0.8882, 0.8932, 0.6953] +2023-10-14 00:56:52.943532: Epoch time: 342.72 s +2023-10-14 00:56:54.128658: +2023-10-14 00:56:54.128819: Epoch 517 +2023-10-14 00:56:54.128983: Current learning rate: 0.00519 +2023-10-14 01:02:36.957637: train_loss -0.7683 +2023-10-14 01:02:36.957776: val_loss -0.7279 +2023-10-14 01:02:36.957894: Pseudo dice [0.9732, 0.872, 0.9036, 0.8805, 0.8996, 0.6959] +2023-10-14 01:02:36.957977: Epoch time: 342.83 s +2023-10-14 01:02:38.147236: +2023-10-14 01:02:38.147339: Epoch 518 +2023-10-14 01:02:38.147439: Current learning rate: 0.00518 +2023-10-14 01:08:20.862761: train_loss -0.7635 +2023-10-14 01:08:20.862926: val_loss -0.7218 +2023-10-14 01:08:20.863043: Pseudo dice [0.974, 0.882, 0.8983, 0.8954, 0.8932, 0.6756] +2023-10-14 01:08:20.863128: Epoch time: 342.72 s +2023-10-14 01:08:22.049418: +2023-10-14 01:08:22.049532: Epoch 519 +2023-10-14 01:08:22.049642: Current learning rate: 0.00518 +2023-10-14 01:14:04.695090: train_loss -0.7559 +2023-10-14 01:14:04.695233: val_loss -0.7087 +2023-10-14 01:14:04.695351: Pseudo dice [0.9731, 0.8762, 0.9132, 0.9102, 0.8864, 0.6641] +2023-10-14 01:14:04.695434: Epoch time: 342.65 s +2023-10-14 01:14:05.883294: +2023-10-14 01:14:05.883419: Epoch 520 +2023-10-14 01:14:05.883519: Current learning rate: 0.00517 +2023-10-14 01:19:48.631706: train_loss -0.7633 +2023-10-14 01:19:48.631899: val_loss -0.6538 +2023-10-14 01:19:48.632002: Pseudo dice [0.9726, 0.8849, 0.8958, 0.8736, 0.8888, 0.6754] +2023-10-14 01:19:48.632086: Epoch time: 342.75 s +2023-10-14 01:19:49.815742: +2023-10-14 01:19:49.815938: Epoch 521 +2023-10-14 01:19:49.816105: Current learning rate: 0.00516 +2023-10-14 01:25:32.650979: train_loss -0.7565 +2023-10-14 01:25:32.651120: val_loss -0.7255 +2023-10-14 01:25:32.651239: Pseudo dice [0.9725, 0.8714, 0.9068, 0.8966, 0.8847, 0.6332] +2023-10-14 01:25:32.651332: Epoch time: 342.84 s +2023-10-14 01:25:33.832436: +2023-10-14 01:25:33.832546: Epoch 522 +2023-10-14 01:25:33.832656: Current learning rate: 0.00515 +2023-10-14 01:31:16.686322: train_loss -0.7527 +2023-10-14 01:31:16.686500: val_loss -0.6473 +2023-10-14 01:31:16.686605: Pseudo dice [0.9729, 0.8828, 0.8913, 0.9124, 0.8774, 0.6959] +2023-10-14 01:31:16.686690: Epoch time: 342.85 s +2023-10-14 01:31:17.873138: +2023-10-14 01:31:17.873245: Epoch 523 +2023-10-14 01:31:17.873358: Current learning rate: 0.00514 +2023-10-14 01:37:00.508588: train_loss -0.7677 +2023-10-14 01:37:00.508729: val_loss -0.6479 +2023-10-14 01:37:00.508845: Pseudo dice [0.9738, 0.8746, 0.908, 0.9005, 0.8563, 0.6466] +2023-10-14 01:37:00.508929: Epoch time: 342.64 s +2023-10-14 01:37:01.691736: +2023-10-14 01:37:01.691912: Epoch 524 +2023-10-14 01:37:01.692019: Current learning rate: 0.00513 +2023-10-14 01:42:44.450337: train_loss -0.7746 +2023-10-14 01:42:44.450506: val_loss -0.6353 +2023-10-14 01:42:44.450609: Pseudo dice [0.9736, 0.8735, 0.9058, 0.8927, 0.9168, 0.6706] +2023-10-14 01:42:44.450693: Epoch time: 342.76 s +2023-10-14 01:42:45.632326: +2023-10-14 01:42:45.632431: Epoch 525 +2023-10-14 01:42:45.632542: Current learning rate: 0.00512 +2023-10-14 01:48:28.476812: train_loss -0.792 +2023-10-14 01:48:28.476954: val_loss -0.6603 +2023-10-14 01:48:28.477071: Pseudo dice [0.974, 0.8715, 0.8842, 0.8632, 0.8785, 0.6725] +2023-10-14 01:48:28.477155: Epoch time: 342.85 s +2023-10-14 01:48:29.836975: +2023-10-14 01:48:29.837085: Epoch 526 +2023-10-14 01:48:29.837200: Current learning rate: 0.00511 +2023-10-14 01:54:12.802082: train_loss -0.7497 +2023-10-14 01:54:12.802228: val_loss -0.6548 +2023-10-14 01:54:12.802325: Pseudo dice [0.9727, 0.8758, 0.8951, 0.873, 0.8989, 0.6782] +2023-10-14 01:54:12.802404: Epoch time: 342.97 s +2023-10-14 01:54:13.993731: +2023-10-14 01:54:13.993873: Epoch 527 +2023-10-14 01:54:13.993973: Current learning rate: 0.0051 +2023-10-14 01:59:56.821689: train_loss -0.7711 +2023-10-14 01:59:56.821836: val_loss -0.6594 +2023-10-14 01:59:56.821953: Pseudo dice [0.9716, 0.8693, 0.9138, 0.8551, 0.9123, 0.6955] +2023-10-14 01:59:56.822037: Epoch time: 342.83 s +2023-10-14 01:59:58.003834: +2023-10-14 01:59:58.004006: Epoch 528 +2023-10-14 01:59:58.004176: Current learning rate: 0.00509 +2023-10-14 02:05:40.926968: train_loss -0.7515 +2023-10-14 02:05:40.927118: val_loss -0.6682 +2023-10-14 02:05:40.927235: Pseudo dice [0.9697, 0.8758, 0.9138, 0.8853, 0.8445, 0.6351] +2023-10-14 02:05:40.927320: Epoch time: 342.92 s +2023-10-14 02:05:42.108870: +2023-10-14 02:05:42.109053: Epoch 529 +2023-10-14 02:05:42.109212: Current learning rate: 0.00508 +2023-10-14 02:11:25.043813: train_loss -0.7755 +2023-10-14 02:11:25.043963: val_loss -0.7076 +2023-10-14 02:11:25.044080: Pseudo dice [0.9712, 0.88, 0.8942, 0.8838, 0.8785, 0.6566] +2023-10-14 02:11:25.044164: Epoch time: 342.94 s +2023-10-14 02:11:26.436989: +2023-10-14 02:11:26.437282: Epoch 530 +2023-10-14 02:11:26.437442: Current learning rate: 0.00507 +2023-10-14 02:17:09.107043: train_loss -0.7584 +2023-10-14 02:17:09.107191: val_loss -0.6956 +2023-10-14 02:17:09.107309: Pseudo dice [0.9726, 0.8682, 0.8936, 0.8929, 0.9094, 0.662] +2023-10-14 02:17:09.107392: Epoch time: 342.67 s +2023-10-14 02:17:10.305228: +2023-10-14 02:17:10.305418: Epoch 531 +2023-10-14 02:17:10.305555: Current learning rate: 0.00506 +2023-10-14 02:22:53.064467: train_loss -0.7343 +2023-10-14 02:22:53.064617: val_loss -0.6414 +2023-10-14 02:22:53.064734: Pseudo dice [0.9697, 0.8703, 0.8942, 0.9267, 0.86, 0.6497] +2023-10-14 02:22:53.064817: Epoch time: 342.76 s +2023-10-14 02:22:54.251869: +2023-10-14 02:22:54.251970: Epoch 532 +2023-10-14 02:22:54.252082: Current learning rate: 0.00505 +2023-10-14 02:28:37.042867: train_loss -0.7774 +2023-10-14 02:28:37.043200: val_loss -0.7221 +2023-10-14 02:28:37.043359: Pseudo dice [0.9737, 0.8728, 0.9259, 0.8949, 0.8799, 0.6641] +2023-10-14 02:28:37.043444: Epoch time: 342.79 s +2023-10-14 02:28:38.410438: +2023-10-14 02:28:38.410583: Epoch 533 +2023-10-14 02:28:38.410685: Current learning rate: 0.00504 +2023-10-14 02:34:21.107591: train_loss -0.7601 +2023-10-14 02:34:21.107756: val_loss -0.6891 +2023-10-14 02:34:21.107865: Pseudo dice [0.9725, 0.8683, 0.9038, 0.8712, 0.9032, 0.6471] +2023-10-14 02:34:21.107953: Epoch time: 342.7 s +2023-10-14 02:34:22.289774: +2023-10-14 02:34:22.290019: Epoch 534 +2023-10-14 02:34:22.290126: Current learning rate: 0.00503 +2023-10-14 02:40:05.035558: train_loss -0.75 +2023-10-14 02:40:05.035711: val_loss -0.6791 +2023-10-14 02:40:05.035830: Pseudo dice [0.9729, 0.87, 0.9134, 0.8769, 0.9051, 0.637] +2023-10-14 02:40:05.035914: Epoch time: 342.75 s +2023-10-14 02:40:06.220230: +2023-10-14 02:40:06.220495: Epoch 535 +2023-10-14 02:40:06.220635: Current learning rate: 0.00502 +2023-10-14 02:45:48.860529: train_loss -0.7369 +2023-10-14 02:45:48.860693: val_loss -0.6574 +2023-10-14 02:45:48.860796: Pseudo dice [0.9732, 0.8554, 0.9029, 0.8568, 0.8956, 0.6139] +2023-10-14 02:45:48.860881: Epoch time: 342.64 s +2023-10-14 02:45:50.048534: +2023-10-14 02:45:50.048640: Epoch 536 +2023-10-14 02:45:50.048751: Current learning rate: 0.00501 +2023-10-14 02:51:32.746356: train_loss -0.7331 +2023-10-14 02:51:32.746529: val_loss -0.6701 +2023-10-14 02:51:32.752693: Pseudo dice [0.9684, 0.8767, 0.9009, 0.8783, 0.8734, 0.6605] +2023-10-14 02:51:32.752934: Epoch time: 342.7 s +2023-10-14 02:51:33.943737: +2023-10-14 02:51:33.943921: Epoch 537 +2023-10-14 02:51:33.944085: Current learning rate: 0.005 +2023-10-14 02:57:16.593335: train_loss -0.7678 +2023-10-14 02:57:16.593486: val_loss -0.5853 +2023-10-14 02:57:16.593604: Pseudo dice [0.9738, 0.8527, 0.8826, 0.9041, 0.8719, 0.6343] +2023-10-14 02:57:16.593687: Epoch time: 342.65 s +2023-10-14 02:57:17.780178: +2023-10-14 02:57:17.780396: Epoch 538 +2023-10-14 02:57:17.780499: Current learning rate: 0.00499 +2023-10-14 03:03:00.520654: train_loss -0.7622 +2023-10-14 03:03:00.520803: val_loss -0.6717 +2023-10-14 03:03:00.520920: Pseudo dice [0.9726, 0.8742, 0.9, 0.875, 0.8031, 0.6043] +2023-10-14 03:03:00.521004: Epoch time: 342.74 s +2023-10-14 03:03:01.702940: +2023-10-14 03:03:01.703067: Epoch 539 +2023-10-14 03:03:01.703184: Current learning rate: 0.00498 +2023-10-14 03:08:44.425042: train_loss -0.773 +2023-10-14 03:08:44.425183: val_loss -0.6727 +2023-10-14 03:08:44.425298: Pseudo dice [0.9691, 0.8717, 0.8895, 0.9006, 0.9029, 0.6697] +2023-10-14 03:08:44.425395: Epoch time: 342.72 s +2023-10-14 03:08:45.611629: +2023-10-14 03:08:45.611805: Epoch 540 +2023-10-14 03:08:45.611968: Current learning rate: 0.00497 +2023-10-14 03:14:28.468173: train_loss -0.7032 +2023-10-14 03:14:28.468314: val_loss -0.6067 +2023-10-14 03:14:28.468429: Pseudo dice [0.9692, 0.8752, 0.8861, 0.896, 0.8561, 0.6156] +2023-10-14 03:14:28.468513: Epoch time: 342.86 s +2023-10-14 03:14:29.843549: +2023-10-14 03:14:29.843667: Epoch 541 +2023-10-14 03:14:29.843778: Current learning rate: 0.00496 +2023-10-14 03:20:12.562958: train_loss -0.7298 +2023-10-14 03:20:12.563128: val_loss -0.6406 +2023-10-14 03:20:12.563231: Pseudo dice [0.9705, 0.8702, 0.896, 0.8525, 0.8248, 0.6297] +2023-10-14 03:20:12.563317: Epoch time: 342.72 s +2023-10-14 03:20:13.746246: +2023-10-14 03:20:13.746440: Epoch 542 +2023-10-14 03:20:13.746586: Current learning rate: 0.00495 +2023-10-14 03:25:56.335531: train_loss -0.7188 +2023-10-14 03:25:56.335699: val_loss -0.6533 +2023-10-14 03:25:56.335802: Pseudo dice [0.97, 0.8661, 0.8853, 0.8726, 0.8735, 0.6308] +2023-10-14 03:25:56.335886: Epoch time: 342.59 s +2023-10-14 03:25:57.522238: +2023-10-14 03:25:57.522353: Epoch 543 +2023-10-14 03:25:57.522465: Current learning rate: 0.00494 +2023-10-14 03:31:40.307137: train_loss -0.7476 +2023-10-14 03:31:40.307287: val_loss -0.6366 +2023-10-14 03:31:40.307405: Pseudo dice [0.9696, 0.8777, 0.8948, 0.8782, 0.8756, 0.654] +2023-10-14 03:31:40.307489: Epoch time: 342.79 s +2023-10-14 03:31:41.491104: +2023-10-14 03:31:41.491252: Epoch 544 +2023-10-14 03:31:41.491384: Current learning rate: 0.00493 +2023-10-14 03:37:24.360770: train_loss -0.7496 +2023-10-14 03:37:24.360918: val_loss -0.6596 +2023-10-14 03:37:24.361035: Pseudo dice [0.9695, 0.8617, 0.9058, 0.9146, 0.9021, 0.6749] +2023-10-14 03:37:24.361120: Epoch time: 342.87 s +2023-10-14 03:37:25.549433: +2023-10-14 03:37:25.549539: Epoch 545 +2023-10-14 03:37:25.549653: Current learning rate: 0.00492 +2023-10-14 03:43:08.206327: train_loss -0.749 +2023-10-14 03:43:08.206486: val_loss -0.6826 +2023-10-14 03:43:08.206599: Pseudo dice [0.9729, 0.8793, 0.904, 0.896, 0.8867, 0.6812] +2023-10-14 03:43:08.206683: Epoch time: 342.66 s +2023-10-14 03:43:09.399510: +2023-10-14 03:43:09.399612: Epoch 546 +2023-10-14 03:43:09.399724: Current learning rate: 0.00491 +2023-10-14 03:48:51.917928: train_loss -0.7698 +2023-10-14 03:48:51.918085: val_loss -0.6813 +2023-10-14 03:48:51.918188: Pseudo dice [0.9707, 0.8737, 0.9098, 0.8771, 0.8609, 0.6703] +2023-10-14 03:48:51.918271: Epoch time: 342.52 s +2023-10-14 03:48:53.108074: +2023-10-14 03:48:53.108179: Epoch 547 +2023-10-14 03:48:53.108291: Current learning rate: 0.0049 +2023-10-14 03:54:35.849107: train_loss -0.7409 +2023-10-14 03:54:35.849247: val_loss -0.6862 +2023-10-14 03:54:35.849374: Pseudo dice [0.9735, 0.8796, 0.9134, 0.8858, 0.8848, 0.6757] +2023-10-14 03:54:35.849458: Epoch time: 342.74 s +2023-10-14 03:54:37.212050: +2023-10-14 03:54:37.212163: Epoch 548 +2023-10-14 03:54:37.212276: Current learning rate: 0.00489 +2023-10-14 04:00:19.932513: train_loss -0.7589 +2023-10-14 04:00:19.932652: val_loss -0.7011 +2023-10-14 04:00:19.932749: Pseudo dice [0.9739, 0.8783, 0.9107, 0.864, 0.8433, 0.6987] +2023-10-14 04:00:19.932827: Epoch time: 342.72 s +2023-10-14 04:00:21.118169: +2023-10-14 04:00:21.118371: Epoch 549 +2023-10-14 04:00:21.118519: Current learning rate: 0.00488 +2023-10-14 04:06:03.882905: train_loss -0.7638 +2023-10-14 04:06:03.883054: val_loss -0.6975 +2023-10-14 04:06:03.883172: Pseudo dice [0.9728, 0.8696, 0.904, 0.9159, 0.9029, 0.6581] +2023-10-14 04:06:03.883254: Epoch time: 342.77 s +2023-10-14 04:06:06.755329: +2023-10-14 04:06:06.755444: Epoch 550 +2023-10-14 04:06:06.755558: Current learning rate: 0.00487 +2023-10-14 04:11:49.346756: train_loss -0.7536 +2023-10-14 04:11:49.347019: val_loss -0.6933 +2023-10-14 04:11:49.347124: Pseudo dice [0.9752, 0.8921, 0.8993, 0.9302, 0.9192, 0.6753] +2023-10-14 04:11:49.347209: Epoch time: 342.59 s +2023-10-14 04:11:50.528858: +2023-10-14 04:11:50.529079: Epoch 551 +2023-10-14 04:11:50.529209: Current learning rate: 0.00486 +2023-10-14 04:17:33.193875: train_loss -0.7627 +2023-10-14 04:17:33.194030: val_loss -0.7203 +2023-10-14 04:17:33.194132: Pseudo dice [0.9653, 0.8788, 0.9169, 0.8872, 0.9078, 0.6493] +2023-10-14 04:17:33.194214: Epoch time: 342.67 s +2023-10-14 04:17:34.426304: +2023-10-14 04:17:34.426464: Epoch 552 +2023-10-14 04:17:34.426625: Current learning rate: 0.00485 +2023-10-14 04:23:17.093513: train_loss -0.7784 +2023-10-14 04:23:17.093699: val_loss -0.6783 +2023-10-14 04:23:17.093801: Pseudo dice [0.967, 0.888, 0.8983, 0.8842, 0.848, 0.6739] +2023-10-14 04:23:17.093884: Epoch time: 342.67 s +2023-10-14 04:23:18.282970: +2023-10-14 04:23:18.283092: Epoch 553 +2023-10-14 04:23:18.283191: Current learning rate: 0.00484 +2023-10-14 04:29:01.133753: train_loss -0.7788 +2023-10-14 04:29:01.133955: val_loss -0.6872 +2023-10-14 04:29:01.134057: Pseudo dice [0.9728, 0.8724, 0.9145, 0.8875, 0.9077, 0.6566] +2023-10-14 04:29:01.134139: Epoch time: 342.85 s +2023-10-14 04:29:02.317680: +2023-10-14 04:29:02.318028: Epoch 554 +2023-10-14 04:29:02.318191: Current learning rate: 0.00484 +2023-10-14 04:34:45.012415: train_loss -0.7789 +2023-10-14 04:34:45.012566: val_loss -0.7069 +2023-10-14 04:34:45.012684: Pseudo dice [0.9749, 0.881, 0.8984, 0.9079, 0.905, 0.6924] +2023-10-14 04:34:45.012769: Epoch time: 342.7 s +2023-10-14 04:34:46.389213: +2023-10-14 04:34:46.389344: Epoch 555 +2023-10-14 04:34:46.389525: Current learning rate: 0.00483 +2023-10-14 04:40:29.069696: train_loss -0.751 +2023-10-14 04:40:29.069846: val_loss -0.6874 +2023-10-14 04:40:29.069963: Pseudo dice [0.9683, 0.8773, 0.9072, 0.886, 0.8758, 0.6247] +2023-10-14 04:40:29.070049: Epoch time: 342.68 s +2023-10-14 04:40:30.252636: +2023-10-14 04:40:30.252805: Epoch 556 +2023-10-14 04:40:30.252963: Current learning rate: 0.00482 +2023-10-14 04:46:12.998168: train_loss -0.7444 +2023-10-14 04:46:12.998322: val_loss -0.6972 +2023-10-14 04:46:12.998427: Pseudo dice [0.9712, 0.8641, 0.894, 0.9013, 0.8736, 0.6483] +2023-10-14 04:46:12.998520: Epoch time: 342.75 s +2023-10-14 04:46:14.187872: +2023-10-14 04:46:14.188096: Epoch 557 +2023-10-14 04:46:14.188235: Current learning rate: 0.00481 +2023-10-14 04:51:56.907253: train_loss -0.7737 +2023-10-14 04:51:56.907401: val_loss -0.672 +2023-10-14 04:51:56.907505: Pseudo dice [0.9727, 0.8679, 0.9112, 0.8516, 0.8704, 0.6473] +2023-10-14 04:51:56.907591: Epoch time: 342.72 s +2023-10-14 04:51:58.101392: +2023-10-14 04:51:58.101506: Epoch 558 +2023-10-14 04:51:58.101621: Current learning rate: 0.0048 +2023-10-14 04:57:40.929823: train_loss -0.7728 +2023-10-14 04:57:40.929959: val_loss -0.7034 +2023-10-14 04:57:40.930077: Pseudo dice [0.9705, 0.8607, 0.9112, 0.8518, 0.8978, 0.6604] +2023-10-14 04:57:40.930162: Epoch time: 342.83 s +2023-10-14 04:57:42.127573: +2023-10-14 04:57:42.127776: Epoch 559 +2023-10-14 04:57:42.127941: Current learning rate: 0.00479 +2023-10-14 05:03:24.942330: train_loss -0.7504 +2023-10-14 05:03:24.942526: val_loss -0.6307 +2023-10-14 05:03:24.942632: Pseudo dice [0.9739, 0.866, 0.8796, 0.8064, 0.8504, 0.6617] +2023-10-14 05:03:24.942716: Epoch time: 342.82 s +2023-10-14 05:03:26.127464: +2023-10-14 05:03:26.127568: Epoch 560 +2023-10-14 05:03:26.127680: Current learning rate: 0.00478 +2023-10-14 05:09:08.956063: train_loss -0.7687 +2023-10-14 05:09:08.956204: val_loss -0.6666 +2023-10-14 05:09:08.956321: Pseudo dice [0.9742, 0.8676, 0.9081, 0.8798, 0.904, 0.671] +2023-10-14 05:09:08.956406: Epoch time: 342.83 s +2023-10-14 05:09:10.147296: +2023-10-14 05:09:10.147399: Epoch 561 +2023-10-14 05:09:10.147511: Current learning rate: 0.00477 +2023-10-14 05:14:53.013970: train_loss -0.769 +2023-10-14 05:14:53.014121: val_loss -0.6616 +2023-10-14 05:14:53.014241: Pseudo dice [0.9736, 0.8722, 0.8995, 0.9168, 0.8781, 0.6828] +2023-10-14 05:14:53.014324: Epoch time: 342.87 s +2023-10-14 05:14:54.200916: +2023-10-14 05:14:54.201092: Epoch 562 +2023-10-14 05:14:54.201257: Current learning rate: 0.00476 +2023-10-14 05:20:37.013178: train_loss -0.7484 +2023-10-14 05:20:37.013337: val_loss -0.6472 +2023-10-14 05:20:37.013453: Pseudo dice [0.9732, 0.8713, 0.9089, 0.8411, 0.902, 0.6378] +2023-10-14 05:20:37.013537: Epoch time: 342.81 s +2023-10-14 05:20:38.382110: +2023-10-14 05:20:38.382325: Epoch 563 +2023-10-14 05:20:38.382489: Current learning rate: 0.00475 +2023-10-14 05:26:21.098588: train_loss -0.7536 +2023-10-14 05:26:21.098743: val_loss -0.715 +2023-10-14 05:26:21.098846: Pseudo dice [0.9739, 0.8798, 0.9112, 0.8867, 0.8861, 0.6802] +2023-10-14 05:26:21.098930: Epoch time: 342.72 s +2023-10-14 05:26:22.288575: +2023-10-14 05:26:22.288774: Epoch 564 +2023-10-14 05:26:22.288931: Current learning rate: 0.00474 +2023-10-14 05:32:05.020309: train_loss -0.767 +2023-10-14 05:32:05.020479: val_loss -0.7155 +2023-10-14 05:32:05.020589: Pseudo dice [0.9724, 0.8694, 0.8995, 0.874, 0.8677, 0.6408] +2023-10-14 05:32:05.020680: Epoch time: 342.73 s +2023-10-14 05:32:06.212259: +2023-10-14 05:32:06.212493: Epoch 565 +2023-10-14 05:32:06.212600: Current learning rate: 0.00473 +2023-10-14 05:37:48.969366: train_loss -0.7563 +2023-10-14 05:37:48.969503: val_loss -0.6532 +2023-10-14 05:37:48.969620: Pseudo dice [0.9659, 0.8562, 0.9052, 0.8908, 0.9005, 0.6389] +2023-10-14 05:37:48.969706: Epoch time: 342.76 s +2023-10-14 05:37:50.154920: +2023-10-14 05:37:50.155102: Epoch 566 +2023-10-14 05:37:50.155266: Current learning rate: 0.00472 +2023-10-14 05:43:32.973319: train_loss -0.7447 +2023-10-14 05:43:32.973484: val_loss -0.6494 +2023-10-14 05:43:32.973588: Pseudo dice [0.97, 0.8632, 0.891, 0.8522, 0.8895, 0.6533] +2023-10-14 05:43:32.973671: Epoch time: 342.82 s +2023-10-14 05:43:34.156064: +2023-10-14 05:43:34.156256: Epoch 567 +2023-10-14 05:43:34.156412: Current learning rate: 0.00471 +2023-10-14 05:49:17.036438: train_loss -0.7674 +2023-10-14 05:49:17.036594: val_loss -0.6944 +2023-10-14 05:49:17.036695: Pseudo dice [0.9708, 0.8578, 0.9087, 0.8455, 0.8461, 0.6822] +2023-10-14 05:49:17.036779: Epoch time: 342.88 s +2023-10-14 05:49:18.216974: +2023-10-14 05:49:18.217178: Epoch 568 +2023-10-14 05:49:18.217311: Current learning rate: 0.0047 +2023-10-14 05:55:01.030546: train_loss -0.7657 +2023-10-14 05:55:01.030704: val_loss -0.6849 +2023-10-14 05:55:01.030807: Pseudo dice [0.9735, 0.885, 0.9036, 0.891, 0.8849, 0.6917] +2023-10-14 05:55:01.030891: Epoch time: 342.81 s +2023-10-14 05:55:02.215778: +2023-10-14 05:55:02.215883: Epoch 569 +2023-10-14 05:55:02.215994: Current learning rate: 0.00469 +2023-10-14 06:00:45.075301: train_loss -0.7695 +2023-10-14 06:00:45.075438: val_loss -0.6605 +2023-10-14 06:00:45.075554: Pseudo dice [0.9728, 0.8692, 0.9075, 0.8475, 0.8759, 0.6835] +2023-10-14 06:00:45.075638: Epoch time: 342.86 s +2023-10-14 06:00:46.438098: +2023-10-14 06:00:46.438213: Epoch 570 +2023-10-14 06:00:46.438325: Current learning rate: 0.00468 +2023-10-14 06:06:29.286267: train_loss -0.7649 +2023-10-14 06:06:29.286445: val_loss -0.6474 +2023-10-14 06:06:29.286563: Pseudo dice [0.9727, 0.8815, 0.8915, 0.8921, 0.9134, 0.6651] +2023-10-14 06:06:29.286654: Epoch time: 342.85 s +2023-10-14 06:06:30.470122: +2023-10-14 06:06:30.470239: Epoch 571 +2023-10-14 06:06:30.470354: Current learning rate: 0.00467 +2023-10-14 06:12:13.198908: train_loss -0.7617 +2023-10-14 06:12:13.199059: val_loss -0.6886 +2023-10-14 06:12:13.199164: Pseudo dice [0.9693, 0.8729, 0.8989, 0.8834, 0.8732, 0.6082] +2023-10-14 06:12:13.199246: Epoch time: 342.73 s +2023-10-14 06:12:14.390220: +2023-10-14 06:12:14.390425: Epoch 572 +2023-10-14 06:12:14.390596: Current learning rate: 0.00466 +2023-10-14 06:17:57.112628: train_loss -0.7637 +2023-10-14 06:17:57.112778: val_loss -0.6913 +2023-10-14 06:17:57.112897: Pseudo dice [0.9705, 0.8662, 0.9081, 0.8929, 0.8731, 0.6384] +2023-10-14 06:17:57.112982: Epoch time: 342.72 s +2023-10-14 06:17:58.323169: +2023-10-14 06:17:58.323287: Epoch 573 +2023-10-14 06:17:58.323389: Current learning rate: 0.00465 +2023-10-14 06:23:41.072819: train_loss -0.7656 +2023-10-14 06:23:41.072965: val_loss -0.7006 +2023-10-14 06:23:41.073070: Pseudo dice [0.9727, 0.8634, 0.9136, 0.8725, 0.9041, 0.6279] +2023-10-14 06:23:41.073166: Epoch time: 342.75 s +2023-10-14 06:23:42.276344: +2023-10-14 06:23:42.276531: Epoch 574 +2023-10-14 06:23:42.276680: Current learning rate: 0.00464 +2023-10-14 06:29:25.033695: train_loss -0.7757 +2023-10-14 06:29:25.033848: val_loss -0.686 +2023-10-14 06:29:25.033951: Pseudo dice [0.9722, 0.872, 0.9012, 0.8921, 0.8913, 0.6377] +2023-10-14 06:29:25.034034: Epoch time: 342.76 s +2023-10-14 06:29:26.245235: +2023-10-14 06:29:26.245340: Epoch 575 +2023-10-14 06:29:26.245453: Current learning rate: 0.00463 +2023-10-14 06:35:08.919660: train_loss -0.7652 +2023-10-14 06:35:08.919836: val_loss -0.634 +2023-10-14 06:35:08.919938: Pseudo dice [0.9715, 0.8671, 0.8993, 0.8527, 0.8829, 0.6495] +2023-10-14 06:35:08.920021: Epoch time: 342.68 s +2023-10-14 06:35:10.127876: +2023-10-14 06:35:10.128058: Epoch 576 +2023-10-14 06:35:10.128206: Current learning rate: 0.00462 +2023-10-14 06:40:52.806626: train_loss -0.7523 +2023-10-14 06:40:52.806779: val_loss -0.6948 +2023-10-14 06:40:52.806882: Pseudo dice [0.9709, 0.8606, 0.9127, 0.8764, 0.8499, 0.6145] +2023-10-14 06:40:52.806964: Epoch time: 342.68 s +2023-10-14 06:40:54.189775: +2023-10-14 06:40:54.189893: Epoch 577 +2023-10-14 06:40:54.190007: Current learning rate: 0.00461 +2023-10-14 06:46:37.177732: train_loss -0.7696 +2023-10-14 06:46:37.177890: val_loss -0.6938 +2023-10-14 06:46:37.177995: Pseudo dice [0.9734, 0.8834, 0.9185, 0.8937, 0.8912, 0.6519] +2023-10-14 06:46:37.178078: Epoch time: 342.99 s +2023-10-14 06:46:38.383624: +2023-10-14 06:46:38.383826: Epoch 578 +2023-10-14 06:46:38.383929: Current learning rate: 0.0046 +2023-10-14 06:52:21.107105: train_loss -0.7494 +2023-10-14 06:52:21.107255: val_loss -0.6241 +2023-10-14 06:52:21.107372: Pseudo dice [0.9732, 0.8678, 0.8929, 0.8773, 0.8776, 0.6354] +2023-10-14 06:52:21.107456: Epoch time: 342.72 s +2023-10-14 06:52:22.317508: +2023-10-14 06:52:22.317707: Epoch 579 +2023-10-14 06:52:22.317854: Current learning rate: 0.00459 +2023-10-14 06:58:05.165906: train_loss -0.7829 +2023-10-14 06:58:05.166064: val_loss -0.6938 +2023-10-14 06:58:05.166169: Pseudo dice [0.9737, 0.866, 0.9245, 0.9001, 0.8676, 0.6476] +2023-10-14 06:58:05.166256: Epoch time: 342.85 s +2023-10-14 06:58:06.373713: +2023-10-14 06:58:06.373830: Epoch 580 +2023-10-14 06:58:06.373947: Current learning rate: 0.00458 +2023-10-14 07:03:49.042347: train_loss -0.7838 +2023-10-14 07:03:49.042484: val_loss -0.6854 +2023-10-14 07:03:49.042621: Pseudo dice [0.9745, 0.879, 0.899, 0.8829, 0.8555, 0.7189] +2023-10-14 07:03:49.042706: Epoch time: 342.67 s +2023-10-14 07:03:50.246419: +2023-10-14 07:03:50.246645: Epoch 581 +2023-10-14 07:03:50.246805: Current learning rate: 0.00457 +2023-10-14 07:09:32.967226: train_loss -0.7721 +2023-10-14 07:09:32.967376: val_loss -0.671 +2023-10-14 07:09:32.967494: Pseudo dice [0.9718, 0.868, 0.9113, 0.8522, 0.9063, 0.6364] +2023-10-14 07:09:32.967577: Epoch time: 342.72 s +2023-10-14 07:09:34.185448: +2023-10-14 07:09:34.185553: Epoch 582 +2023-10-14 07:09:34.185665: Current learning rate: 0.00456 +2023-10-14 07:15:16.867659: train_loss -0.7696 +2023-10-14 07:15:16.867799: val_loss -0.6894 +2023-10-14 07:15:16.867914: Pseudo dice [0.9709, 0.8879, 0.905, 0.9088, 0.8777, 0.6733] +2023-10-14 07:15:16.867997: Epoch time: 342.68 s +2023-10-14 07:15:18.073358: +2023-10-14 07:15:18.073462: Epoch 583 +2023-10-14 07:15:18.073573: Current learning rate: 0.00455 +2023-10-14 07:21:00.693540: train_loss -0.7685 +2023-10-14 07:21:00.693717: val_loss -0.6877 +2023-10-14 07:21:00.693819: Pseudo dice [0.9733, 0.8759, 0.9122, 0.8364, 0.8706, 0.6334] +2023-10-14 07:21:00.693904: Epoch time: 342.62 s +2023-10-14 07:21:01.896423: +2023-10-14 07:21:01.896641: Epoch 584 +2023-10-14 07:21:01.896797: Current learning rate: 0.00454 +2023-10-14 07:26:44.697519: train_loss -0.7754 +2023-10-14 07:26:44.697668: val_loss -0.6824 +2023-10-14 07:26:44.697770: Pseudo dice [0.9729, 0.8752, 0.9022, 0.8921, 0.8843, 0.6049] +2023-10-14 07:26:44.697854: Epoch time: 342.8 s +2023-10-14 07:26:46.087703: +2023-10-14 07:26:46.087885: Epoch 585 +2023-10-14 07:26:46.088050: Current learning rate: 0.00453 +2023-10-14 07:32:28.881772: train_loss -0.7398 +2023-10-14 07:32:28.881938: val_loss -0.695 +2023-10-14 07:32:28.882039: Pseudo dice [0.9717, 0.8706, 0.9084, 0.8633, 0.8794, 0.6588] +2023-10-14 07:32:28.882121: Epoch time: 342.79 s +2023-10-14 07:32:30.087863: +2023-10-14 07:32:30.087975: Epoch 586 +2023-10-14 07:32:30.088091: Current learning rate: 0.00452 +2023-10-14 07:38:12.763392: train_loss -0.7895 +2023-10-14 07:38:12.763541: val_loss -0.7275 +2023-10-14 07:38:12.763659: Pseudo dice [0.9704, 0.877, 0.9195, 0.8877, 0.9223, 0.6821] +2023-10-14 07:38:12.763741: Epoch time: 342.68 s +2023-10-14 07:38:13.977688: +2023-10-14 07:38:13.977880: Epoch 587 +2023-10-14 07:38:13.978019: Current learning rate: 0.00451 +2023-10-14 07:43:56.861162: train_loss -0.7766 +2023-10-14 07:43:56.861299: val_loss -0.6943 +2023-10-14 07:43:56.861415: Pseudo dice [0.9731, 0.8756, 0.9085, 0.898, 0.9031, 0.6296] +2023-10-14 07:43:56.861500: Epoch time: 342.88 s +2023-10-14 07:43:58.069707: +2023-10-14 07:43:58.069820: Epoch 588 +2023-10-14 07:43:58.069933: Current learning rate: 0.0045 +2023-10-14 07:49:40.874047: train_loss -0.7606 +2023-10-14 07:49:40.874204: val_loss -0.6965 +2023-10-14 07:49:40.874305: Pseudo dice [0.9711, 0.8812, 0.9092, 0.8781, 0.9041, 0.617] +2023-10-14 07:49:40.874402: Epoch time: 342.81 s +2023-10-14 07:49:42.079949: +2023-10-14 07:49:42.080128: Epoch 589 +2023-10-14 07:49:42.080255: Current learning rate: 0.00449 diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_14_08_44_57.txt b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_14_08_44_57.txt new file mode 100755 index 0000000000000000000000000000000000000000..72abb0ae5fbe47f13479fa03fcbcbb4aa9ad1cf6 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_14_08_44_57.txt @@ -0,0 +1,782 @@ + +####################################################################### +Please cite the following paper when using nnU-Net: +Isensee, F., Jaeger, P. F., Kohl, S. A., Petersen, J., & Maier-Hein, K. H. (2021). nnU-Net: a self-configuring method for deep learning-based biomedical image segmentation. Nature methods, 18(2), 203-211. +####################################################################### + + +This is the configuration used by this training: +Configuration name: 3d_fullres + {'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [241.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True} + +These are the global plan.json settings: + {'dataset_name': 'Dataset720_TSPrime', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [241, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 1620.0, 'mean': -38.229164123535156, 'median': -54.0, 'min': -1000.0, 'percentile_00_5': -941.0, 'percentile_99_5': 897.0, 'std': 192.37086486816406}}} + +2023-10-14 08:44:59.641317: unpacking dataset... +2023-10-14 08:45:03.933325: unpacking done... +2023-10-14 08:45:03.934265: do_dummy_2d_data_aug: False +2023-10-14 08:45:03.934828: Using splits from existing split file: ./data/nnUNet_preprocessed/Dataset720_TSPrime/splits_final.json +2023-10-14 08:45:03.955630: The split file contains 5 splits. +2023-10-14 08:45:03.955693: Desired fold for training: 0 +2023-10-14 08:45:03.955763: This split has 48 training and 12 validation cases. +2023-10-14 08:45:26.437840: Unable to plot network architecture: +2023-10-14 08:45:26.438034: failed to execute PosixPath('dot'), make sure the Graphviz executables are on your systems' PATH +2023-10-14 08:45:26.536218: +2023-10-14 08:45:26.536288: Epoch 550 +2023-10-14 08:45:26.536392: Current learning rate: 0.00487 +2023-10-14 08:53:03.674403: train_loss -0.7726 +2023-10-14 08:53:03.700099: val_loss -0.6799 +2023-10-14 08:53:03.700228: Pseudo dice [0.971, 0.8644, 0.908, 0.8821, 0.8677, 0.666] +2023-10-14 08:53:03.700332: Epoch time: 457.14 s +2023-10-14 08:53:05.057412: +2023-10-14 08:53:05.057607: Epoch 551 +2023-10-14 08:53:05.057753: Current learning rate: 0.00486 +2023-10-14 08:58:50.064842: train_loss -0.758 +2023-10-14 08:58:50.064996: val_loss -0.682 +2023-10-14 08:58:50.065178: Pseudo dice [0.9698, 0.87, 0.9137, 0.9011, 0.8763, 0.7155] +2023-10-14 08:58:50.065330: Epoch time: 345.01 s +2023-10-14 08:58:51.281017: +2023-10-14 08:58:51.281138: Epoch 552 +2023-10-14 08:58:51.281244: Current learning rate: 0.00485 +2023-10-14 09:04:36.496282: train_loss -0.7495 +2023-10-14 09:04:36.498847: val_loss -0.6299 +2023-10-14 09:04:36.498962: Pseudo dice [0.9742, 0.8785, 0.8972, 0.8699, 0.8523, 0.6534] +2023-10-14 09:04:36.499053: Epoch time: 345.22 s +2023-10-14 09:04:37.717736: +2023-10-14 09:04:37.717844: Epoch 553 +2023-10-14 09:04:37.717949: Current learning rate: 0.00484 +2023-10-14 09:10:22.975977: train_loss -0.7464 +2023-10-14 09:10:22.976139: val_loss -0.6956 +2023-10-14 09:10:22.976246: Pseudo dice [0.9605, 0.8878, 0.9084, 0.9079, 0.8974, 0.6374] +2023-10-14 09:10:22.976335: Epoch time: 345.26 s +2023-10-14 09:10:24.198349: +2023-10-14 09:10:24.198598: Epoch 554 +2023-10-14 09:10:24.198841: Current learning rate: 0.00484 +2023-10-14 09:16:09.415752: train_loss -0.7741 +2023-10-14 09:16:09.415916: val_loss -0.6677 +2023-10-14 09:16:09.416036: Pseudo dice [0.9697, 0.8769, 0.8922, 0.8684, 0.9019, 0.6411] +2023-10-14 09:16:09.416137: Epoch time: 345.22 s +2023-10-14 09:16:10.620770: +2023-10-14 09:16:10.620962: Epoch 555 +2023-10-14 09:16:10.621224: Current learning rate: 0.00483 +2023-10-14 09:21:55.950917: train_loss -0.7982 +2023-10-14 09:21:55.951100: val_loss -0.6904 +2023-10-14 09:21:55.951208: Pseudo dice [0.9724, 0.8793, 0.9091, 0.8707, 0.878, 0.6579] +2023-10-14 09:21:55.951298: Epoch time: 345.33 s +2023-10-14 09:21:57.192797: +2023-10-14 09:21:57.192911: Epoch 556 +2023-10-14 09:21:57.193016: Current learning rate: 0.00482 +2023-10-14 09:27:42.440943: train_loss -0.7506 +2023-10-14 09:27:42.441109: val_loss -0.6581 +2023-10-14 09:27:42.441239: Pseudo dice [0.9699, 0.8746, 0.9064, 0.9102, 0.8858, 0.6497] +2023-10-14 09:27:42.441343: Epoch time: 345.25 s +2023-10-14 09:27:43.650142: +2023-10-14 09:27:43.650428: Epoch 557 +2023-10-14 09:27:43.650620: Current learning rate: 0.00481 +2023-10-14 09:33:28.773947: train_loss -0.7664 +2023-10-14 09:33:28.774114: val_loss -0.6489 +2023-10-14 09:33:28.774237: Pseudo dice [0.9673, 0.8709, 0.8961, 0.8558, 0.8833, 0.668] +2023-10-14 09:33:28.774338: Epoch time: 345.12 s +2023-10-14 09:33:30.001891: +2023-10-14 09:33:30.002107: Epoch 558 +2023-10-14 09:33:30.002374: Current learning rate: 0.0048 +2023-10-14 09:39:15.284474: train_loss -0.7648 +2023-10-14 09:39:15.284639: val_loss -0.667 +2023-10-14 09:39:15.284746: Pseudo dice [0.9739, 0.8739, 0.9031, 0.9063, 0.8898, 0.6799] +2023-10-14 09:39:15.284836: Epoch time: 345.28 s +2023-10-14 09:39:16.520813: +2023-10-14 09:39:16.520922: Epoch 559 +2023-10-14 09:39:16.521024: Current learning rate: 0.00479 +2023-10-14 09:45:01.838422: train_loss -0.7745 +2023-10-14 09:45:01.838580: val_loss -0.7023 +2023-10-14 09:45:01.838696: Pseudo dice [0.9725, 0.8726, 0.8938, 0.9313, 0.8859, 0.6822] +2023-10-14 09:45:01.838784: Epoch time: 345.32 s +2023-10-14 09:45:03.072559: +2023-10-14 09:45:03.072677: Epoch 560 +2023-10-14 09:45:03.072782: Current learning rate: 0.00478 +2023-10-14 09:50:48.482734: train_loss -0.7542 +2023-10-14 09:50:48.482884: val_loss -0.6193 +2023-10-14 09:50:48.483016: Pseudo dice [0.9709, 0.8778, 0.8924, 0.8774, 0.9118, 0.6265] +2023-10-14 09:50:48.483118: Epoch time: 345.41 s +2023-10-14 09:50:49.873660: +2023-10-14 09:50:49.873856: Epoch 561 +2023-10-14 09:50:49.874034: Current learning rate: 0.00477 +2023-10-14 09:56:35.256728: train_loss -0.7759 +2023-10-14 09:56:35.256931: val_loss -0.6749 +2023-10-14 09:56:35.257066: Pseudo dice [0.9724, 0.8705, 0.8941, 0.8893, 0.8904, 0.6678] +2023-10-14 09:56:35.257167: Epoch time: 345.38 s +2023-10-14 09:56:36.487928: +2023-10-14 09:56:36.488053: Epoch 562 +2023-10-14 09:56:36.488159: Current learning rate: 0.00476 +2023-10-14 10:02:21.787369: train_loss -0.7461 +2023-10-14 10:02:21.787533: val_loss -0.6248 +2023-10-14 10:02:21.788099: Pseudo dice [0.9743, 0.8629, 0.8877, 0.8809, 0.8345, 0.6541] +2023-10-14 10:02:21.788211: Epoch time: 345.3 s +2023-10-14 10:02:23.005492: +2023-10-14 10:02:23.005673: Epoch 563 +2023-10-14 10:02:23.005828: Current learning rate: 0.00475 +2023-10-14 10:08:08.231398: train_loss -0.7442 +2023-10-14 10:08:08.231559: val_loss -0.6617 +2023-10-14 10:08:08.231754: Pseudo dice [0.9706, 0.8615, 0.9109, 0.8754, 0.8734, 0.6588] +2023-10-14 10:08:08.231893: Epoch time: 345.23 s +2023-10-14 10:08:09.463090: +2023-10-14 10:08:09.463255: Epoch 564 +2023-10-14 10:08:09.463451: Current learning rate: 0.00474 +2023-10-14 10:13:54.732506: train_loss -0.7684 +2023-10-14 10:13:54.732661: val_loss -0.6995 +2023-10-14 10:13:54.732769: Pseudo dice [0.9735, 0.8821, 0.9084, 0.8772, 0.9093, 0.6494] +2023-10-14 10:13:54.732857: Epoch time: 345.27 s +2023-10-14 10:13:55.953084: +2023-10-14 10:13:55.953255: Epoch 565 +2023-10-14 10:13:55.953446: Current learning rate: 0.00473 +2023-10-14 10:19:41.225027: train_loss -0.7761 +2023-10-14 10:19:41.225191: val_loss -0.683 +2023-10-14 10:19:41.225320: Pseudo dice [0.971, 0.8826, 0.91, 0.9083, 0.9002, 0.7113] +2023-10-14 10:19:41.225426: Epoch time: 345.27 s +2023-10-14 10:19:42.462402: +2023-10-14 10:19:42.462528: Epoch 566 +2023-10-14 10:19:42.462646: Current learning rate: 0.00472 +2023-10-14 10:25:27.782266: train_loss -0.7355 +2023-10-14 10:25:27.782458: val_loss -0.7007 +2023-10-14 10:25:27.782598: Pseudo dice [0.9663, 0.8682, 0.9063, 0.8794, 0.8787, 0.6951] +2023-10-14 10:25:27.782697: Epoch time: 345.32 s +2023-10-14 10:25:28.998262: +2023-10-14 10:25:28.998374: Epoch 567 +2023-10-14 10:25:28.998532: Current learning rate: 0.00471 +2023-10-14 10:31:14.337671: train_loss -0.7735 +2023-10-14 10:31:14.337815: val_loss -0.7058 +2023-10-14 10:31:14.337922: Pseudo dice [0.9742, 0.879, 0.9126, 0.883, 0.8763, 0.6931] +2023-10-14 10:31:14.338011: Epoch time: 345.34 s +2023-10-14 10:31:15.750798: +2023-10-14 10:31:15.750981: Epoch 568 +2023-10-14 10:31:15.751223: Current learning rate: 0.0047 +2023-10-14 10:37:01.195973: train_loss -0.7652 +2023-10-14 10:37:01.196126: val_loss -0.7085 +2023-10-14 10:37:01.196294: Pseudo dice [0.9726, 0.8601, 0.9137, 0.8982, 0.8773, 0.6364] +2023-10-14 10:37:01.196413: Epoch time: 345.45 s +2023-10-14 10:37:02.437130: +2023-10-14 10:37:02.437374: Epoch 569 +2023-10-14 10:37:02.437515: Current learning rate: 0.00469 +2023-10-14 10:42:47.777742: train_loss -0.7729 +2023-10-14 10:42:47.777902: val_loss -0.6988 +2023-10-14 10:42:47.778010: Pseudo dice [0.9685, 0.8696, 0.9031, 0.8781, 0.8985, 0.6406] +2023-10-14 10:42:47.778097: Epoch time: 345.34 s +2023-10-14 10:42:49.021302: +2023-10-14 10:42:49.021410: Epoch 570 +2023-10-14 10:42:49.021516: Current learning rate: 0.00468 +2023-10-14 10:48:34.305914: train_loss -0.7413 +2023-10-14 10:48:34.306067: val_loss -0.7214 +2023-10-14 10:48:34.306176: Pseudo dice [0.9726, 0.8777, 0.9129, 0.9223, 0.8883, 0.6428] +2023-10-14 10:48:34.306266: Epoch time: 345.29 s +2023-10-14 10:48:35.529996: +2023-10-14 10:48:35.530115: Epoch 571 +2023-10-14 10:48:35.530219: Current learning rate: 0.00467 +2023-10-14 10:54:20.924050: train_loss -0.7771 +2023-10-14 10:54:20.924205: val_loss -0.6726 +2023-10-14 10:54:20.924314: Pseudo dice [0.972, 0.871, 0.9062, 0.9067, 0.8683, 0.6494] +2023-10-14 10:54:20.924403: Epoch time: 345.39 s +2023-10-14 10:54:22.148553: +2023-10-14 10:54:22.148817: Epoch 572 +2023-10-14 10:54:22.149071: Current learning rate: 0.00466 +2023-10-14 11:00:07.467053: train_loss -0.7492 +2023-10-14 11:00:07.467210: val_loss -0.6786 +2023-10-14 11:00:07.467318: Pseudo dice [0.9718, 0.8647, 0.9095, 0.8812, 0.8904, 0.6684] +2023-10-14 11:00:07.467407: Epoch time: 345.32 s +2023-10-14 11:00:09.043088: +2023-10-14 11:00:09.043208: Epoch 573 +2023-10-14 11:00:09.043312: Current learning rate: 0.00465 +2023-10-14 11:05:54.443021: train_loss -0.7607 +2023-10-14 11:05:54.443180: val_loss -0.7221 +2023-10-14 11:05:54.443288: Pseudo dice [0.9718, 0.8747, 0.9108, 0.8771, 0.8841, 0.6492] +2023-10-14 11:05:54.443378: Epoch time: 345.4 s +2023-10-14 11:05:55.779569: +2023-10-14 11:05:55.779737: Epoch 574 +2023-10-14 11:05:55.779896: Current learning rate: 0.00464 +2023-10-14 11:11:41.234959: train_loss -0.7702 +2023-10-14 11:11:41.235169: val_loss -0.7058 +2023-10-14 11:11:41.235277: Pseudo dice [0.9711, 0.8677, 0.9159, 0.8803, 0.8847, 0.6971] +2023-10-14 11:11:41.235377: Epoch time: 345.46 s +2023-10-14 11:11:42.492971: +2023-10-14 11:11:42.493136: Epoch 575 +2023-10-14 11:11:42.493296: Current learning rate: 0.00463 +2023-10-14 11:17:27.932884: train_loss -0.7933 +2023-10-14 11:17:27.933038: val_loss -0.6719 +2023-10-14 11:17:27.933154: Pseudo dice [0.9695, 0.8583, 0.9105, 0.8701, 0.896, 0.6404] +2023-10-14 11:17:27.933241: Epoch time: 345.44 s +2023-10-14 11:17:29.175566: +2023-10-14 11:17:29.175782: Epoch 576 +2023-10-14 11:17:29.175936: Current learning rate: 0.00462 +2023-10-14 11:23:14.508476: train_loss -0.7557 +2023-10-14 11:23:14.508634: val_loss -0.678 +2023-10-14 11:23:14.508741: Pseudo dice [0.9744, 0.8663, 0.9076, 0.8893, 0.9128, 0.6918] +2023-10-14 11:23:14.508830: Epoch time: 345.33 s +2023-10-14 11:23:15.748343: +2023-10-14 11:23:15.750563: Epoch 577 +2023-10-14 11:23:15.750682: Current learning rate: 0.00461 +2023-10-14 11:29:01.167244: train_loss -0.7662 +2023-10-14 11:29:01.167400: val_loss -0.6888 +2023-10-14 11:29:01.167508: Pseudo dice [0.9724, 0.8683, 0.9002, 0.8689, 0.8669, 0.6766] +2023-10-14 11:29:01.167598: Epoch time: 345.42 s +2023-10-14 11:29:02.412012: +2023-10-14 11:29:02.412133: Epoch 578 +2023-10-14 11:29:02.412236: Current learning rate: 0.0046 +2023-10-14 11:34:47.791409: train_loss -0.7781 +2023-10-14 11:34:47.791618: val_loss -0.6522 +2023-10-14 11:34:47.791725: Pseudo dice [0.9755, 0.8814, 0.8988, 0.8996, 0.8768, 0.6853] +2023-10-14 11:34:47.791815: Epoch time: 345.38 s +2023-10-14 11:34:49.055247: +2023-10-14 11:34:49.055444: Epoch 579 +2023-10-14 11:34:49.055591: Current learning rate: 0.00459 +2023-10-14 11:40:34.502784: train_loss -0.7883 +2023-10-14 11:40:34.502934: val_loss -0.6966 +2023-10-14 11:40:34.503041: Pseudo dice [0.9716, 0.8709, 0.9169, 0.8648, 0.8968, 0.6263] +2023-10-14 11:40:34.503128: Epoch time: 345.45 s +2023-10-14 11:40:35.741698: +2023-10-14 11:40:35.741818: Epoch 580 +2023-10-14 11:40:35.741922: Current learning rate: 0.00458 +2023-10-14 11:46:21.154979: train_loss -0.776 +2023-10-14 11:46:21.155136: val_loss -0.6672 +2023-10-14 11:46:21.155242: Pseudo dice [0.9714, 0.8654, 0.9134, 0.832, 0.8995, 0.6908] +2023-10-14 11:46:21.155330: Epoch time: 345.41 s +2023-10-14 11:46:22.391256: +2023-10-14 11:46:22.391378: Epoch 581 +2023-10-14 11:46:22.391483: Current learning rate: 0.00457 +2023-10-14 11:52:07.812470: train_loss -0.7667 +2023-10-14 11:52:07.812626: val_loss -0.6644 +2023-10-14 11:52:07.812735: Pseudo dice [0.9703, 0.8694, 0.9021, 0.9013, 0.9243, 0.6478] +2023-10-14 11:52:07.812823: Epoch time: 345.42 s +2023-10-14 11:52:09.212539: +2023-10-14 11:52:09.212670: Epoch 582 +2023-10-14 11:52:09.212773: Current learning rate: 0.00456 +2023-10-14 11:57:54.671791: train_loss -0.7335 +2023-10-14 11:57:54.671950: val_loss -0.674 +2023-10-14 11:57:54.672076: Pseudo dice [0.9707, 0.8749, 0.9071, 0.8735, 0.8597, 0.6389] +2023-10-14 11:57:54.672175: Epoch time: 345.46 s +2023-10-14 11:57:55.910544: +2023-10-14 11:57:55.910660: Epoch 583 +2023-10-14 11:57:55.910774: Current learning rate: 0.00455 +2023-10-14 12:03:41.224444: train_loss -0.7809 +2023-10-14 12:03:41.224600: val_loss -0.6982 +2023-10-14 12:03:41.224707: Pseudo dice [0.9737, 0.8681, 0.91, 0.8818, 0.8758, 0.6505] +2023-10-14 12:03:41.224799: Epoch time: 345.31 s +2023-10-14 12:03:42.465361: +2023-10-14 12:03:42.465484: Epoch 584 +2023-10-14 12:03:42.465588: Current learning rate: 0.00454 +2023-10-14 12:09:27.768269: train_loss -0.7659 +2023-10-14 12:09:27.768430: val_loss -0.6903 +2023-10-14 12:09:27.768552: Pseudo dice [0.9717, 0.8757, 0.9116, 0.8827, 0.8703, 0.6597] +2023-10-14 12:09:27.768653: Epoch time: 345.3 s +2023-10-14 12:09:29.019685: +2023-10-14 12:09:29.019790: Epoch 585 +2023-10-14 12:09:29.019923: Current learning rate: 0.00453 +2023-10-14 12:15:14.337044: train_loss -0.78 +2023-10-14 12:15:14.337203: val_loss -0.6906 +2023-10-14 12:15:14.337310: Pseudo dice [0.9702, 0.8768, 0.8986, 0.8672, 0.8984, 0.6971] +2023-10-14 12:15:14.337398: Epoch time: 345.32 s +2023-10-14 12:15:15.580557: +2023-10-14 12:15:15.580729: Epoch 586 +2023-10-14 12:15:15.580879: Current learning rate: 0.00452 +2023-10-14 12:21:00.794588: train_loss -0.7613 +2023-10-14 12:21:00.794739: val_loss -0.6561 +2023-10-14 12:21:00.794865: Pseudo dice [0.9704, 0.869, 0.8913, 0.86, 0.8707, 0.6004] +2023-10-14 12:21:00.794974: Epoch time: 345.21 s +2023-10-14 12:21:02.047369: +2023-10-14 12:21:02.047487: Epoch 587 +2023-10-14 12:21:02.047603: Current learning rate: 0.00451 +2023-10-14 12:26:47.170517: train_loss -0.7592 +2023-10-14 12:26:47.170678: val_loss -0.7093 +2023-10-14 12:26:47.170804: Pseudo dice [0.9738, 0.8839, 0.908, 0.8469, 0.8882, 0.6799] +2023-10-14 12:26:47.170903: Epoch time: 345.12 s +2023-10-14 12:26:48.454413: +2023-10-14 12:26:48.454673: Epoch 588 +2023-10-14 12:26:48.454905: Current learning rate: 0.0045 +2023-10-14 12:32:33.824881: train_loss -0.7433 +2023-10-14 12:32:33.825027: val_loss -0.7018 +2023-10-14 12:32:33.825155: Pseudo dice [0.9762, 0.8921, 0.9013, 0.8873, 0.8627, 0.6741] +2023-10-14 12:32:33.825258: Epoch time: 345.37 s +2023-10-14 12:32:35.246654: +2023-10-14 12:32:35.246777: Epoch 589 +2023-10-14 12:32:35.246901: Current learning rate: 0.00449 +2023-10-14 12:38:20.571609: train_loss -0.7588 +2023-10-14 12:38:20.571753: val_loss -0.6777 +2023-10-14 12:38:20.571877: Pseudo dice [0.9714, 0.8796, 0.9018, 0.9064, 0.8735, 0.662] +2023-10-14 12:38:20.571964: Epoch time: 345.33 s +2023-10-14 12:38:21.826131: +2023-10-14 12:38:21.826255: Epoch 590 +2023-10-14 12:38:21.826372: Current learning rate: 0.00448 +2023-10-14 12:44:07.192964: train_loss -0.7657 +2023-10-14 12:44:07.193155: val_loss -0.6732 +2023-10-14 12:44:07.193263: Pseudo dice [0.9743, 0.8746, 0.9044, 0.879, 0.9048, 0.6682] +2023-10-14 12:44:07.193352: Epoch time: 345.37 s +2023-10-14 12:44:08.441660: +2023-10-14 12:44:08.441845: Epoch 591 +2023-10-14 12:44:08.441998: Current learning rate: 0.00447 +2023-10-14 12:49:53.791086: train_loss -0.7813 +2023-10-14 12:49:53.791235: val_loss -0.6501 +2023-10-14 12:49:53.791344: Pseudo dice [0.9734, 0.8784, 0.8948, 0.8317, 0.8887, 0.6691] +2023-10-14 12:49:53.791434: Epoch time: 345.35 s +2023-10-14 12:49:55.040131: +2023-10-14 12:49:55.040316: Epoch 592 +2023-10-14 12:49:55.040468: Current learning rate: 0.00446 +2023-10-14 12:55:40.360149: train_loss -0.7668 +2023-10-14 12:55:40.360301: val_loss -0.71 +2023-10-14 12:55:40.360408: Pseudo dice [0.9715, 0.8831, 0.9131, 0.89, 0.9071, 0.6549] +2023-10-14 12:55:40.360495: Epoch time: 345.32 s +2023-10-14 12:55:41.608411: +2023-10-14 12:55:41.608523: Epoch 593 +2023-10-14 12:55:41.608628: Current learning rate: 0.00445 +2023-10-14 13:01:27.035468: train_loss -0.7672 +2023-10-14 13:01:27.035642: val_loss -0.695 +2023-10-14 13:01:27.035750: Pseudo dice [0.9729, 0.8742, 0.9165, 0.8934, 0.8787, 0.6867] +2023-10-14 13:01:27.035839: Epoch time: 345.43 s +2023-10-14 13:01:28.273444: +2023-10-14 13:01:28.273566: Epoch 594 +2023-10-14 13:01:28.273671: Current learning rate: 0.00444 +2023-10-14 13:07:13.489238: train_loss -0.7572 +2023-10-14 13:07:13.489397: val_loss -0.6996 +2023-10-14 13:07:13.489504: Pseudo dice [0.973, 0.8829, 0.8863, 0.9056, 0.8796, 0.6745] +2023-10-14 13:07:13.489593: Epoch time: 345.22 s +2023-10-14 13:07:14.721650: +2023-10-14 13:07:14.721843: Epoch 595 +2023-10-14 13:07:14.722018: Current learning rate: 0.00443 +2023-10-14 13:12:59.954200: train_loss -0.77 +2023-10-14 13:12:59.954342: val_loss -0.6637 +2023-10-14 13:12:59.954451: Pseudo dice [0.9695, 0.8752, 0.9112, 0.9069, 0.8798, 0.6646] +2023-10-14 13:12:59.954548: Epoch time: 345.23 s +2023-10-14 13:13:01.391377: +2023-10-14 13:13:01.391502: Epoch 596 +2023-10-14 13:13:01.391612: Current learning rate: 0.00442 +2023-10-14 13:18:46.692570: train_loss -0.7654 +2023-10-14 13:18:46.692728: val_loss -0.7086 +2023-10-14 13:18:46.692838: Pseudo dice [0.9725, 0.8777, 0.9106, 0.8923, 0.8924, 0.6852] +2023-10-14 13:18:46.692928: Epoch time: 345.3 s +2023-10-14 13:18:47.935062: +2023-10-14 13:18:47.935189: Epoch 597 +2023-10-14 13:18:47.935295: Current learning rate: 0.00441 +2023-10-14 13:24:33.156263: train_loss -0.7385 +2023-10-14 13:24:33.156409: val_loss -0.6835 +2023-10-14 13:24:33.156517: Pseudo dice [0.974, 0.8899, 0.8993, 0.8955, 0.8543, 0.6984] +2023-10-14 13:24:33.156605: Epoch time: 345.22 s +2023-10-14 13:24:34.406686: +2023-10-14 13:24:34.406871: Epoch 598 +2023-10-14 13:24:34.407012: Current learning rate: 0.0044 +2023-10-14 13:30:19.651484: train_loss -0.7959 +2023-10-14 13:30:19.651639: val_loss -0.6565 +2023-10-14 13:30:19.651747: Pseudo dice [0.9677, 0.8802, 0.9159, 0.8923, 0.8631, 0.689] +2023-10-14 13:30:19.651837: Epoch time: 345.25 s +2023-10-14 13:30:20.914225: +2023-10-14 13:30:20.914349: Epoch 599 +2023-10-14 13:30:20.914452: Current learning rate: 0.00439 +2023-10-14 13:36:06.187173: train_loss -0.7701 +2023-10-14 13:36:06.187314: val_loss -0.7048 +2023-10-14 13:36:06.187428: Pseudo dice [0.9732, 0.8707, 0.9143, 0.8791, 0.8855, 0.6655] +2023-10-14 13:36:06.187528: Epoch time: 345.27 s +2023-10-14 13:36:09.221221: +2023-10-14 13:36:09.221345: Epoch 600 +2023-10-14 13:36:09.221469: Current learning rate: 0.00438 +2023-10-14 13:41:54.536702: train_loss -0.758 +2023-10-14 13:41:54.562621: val_loss -0.6929 +2023-10-14 13:41:54.562764: Pseudo dice [0.9703, 0.8739, 0.8939, 0.904, 0.9011, 0.673] +2023-10-14 13:41:54.562858: Epoch time: 345.32 s +2023-10-14 13:41:55.812133: +2023-10-14 13:41:55.812340: Epoch 601 +2023-10-14 13:41:55.812508: Current learning rate: 0.00437 +2023-10-14 13:47:41.026841: train_loss -0.7647 +2023-10-14 13:47:41.027003: val_loss -0.7232 +2023-10-14 13:47:41.027112: Pseudo dice [0.9738, 0.8782, 0.9069, 0.8824, 0.8664, 0.6583] +2023-10-14 13:47:41.027201: Epoch time: 345.22 s +2023-10-14 13:47:42.263849: +2023-10-14 13:47:42.264035: Epoch 602 +2023-10-14 13:47:42.264179: Current learning rate: 0.00436 +2023-10-14 13:53:27.466546: train_loss -0.7432 +2023-10-14 13:53:27.466691: val_loss -0.7264 +2023-10-14 13:53:27.466818: Pseudo dice [0.972, 0.8614, 0.9016, 0.8538, 0.8984, 0.6688] +2023-10-14 13:53:27.466918: Epoch time: 345.2 s +2023-10-14 13:53:28.878375: +2023-10-14 13:53:28.878511: Epoch 603 +2023-10-14 13:53:28.878630: Current learning rate: 0.00435 +2023-10-14 13:59:14.072809: train_loss -0.762 +2023-10-14 13:59:14.072963: val_loss -0.691 +2023-10-14 13:59:14.073070: Pseudo dice [0.9733, 0.8664, 0.911, 0.8664, 0.9055, 0.6559] +2023-10-14 13:59:14.073159: Epoch time: 345.2 s +2023-10-14 13:59:15.308695: +2023-10-14 13:59:15.308889: Epoch 604 +2023-10-14 13:59:15.309061: Current learning rate: 0.00434 +2023-10-14 14:05:00.432752: train_loss -0.7551 +2023-10-14 14:05:00.432909: val_loss -0.6737 +2023-10-14 14:05:00.433038: Pseudo dice [0.9732, 0.8841, 0.8865, 0.8843, 0.8986, 0.6903] +2023-10-14 14:05:00.433140: Epoch time: 345.12 s +2023-10-14 14:05:01.668171: +2023-10-14 14:05:01.668294: Epoch 605 +2023-10-14 14:05:01.668412: Current learning rate: 0.00433 +2023-10-14 14:10:46.701825: train_loss -0.7394 +2023-10-14 14:10:46.701970: val_loss -0.6639 +2023-10-14 14:10:46.702087: Pseudo dice [0.9726, 0.8663, 0.9001, 0.882, 0.81, 0.6324] +2023-10-14 14:10:46.702175: Epoch time: 345.03 s +2023-10-14 14:10:47.937988: +2023-10-14 14:10:47.938114: Epoch 606 +2023-10-14 14:10:47.938219: Current learning rate: 0.00432 +2023-10-14 14:16:33.035287: train_loss -0.7721 +2023-10-14 14:16:33.035452: val_loss -0.7028 +2023-10-14 14:16:33.035562: Pseudo dice [0.9675, 0.8681, 0.9084, 0.8931, 0.8772, 0.7123] +2023-10-14 14:16:33.035650: Epoch time: 345.1 s +2023-10-14 14:16:34.287815: +2023-10-14 14:16:34.287935: Epoch 607 +2023-10-14 14:16:34.288050: Current learning rate: 0.00431 +2023-10-14 14:22:19.361040: train_loss -0.7542 +2023-10-14 14:22:19.361205: val_loss -0.6814 +2023-10-14 14:22:19.361313: Pseudo dice [0.9678, 0.8706, 0.8949, 0.8421, 0.8567, 0.639] +2023-10-14 14:22:19.361403: Epoch time: 345.07 s +2023-10-14 14:22:20.597335: +2023-10-14 14:22:20.597454: Epoch 608 +2023-10-14 14:22:20.597558: Current learning rate: 0.0043 +2023-10-14 14:28:05.865697: train_loss -0.7874 +2023-10-14 14:28:05.865849: val_loss -0.689 +2023-10-14 14:28:05.865958: Pseudo dice [0.9689, 0.8736, 0.8951, 0.8791, 0.9071, 0.6724] +2023-10-14 14:28:05.866059: Epoch time: 345.27 s +2023-10-14 14:28:07.096414: +2023-10-14 14:28:07.096534: Epoch 609 +2023-10-14 14:28:07.096638: Current learning rate: 0.00429 +2023-10-14 14:33:52.213276: train_loss -0.7678 +2023-10-14 14:33:52.213439: val_loss -0.6511 +2023-10-14 14:33:52.213578: Pseudo dice [0.9682, 0.8618, 0.8946, 0.8588, 0.852, 0.6899] +2023-10-14 14:33:52.213680: Epoch time: 345.12 s +2023-10-14 14:33:53.617540: +2023-10-14 14:33:53.617666: Epoch 610 +2023-10-14 14:33:53.617785: Current learning rate: 0.00429 +2023-10-14 14:39:38.881116: train_loss -0.7615 +2023-10-14 14:39:38.881281: val_loss -0.685 +2023-10-14 14:39:38.881410: Pseudo dice [0.9724, 0.875, 0.9019, 0.9056, 0.913, 0.686] +2023-10-14 14:39:38.881510: Epoch time: 345.26 s +2023-10-14 14:39:40.115401: +2023-10-14 14:39:40.115527: Epoch 611 +2023-10-14 14:39:40.115643: Current learning rate: 0.00428 +2023-10-14 14:45:25.545999: train_loss -0.7808 +2023-10-14 14:45:25.546160: val_loss -0.7257 +2023-10-14 14:45:25.546287: Pseudo dice [0.973, 0.8702, 0.9215, 0.8727, 0.8979, 0.638] +2023-10-14 14:45:25.546388: Epoch time: 345.43 s +2023-10-14 14:45:26.776140: +2023-10-14 14:45:26.776267: Epoch 612 +2023-10-14 14:45:26.776371: Current learning rate: 0.00427 +2023-10-14 14:51:12.205135: train_loss -0.779 +2023-10-14 14:51:12.205280: val_loss -0.7399 +2023-10-14 14:51:12.205399: Pseudo dice [0.9719, 0.8736, 0.9185, 0.8698, 0.8706, 0.7204] +2023-10-14 14:51:12.205492: Epoch time: 345.43 s +2023-10-14 14:51:13.434371: +2023-10-14 14:51:13.434646: Epoch 613 +2023-10-14 14:51:13.434908: Current learning rate: 0.00426 +2023-10-14 14:56:58.723003: train_loss -0.755 +2023-10-14 14:56:58.723155: val_loss -0.687 +2023-10-14 14:56:58.723260: Pseudo dice [0.9701, 0.8712, 0.9066, 0.897, 0.8999, 0.6792] +2023-10-14 14:56:58.723349: Epoch time: 345.29 s +2023-10-14 14:56:59.948792: +2023-10-14 14:56:59.948977: Epoch 614 +2023-10-14 14:56:59.949131: Current learning rate: 0.00425 +2023-10-14 15:02:45.222680: train_loss -0.7638 +2023-10-14 15:02:45.222840: val_loss -0.678 +2023-10-14 15:02:45.222947: Pseudo dice [0.9662, 0.8619, 0.9091, 0.878, 0.9066, 0.6529] +2023-10-14 15:02:45.223037: Epoch time: 345.27 s +2023-10-14 15:02:46.451826: +2023-10-14 15:02:46.452006: Epoch 615 +2023-10-14 15:02:46.452170: Current learning rate: 0.00424 +2023-10-14 15:08:31.715137: train_loss -0.7744 +2023-10-14 15:08:31.715300: val_loss -0.649 +2023-10-14 15:08:31.715408: Pseudo dice [0.9734, 0.8757, 0.9048, 0.8251, 0.8372, 0.6787] +2023-10-14 15:08:31.715497: Epoch time: 345.26 s +2023-10-14 15:08:32.946714: +2023-10-14 15:08:32.946826: Epoch 616 +2023-10-14 15:08:32.946929: Current learning rate: 0.00423 +2023-10-14 15:14:18.370008: train_loss -0.7348 +2023-10-14 15:14:18.370180: val_loss -0.7027 +2023-10-14 15:14:18.370288: Pseudo dice [0.9722, 0.8598, 0.9081, 0.9042, 0.8572, 0.6507] +2023-10-14 15:14:18.370378: Epoch time: 345.42 s +2023-10-14 15:14:19.774088: +2023-10-14 15:14:19.774227: Epoch 617 +2023-10-14 15:14:19.774343: Current learning rate: 0.00422 +2023-10-14 15:20:05.151113: train_loss -0.7712 +2023-10-14 15:20:05.151275: val_loss -0.6953 +2023-10-14 15:20:05.151402: Pseudo dice [0.9739, 0.8773, 0.9011, 0.8979, 0.8697, 0.6598] +2023-10-14 15:20:05.151501: Epoch time: 345.38 s +2023-10-14 15:20:06.379423: +2023-10-14 15:20:06.379615: Epoch 618 +2023-10-14 15:20:06.379761: Current learning rate: 0.00421 +2023-10-14 15:25:51.732977: train_loss -0.7603 +2023-10-14 15:25:51.733143: val_loss -0.6633 +2023-10-14 15:25:51.733252: Pseudo dice [0.973, 0.8762, 0.8955, 0.8469, 0.8547, 0.6434] +2023-10-14 15:25:51.733340: Epoch time: 345.35 s +2023-10-14 15:25:52.956948: +2023-10-14 15:25:52.957223: Epoch 619 +2023-10-14 15:25:52.957383: Current learning rate: 0.0042 +2023-10-14 15:31:38.207267: train_loss -0.7639 +2023-10-14 15:31:38.207419: val_loss -0.6473 +2023-10-14 15:31:38.207534: Pseudo dice [0.9733, 0.8753, 0.9119, 0.9126, 0.8255, 0.6438] +2023-10-14 15:31:38.207627: Epoch time: 345.25 s +2023-10-14 15:31:39.433022: +2023-10-14 15:31:39.433202: Epoch 620 +2023-10-14 15:31:39.433369: Current learning rate: 0.00419 +2023-10-14 15:37:24.750193: train_loss -0.7808 +2023-10-14 15:37:24.750353: val_loss -0.6758 +2023-10-14 15:37:24.750461: Pseudo dice [0.9724, 0.8676, 0.9047, 0.8751, 0.8983, 0.6504] +2023-10-14 15:37:24.750569: Epoch time: 345.32 s +2023-10-14 15:37:26.024541: +2023-10-14 15:37:26.024662: Epoch 621 +2023-10-14 15:37:26.024769: Current learning rate: 0.00418 +2023-10-14 15:43:11.284546: train_loss -0.7552 +2023-10-14 15:43:11.284713: val_loss -0.6358 +2023-10-14 15:43:11.284839: Pseudo dice [0.9726, 0.8642, 0.8908, 0.8684, 0.9013, 0.6421] +2023-10-14 15:43:11.284938: Epoch time: 345.26 s +2023-10-14 15:43:12.513359: +2023-10-14 15:43:12.513530: Epoch 622 +2023-10-14 15:43:12.513690: Current learning rate: 0.00417 +2023-10-14 15:48:57.758520: train_loss -0.771 +2023-10-14 15:48:57.758674: val_loss -0.6537 +2023-10-14 15:48:57.758806: Pseudo dice [0.9727, 0.8635, 0.9034, 0.8372, 0.8479, 0.6164] +2023-10-14 15:48:57.758916: Epoch time: 345.25 s +2023-10-14 15:48:58.997432: +2023-10-14 15:48:58.997557: Epoch 623 +2023-10-14 15:48:58.997694: Current learning rate: 0.00416 +2023-10-14 15:54:44.139509: train_loss -0.792 +2023-10-14 15:54:44.139672: val_loss -0.6581 +2023-10-14 15:54:44.139781: Pseudo dice [0.9724, 0.8767, 0.9003, 0.8994, 0.8994, 0.6691] +2023-10-14 15:54:44.139871: Epoch time: 345.14 s +2023-10-14 15:54:45.546206: +2023-10-14 15:54:45.546422: Epoch 624 +2023-10-14 15:54:45.546611: Current learning rate: 0.00415 +2023-10-14 16:00:30.779316: train_loss -0.7774 +2023-10-14 16:00:30.779478: val_loss -0.6956 +2023-10-14 16:00:30.779586: Pseudo dice [0.9714, 0.875, 0.9005, 0.8732, 0.8709, 0.6667] +2023-10-14 16:00:30.779676: Epoch time: 345.23 s +2023-10-14 16:00:32.006842: +2023-10-14 16:00:32.006967: Epoch 625 +2023-10-14 16:00:32.007073: Current learning rate: 0.00414 +2023-10-14 16:06:17.287866: train_loss -0.7442 +2023-10-14 16:06:17.288028: val_loss -0.6678 +2023-10-14 16:06:17.288138: Pseudo dice [0.9715, 0.8641, 0.9098, 0.7926, 0.8992, 0.6277] +2023-10-14 16:06:17.288226: Epoch time: 345.28 s +2023-10-14 16:06:18.518711: +2023-10-14 16:06:18.518934: Epoch 626 +2023-10-14 16:06:18.519116: Current learning rate: 0.00413 +2023-10-14 16:12:03.838896: train_loss -0.7532 +2023-10-14 16:12:03.839072: val_loss -0.6503 +2023-10-14 16:12:03.839226: Pseudo dice [0.9714, 0.8681, 0.9088, 0.8553, 0.8588, 0.6468] +2023-10-14 16:12:03.839345: Epoch time: 345.32 s +2023-10-14 16:12:05.069823: +2023-10-14 16:12:05.069941: Epoch 627 +2023-10-14 16:12:05.070088: Current learning rate: 0.00412 +2023-10-14 16:17:50.271927: train_loss -0.7608 +2023-10-14 16:17:50.272089: val_loss -0.7213 +2023-10-14 16:17:50.272195: Pseudo dice [0.9718, 0.8799, 0.9169, 0.8719, 0.8759, 0.6543] +2023-10-14 16:17:50.272283: Epoch time: 345.2 s +2023-10-14 16:17:51.503624: +2023-10-14 16:17:51.503731: Epoch 628 +2023-10-14 16:17:51.503853: Current learning rate: 0.00411 +2023-10-14 16:23:36.594076: train_loss -0.7626 +2023-10-14 16:23:36.594239: val_loss -0.6653 +2023-10-14 16:23:36.594346: Pseudo dice [0.9741, 0.8788, 0.9138, 0.8925, 0.8815, 0.6214] +2023-10-14 16:23:36.594435: Epoch time: 345.09 s +2023-10-14 16:23:37.819789: +2023-10-14 16:23:37.819902: Epoch 629 +2023-10-14 16:23:37.820030: Current learning rate: 0.0041 +2023-10-14 16:29:22.992501: train_loss -0.7483 +2023-10-14 16:29:22.992672: val_loss -0.6787 +2023-10-14 16:29:22.992779: Pseudo dice [0.9755, 0.8821, 0.8981, 0.8838, 0.8583, 0.7084] +2023-10-14 16:29:22.992866: Epoch time: 345.17 s +2023-10-14 16:29:24.221108: +2023-10-14 16:29:24.221229: Epoch 630 +2023-10-14 16:29:24.221332: Current learning rate: 0.00409 +2023-10-14 16:35:09.353429: train_loss -0.7734 +2023-10-14 16:35:09.353594: val_loss -0.6573 +2023-10-14 16:35:09.353700: Pseudo dice [0.9677, 0.8737, 0.8995, 0.9009, 0.9125, 0.6357] +2023-10-14 16:35:09.353797: Epoch time: 345.13 s +2023-10-14 16:35:10.758466: +2023-10-14 16:35:10.758607: Epoch 631 +2023-10-14 16:35:10.758712: Current learning rate: 0.00408 +2023-10-14 16:40:55.876935: train_loss -0.7641 +2023-10-14 16:40:55.877102: val_loss -0.6507 +2023-10-14 16:40:55.877269: Pseudo dice [0.9708, 0.8651, 0.8982, 0.8435, 0.911, 0.6598] +2023-10-14 16:40:55.877389: Epoch time: 345.12 s +2023-10-14 16:40:57.107229: +2023-10-14 16:40:57.107358: Epoch 632 +2023-10-14 16:40:57.107470: Current learning rate: 0.00407 +2023-10-14 16:46:42.215181: train_loss -0.7705 +2023-10-14 16:46:42.215334: val_loss -0.6972 +2023-10-14 16:46:42.215441: Pseudo dice [0.971, 0.8804, 0.8964, 0.8674, 0.9125, 0.6708] +2023-10-14 16:46:42.215530: Epoch time: 345.11 s +2023-10-14 16:46:43.440230: +2023-10-14 16:46:43.440356: Epoch 633 +2023-10-14 16:46:43.440461: Current learning rate: 0.00406 +2023-10-14 16:52:28.556360: train_loss -0.7688 +2023-10-14 16:52:28.556517: val_loss -0.6371 +2023-10-14 16:52:28.556624: Pseudo dice [0.9736, 0.8787, 0.8855, 0.893, 0.8857, 0.6842] +2023-10-14 16:52:28.556714: Epoch time: 345.12 s +2023-10-14 16:52:29.781759: +2023-10-14 16:52:29.781941: Epoch 634 +2023-10-14 16:52:29.782093: Current learning rate: 0.00405 +2023-10-14 16:58:14.835334: train_loss -0.7634 +2023-10-14 16:58:14.835504: val_loss -0.7116 +2023-10-14 16:58:14.835631: Pseudo dice [0.9733, 0.8843, 0.8985, 0.8945, 0.8925, 0.6721] +2023-10-14 16:58:14.835730: Epoch time: 345.05 s +2023-10-14 16:58:16.061664: +2023-10-14 16:58:16.061890: Epoch 635 +2023-10-14 16:58:16.062093: Current learning rate: 0.00404 +2023-10-14 17:04:01.232498: train_loss -0.7566 +2023-10-14 17:04:01.232659: val_loss -0.6579 +2023-10-14 17:04:01.232765: Pseudo dice [0.9712, 0.8645, 0.8988, 0.9094, 0.9212, 0.6647] +2023-10-14 17:04:01.232853: Epoch time: 345.17 s +2023-10-14 17:04:02.460922: +2023-10-14 17:04:02.461122: Epoch 636 +2023-10-14 17:04:02.461257: Current learning rate: 0.00403 +2023-10-14 17:09:47.632440: train_loss -0.7576 +2023-10-14 17:09:47.632618: val_loss -0.6736 +2023-10-14 17:09:47.632725: Pseudo dice [0.9737, 0.8742, 0.9068, 0.8537, 0.9097, 0.671] +2023-10-14 17:09:47.632813: Epoch time: 345.17 s +2023-10-14 17:09:49.027920: +2023-10-14 17:09:49.028105: Epoch 637 +2023-10-14 17:09:49.028272: Current learning rate: 0.00402 +2023-10-14 17:15:34.278707: train_loss -0.7735 +2023-10-14 17:15:34.278879: val_loss -0.6382 +2023-10-14 17:15:34.278991: Pseudo dice [0.9737, 0.8726, 0.9154, 0.8873, 0.8475, 0.6396] +2023-10-14 17:15:34.279081: Epoch time: 345.25 s +2023-10-14 17:15:35.508776: +2023-10-14 17:15:35.508887: Epoch 638 +2023-10-14 17:15:35.508996: Current learning rate: 0.00401 +2023-10-14 17:21:20.734416: train_loss -0.7648 +2023-10-14 17:21:20.734588: val_loss -0.7062 +2023-10-14 17:21:20.734701: Pseudo dice [0.9703, 0.8699, 0.9038, 0.905, 0.9083, 0.6299] +2023-10-14 17:21:20.734789: Epoch time: 345.23 s +2023-10-14 17:21:21.964436: +2023-10-14 17:21:21.964605: Epoch 639 +2023-10-14 17:21:21.964769: Current learning rate: 0.004 +2023-10-14 17:27:07.221666: train_loss -0.7648 +2023-10-14 17:27:07.221833: val_loss -0.6922 +2023-10-14 17:27:07.221941: Pseudo dice [0.9724, 0.8682, 0.9092, 0.8575, 0.9191, 0.6126] +2023-10-14 17:27:07.222028: Epoch time: 345.26 s +2023-10-14 17:27:08.456499: +2023-10-14 17:27:08.456623: Epoch 640 +2023-10-14 17:27:08.456729: Current learning rate: 0.00399 +2023-10-14 17:32:53.649395: train_loss -0.7576 +2023-10-14 17:32:53.649562: val_loss -0.6936 +2023-10-14 17:32:53.649688: Pseudo dice [0.9735, 0.8827, 0.9038, 0.8838, 0.8844, 0.6632] +2023-10-14 17:32:53.649786: Epoch time: 345.19 s +2023-10-14 17:32:54.871769: +2023-10-14 17:32:54.871948: Epoch 641 +2023-10-14 17:32:54.872139: Current learning rate: 0.00398 +2023-10-14 17:38:39.993190: train_loss -0.752 +2023-10-14 17:38:39.993353: val_loss -0.656 +2023-10-14 17:38:39.993459: Pseudo dice [0.9706, 0.873, 0.9123, 0.9178, 0.9002, 0.678] +2023-10-14 17:38:39.993546: Epoch time: 345.12 s +2023-10-14 17:38:41.223131: +2023-10-14 17:38:41.223406: Epoch 642 +2023-10-14 17:38:41.223625: Current learning rate: 0.00397 +2023-10-14 17:44:26.419851: train_loss -0.7631 +2023-10-14 17:44:26.420016: val_loss -0.6677 +2023-10-14 17:44:26.420125: Pseudo dice [0.9716, 0.8758, 0.9048, 0.8369, 0.8966, 0.5712] +2023-10-14 17:44:26.420213: Epoch time: 345.2 s +2023-10-14 17:44:27.645399: +2023-10-14 17:44:27.645557: Epoch 643 +2023-10-14 17:44:27.645711: Current learning rate: 0.00396 +2023-10-14 17:50:12.877445: train_loss -0.7848 +2023-10-14 17:50:12.877609: val_loss -0.6909 +2023-10-14 17:50:12.877716: Pseudo dice [0.9734, 0.8767, 0.8875, 0.9047, 0.8963, 0.6348] +2023-10-14 17:50:12.877804: Epoch time: 345.23 s +2023-10-14 17:50:14.322228: +2023-10-14 17:50:14.322353: Epoch 644 +2023-10-14 17:50:14.322461: Current learning rate: 0.00395 +2023-10-14 17:55:59.667900: train_loss -0.7964 +2023-10-14 17:55:59.668062: val_loss -0.6591 +2023-10-14 17:55:59.668168: Pseudo dice [0.9715, 0.8696, 0.9139, 0.8615, 0.8939, 0.6804] +2023-10-14 17:55:59.668256: Epoch time: 345.35 s +2023-10-14 17:56:00.962695: +2023-10-14 17:56:00.963004: Epoch 645 +2023-10-14 17:56:00.963206: Current learning rate: 0.00394 +2023-10-14 18:01:46.228445: train_loss -0.7786 +2023-10-14 18:01:46.272744: val_loss -0.6726 +2023-10-14 18:01:46.273038: Pseudo dice [0.9693, 0.8762, 0.9047, 0.8573, 0.889, 0.665] +2023-10-14 18:01:46.273135: Epoch time: 345.27 s +2023-10-14 18:01:47.589313: +2023-10-14 18:01:47.589441: Epoch 646 +2023-10-14 18:01:47.589548: Current learning rate: 0.00393 +2023-10-14 18:07:32.782410: train_loss -0.7808 +2023-10-14 18:07:32.806095: val_loss -0.665 +2023-10-14 18:07:32.806214: Pseudo dice [0.9722, 0.8793, 0.9051, 0.8769, 0.888, 0.6735] +2023-10-14 18:07:32.806305: Epoch time: 345.19 s +2023-10-14 18:07:34.099209: +2023-10-14 18:07:34.099390: Epoch 647 +2023-10-14 18:07:34.099658: Current learning rate: 0.00392 +2023-10-14 18:13:19.320418: train_loss -0.7735 +2023-10-14 18:13:19.338535: val_loss -0.72 +2023-10-14 18:13:19.338682: Pseudo dice [0.9723, 0.8818, 0.9029, 0.9075, 0.9213, 0.6385] +2023-10-14 18:13:19.338782: Epoch time: 345.22 s +2023-10-14 18:13:20.618846: +2023-10-14 18:13:20.618957: Epoch 648 +2023-10-14 18:13:20.619078: Current learning rate: 0.00391 +2023-10-14 18:19:05.833261: train_loss -0.7531 +2023-10-14 18:19:05.854315: val_loss -0.7067 +2023-10-14 18:19:05.854431: Pseudo dice [0.9717, 0.8727, 0.9134, 0.8782, 0.9091, 0.6161] +2023-10-14 18:19:05.854533: Epoch time: 345.22 s +2023-10-14 18:19:07.083223: +2023-10-14 18:19:07.083346: Epoch 649 +2023-10-14 18:19:07.083539: Current learning rate: 0.0039 +2023-10-14 18:24:52.414732: train_loss -0.7412 +2023-10-14 18:24:52.414915: val_loss -0.6814 +2023-10-14 18:24:52.415040: Pseudo dice [0.9731, 0.8838, 0.9099, 0.8508, 0.9159, 0.6808] +2023-10-14 18:24:52.415140: Epoch time: 345.33 s +2023-10-14 18:24:55.490115: +2023-10-14 18:24:55.490302: Epoch 650 +2023-10-14 18:24:55.490507: Current learning rate: 0.00389 +2023-10-14 18:30:40.699064: train_loss -0.7826 +2023-10-14 18:30:40.699532: val_loss -0.6758 +2023-10-14 18:30:40.699661: Pseudo dice [0.9737, 0.868, 0.8963, 0.8625, 0.8257, 0.6266] +2023-10-14 18:30:40.699759: Epoch time: 345.21 s +2023-10-14 18:30:42.128822: +2023-10-14 18:30:42.129018: Epoch 651 +2023-10-14 18:30:42.129209: Current learning rate: 0.00388 +2023-10-14 18:36:27.414385: train_loss -0.7857 +2023-10-14 18:36:27.414559: val_loss -0.6537 +2023-10-14 18:36:27.414689: Pseudo dice [0.9745, 0.8709, 0.9013, 0.8435, 0.87, 0.6651] +2023-10-14 18:36:27.414789: Epoch time: 345.29 s +2023-10-14 18:36:28.638571: +2023-10-14 18:36:28.638687: Epoch 652 +2023-10-14 18:36:28.638800: Current learning rate: 0.00387 +2023-10-14 18:42:13.898769: train_loss -0.7566 +2023-10-14 18:42:13.898952: val_loss -0.6672 +2023-10-14 18:42:13.899078: Pseudo dice [0.9721, 0.8805, 0.905, 0.8985, 0.8761, 0.6544] +2023-10-14 18:42:13.899177: Epoch time: 345.26 s +2023-10-14 18:42:15.124807: +2023-10-14 18:42:15.125002: Epoch 653 +2023-10-14 18:42:15.125160: Current learning rate: 0.00386 +2023-10-14 18:48:00.439467: train_loss -0.7553 +2023-10-14 18:48:00.439612: val_loss -0.6673 +2023-10-14 18:48:00.439730: Pseudo dice [0.9724, 0.8704, 0.8984, 0.8762, 0.8767, 0.6696] +2023-10-14 18:48:00.439824: Epoch time: 345.32 s +2023-10-14 18:48:01.674665: +2023-10-14 18:48:01.674914: Epoch 654 +2023-10-14 18:48:01.675084: Current learning rate: 0.00385 +2023-10-14 18:53:46.911062: train_loss -0.7787 +2023-10-14 18:53:46.911207: val_loss -0.6667 +2023-10-14 18:53:46.911326: Pseudo dice [0.9731, 0.8704, 0.9041, 0.8151, 0.8472, 0.6625] +2023-10-14 18:53:46.911428: Epoch time: 345.24 s +2023-10-14 18:53:48.138912: +2023-10-14 18:53:48.139034: Epoch 655 +2023-10-14 18:53:48.139151: Current learning rate: 0.00384 +2023-10-14 18:59:33.442257: train_loss -0.7725 +2023-10-14 18:59:33.442420: val_loss -0.6797 +2023-10-14 18:59:33.442545: Pseudo dice [0.9744, 0.8791, 0.9072, 0.8569, 0.8972, 0.6969] +2023-10-14 18:59:33.442636: Epoch time: 345.3 s +2023-10-14 18:59:34.668129: +2023-10-14 18:59:34.668350: Epoch 656 +2023-10-14 18:59:34.668544: Current learning rate: 0.00383 +2023-10-14 19:05:19.916166: train_loss -0.7891 +2023-10-14 19:05:19.916329: val_loss -0.6891 +2023-10-14 19:05:19.916438: Pseudo dice [0.9716, 0.8722, 0.8968, 0.8606, 0.8814, 0.6483] +2023-10-14 19:05:19.916527: Epoch time: 345.25 s +2023-10-14 19:05:21.143101: +2023-10-14 19:05:21.143220: Epoch 657 +2023-10-14 19:05:21.143395: Current learning rate: 0.00382 +2023-10-14 19:11:06.492922: train_loss -0.748 +2023-10-14 19:11:06.493066: val_loss -0.6908 +2023-10-14 19:11:06.493173: Pseudo dice [0.9607, 0.8645, 0.9061, 0.8701, 0.8748, 0.654] +2023-10-14 19:11:06.493262: Epoch time: 345.35 s +2023-10-14 19:11:08.721048: +2023-10-14 19:11:08.721277: Epoch 658 +2023-10-14 19:11:08.721462: Current learning rate: 0.00381 diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_16_11_52_25.txt b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_16_11_52_25.txt new file mode 100755 index 0000000000000000000000000000000000000000..43f088eb1f20c3939d7f21212f63949009a30455 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/fold_0/training_log_2023_10_16_11_52_25.txt @@ -0,0 +1,887 @@ + +####################################################################### +Please cite the following paper when using nnU-Net: +Isensee, F., Jaeger, P. F., Kohl, S. A., Petersen, J., & Maier-Hein, K. H. (2021). nnU-Net: a self-configuring method for deep learning-based biomedical image segmentation. Nature methods, 18(2), 203-211. +####################################################################### + + +This is the configuration used by this training: +Configuration name: 3d_fullres + {'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [241.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True} + +These are the global plan.json settings: + {'dataset_name': 'Dataset720_TSPrime', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [241, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 1620.0, 'mean': -38.229164123535156, 'median': -54.0, 'min': -1000.0, 'percentile_00_5': -941.0, 'percentile_99_5': 897.0, 'std': 192.37086486816406}}} + +2023-10-16 11:52:36.021841: unpacking dataset... +2023-10-16 11:52:39.807831: unpacking done... +2023-10-16 11:52:39.809407: do_dummy_2d_data_aug: False +2023-10-16 11:52:39.809969: Using splits from existing split file: ./data/nnUNet_preprocessed/Dataset720_TSPrime/splits_final.json +2023-10-16 11:52:39.826854: The split file contains 5 splits. +2023-10-16 11:52:39.826942: Desired fold for training: 0 +2023-10-16 11:52:39.827013: This split has 48 training and 12 validation cases. +2023-10-16 11:53:10.871201: Unable to plot network architecture: +2023-10-16 11:53:10.871277: failed to execute PosixPath('dot'), make sure the Graphviz executables are on your systems' PATH +2023-10-16 11:53:10.957982: +2023-10-16 11:53:10.958041: Epoch 650 +2023-10-16 11:53:10.958157: Current learning rate: 0.00389 +2023-10-16 12:01:33.061845: train_loss -0.7657 +2023-10-16 12:01:33.126682: val_loss -0.6653 +2023-10-16 12:01:33.126856: Pseudo dice [0.9702, 0.8726, 0.8798, 0.9072, 0.9047, 0.6282] +2023-10-16 12:01:33.126974: Epoch time: 502.1 s +2023-10-16 12:01:35.984811: +2023-10-16 12:01:35.985008: Epoch 651 +2023-10-16 12:01:35.985184: Current learning rate: 0.00388 +2023-10-16 12:07:22.139421: train_loss -0.773 +2023-10-16 12:07:22.139565: val_loss -0.6575 +2023-10-16 12:07:22.139684: Pseudo dice [0.9719, 0.8755, 0.8979, 0.8985, 0.9027, 0.6282] +2023-10-16 12:07:22.139770: Epoch time: 346.16 s +2023-10-16 12:07:23.367573: +2023-10-16 12:07:23.367742: Epoch 652 +2023-10-16 12:07:23.367918: Current learning rate: 0.00387 +2023-10-16 12:13:09.660832: train_loss -0.7574 +2023-10-16 12:13:09.660987: val_loss -0.68 +2023-10-16 12:13:09.661093: Pseudo dice [0.9742, 0.8747, 0.9106, 0.8856, 0.8901, 0.6318] +2023-10-16 12:13:09.661180: Epoch time: 346.29 s +2023-10-16 12:13:10.920336: +2023-10-16 12:13:10.920450: Epoch 653 +2023-10-16 12:13:10.920563: Current learning rate: 0.00386 +2023-10-16 12:18:57.220682: train_loss -0.7666 +2023-10-16 12:18:57.220850: val_loss -0.6945 +2023-10-16 12:18:57.220954: Pseudo dice [0.9703, 0.8777, 0.9185, 0.8674, 0.9026, 0.646] +2023-10-16 12:18:57.221044: Epoch time: 346.3 s +2023-10-16 12:18:58.643119: +2023-10-16 12:18:58.643296: Epoch 654 +2023-10-16 12:18:58.643456: Current learning rate: 0.00385 +2023-10-16 12:24:45.117256: train_loss -0.7659 +2023-10-16 12:24:45.117415: val_loss -0.7027 +2023-10-16 12:24:45.117518: Pseudo dice [0.9738, 0.8922, 0.9142, 0.8803, 0.8772, 0.685] +2023-10-16 12:24:45.117602: Epoch time: 346.47 s +2023-10-16 12:24:46.332685: +2023-10-16 12:24:46.332862: Epoch 655 +2023-10-16 12:24:46.333028: Current learning rate: 0.00384 +2023-10-16 12:30:32.836236: train_loss -0.7869 +2023-10-16 12:30:32.836398: val_loss -0.6938 +2023-10-16 12:30:32.836502: Pseudo dice [0.9713, 0.8664, 0.8991, 0.8783, 0.8994, 0.6656] +2023-10-16 12:30:32.836591: Epoch time: 346.5 s +2023-10-16 12:30:34.058037: +2023-10-16 12:30:34.058140: Epoch 656 +2023-10-16 12:30:34.058258: Current learning rate: 0.00383 +2023-10-16 12:36:20.418155: train_loss -0.7692 +2023-10-16 12:36:20.418319: val_loss -0.6525 +2023-10-16 12:36:20.418422: Pseudo dice [0.9751, 0.8812, 0.9, 0.924, 0.8911, 0.6772] +2023-10-16 12:36:20.418515: Epoch time: 346.36 s +2023-10-16 12:36:21.640579: +2023-10-16 12:36:21.640756: Epoch 657 +2023-10-16 12:36:21.640867: Current learning rate: 0.00382 +2023-10-16 12:42:08.193615: train_loss -0.7616 +2023-10-16 12:42:08.193762: val_loss -0.6783 +2023-10-16 12:42:08.193863: Pseudo dice [0.9695, 0.8775, 0.9076, 0.8641, 0.8654, 0.647] +2023-10-16 12:42:08.193949: Epoch time: 346.55 s +2023-10-16 12:42:09.417391: +2023-10-16 12:42:09.417494: Epoch 658 +2023-10-16 12:42:09.417598: Current learning rate: 0.00381 +2023-10-16 12:47:55.878795: train_loss -0.75 +2023-10-16 12:47:55.878945: val_loss -0.7167 +2023-10-16 12:47:55.879050: Pseudo dice [0.971, 0.8696, 0.8976, 0.8648, 0.9152, 0.6271] +2023-10-16 12:47:55.879137: Epoch time: 346.46 s +2023-10-16 12:47:57.113813: +2023-10-16 12:47:57.113915: Epoch 659 +2023-10-16 12:47:57.114023: Current learning rate: 0.0038 +2023-10-16 12:53:43.509501: train_loss -0.7638 +2023-10-16 12:53:43.509643: val_loss -0.6401 +2023-10-16 12:53:43.509757: Pseudo dice [0.9735, 0.8785, 0.9114, 0.8061, 0.8515, 0.6451] +2023-10-16 12:53:43.509848: Epoch time: 346.4 s +2023-10-16 12:53:44.733188: +2023-10-16 12:53:44.733356: Epoch 660 +2023-10-16 12:53:44.733542: Current learning rate: 0.00379 +2023-10-16 12:59:31.136624: train_loss -0.7907 +2023-10-16 12:59:31.136807: val_loss -0.6695 +2023-10-16 12:59:31.136927: Pseudo dice [0.9751, 0.8736, 0.9003, 0.8785, 0.9003, 0.6579] +2023-10-16 12:59:31.137013: Epoch time: 346.4 s +2023-10-16 12:59:32.528379: +2023-10-16 12:59:32.528518: Epoch 661 +2023-10-16 12:59:32.528620: Current learning rate: 0.00378 +2023-10-16 13:05:19.065362: train_loss -0.7425 +2023-10-16 13:05:19.065505: val_loss -0.6758 +2023-10-16 13:05:19.065624: Pseudo dice [0.9717, 0.8713, 0.8965, 0.8627, 0.9088, 0.7013] +2023-10-16 13:05:19.065720: Epoch time: 346.54 s +2023-10-16 13:05:20.282246: +2023-10-16 13:05:20.282420: Epoch 662 +2023-10-16 13:05:20.282581: Current learning rate: 0.00377 +2023-10-16 13:11:06.956717: train_loss -0.7424 +2023-10-16 13:11:06.956860: val_loss -0.656 +2023-10-16 13:11:06.956975: Pseudo dice [0.9719, 0.8685, 0.8859, 0.8873, 0.9081, 0.6718] +2023-10-16 13:11:06.957066: Epoch time: 346.68 s +2023-10-16 13:11:08.177942: +2023-10-16 13:11:08.178053: Epoch 663 +2023-10-16 13:11:08.178155: Current learning rate: 0.00376 +2023-10-16 13:16:55.040390: train_loss -0.7843 +2023-10-16 13:16:55.040555: val_loss -0.6718 +2023-10-16 13:16:55.040658: Pseudo dice [0.9739, 0.8831, 0.9084, 0.8481, 0.9041, 0.7072] +2023-10-16 13:16:55.040744: Epoch time: 346.86 s +2023-10-16 13:16:56.278773: +2023-10-16 13:16:56.279001: Epoch 664 +2023-10-16 13:16:56.279125: Current learning rate: 0.00375 +2023-10-16 13:22:43.218038: train_loss -0.7699 +2023-10-16 13:22:43.218202: val_loss -0.6593 +2023-10-16 13:22:43.218335: Pseudo dice [0.9702, 0.8744, 0.9099, 0.8694, 0.9013, 0.6695] +2023-10-16 13:22:43.218433: Epoch time: 346.94 s +2023-10-16 13:22:44.439598: +2023-10-16 13:22:44.439700: Epoch 665 +2023-10-16 13:22:44.439813: Current learning rate: 0.00374 +2023-10-16 13:28:31.328723: train_loss -0.7893 +2023-10-16 13:28:31.328860: val_loss -0.6805 +2023-10-16 13:28:31.328979: Pseudo dice [0.9727, 0.8743, 0.9098, 0.8652, 0.8763, 0.6501] +2023-10-16 13:28:31.329065: Epoch time: 346.89 s +2023-10-16 13:28:32.552882: +2023-10-16 13:28:32.553001: Epoch 666 +2023-10-16 13:28:32.553102: Current learning rate: 0.00373 +2023-10-16 13:34:19.444855: train_loss -0.8055 +2023-10-16 13:34:19.444997: val_loss -0.714 +2023-10-16 13:34:19.445113: Pseudo dice [0.9722, 0.8727, 0.9127, 0.8944, 0.8632, 0.6484] +2023-10-16 13:34:19.445198: Epoch time: 346.89 s +2023-10-16 13:34:20.666053: +2023-10-16 13:34:20.666156: Epoch 667 +2023-10-16 13:34:20.666275: Current learning rate: 0.00372 +2023-10-16 13:40:07.663087: train_loss -0.7773 +2023-10-16 13:40:07.663230: val_loss -0.6389 +2023-10-16 13:40:07.663347: Pseudo dice [0.9724, 0.8718, 0.8939, 0.8236, 0.8816, 0.6543] +2023-10-16 13:40:07.663431: Epoch time: 347.0 s +2023-10-16 13:40:09.075113: +2023-10-16 13:40:09.075330: Epoch 668 +2023-10-16 13:40:09.075481: Current learning rate: 0.00371 +2023-10-16 13:45:55.930044: train_loss -0.7507 +2023-10-16 13:45:55.930204: val_loss -0.6692 +2023-10-16 13:45:55.930308: Pseudo dice [0.9716, 0.8709, 0.8977, 0.8937, 0.8658, 0.6703] +2023-10-16 13:45:55.930395: Epoch time: 346.86 s +2023-10-16 13:45:57.174367: +2023-10-16 13:45:57.174487: Epoch 669 +2023-10-16 13:45:57.174600: Current learning rate: 0.0037 +2023-10-16 13:51:44.146317: train_loss -0.7798 +2023-10-16 13:51:44.146501: val_loss -0.6813 +2023-10-16 13:51:44.146618: Pseudo dice [0.9723, 0.8781, 0.8978, 0.9021, 0.919, 0.7104] +2023-10-16 13:51:44.146704: Epoch time: 346.97 s +2023-10-16 13:51:45.386658: +2023-10-16 13:51:45.386854: Epoch 670 +2023-10-16 13:51:45.387032: Current learning rate: 0.00369 +2023-10-16 13:57:32.357284: train_loss -0.7779 +2023-10-16 13:57:32.357433: val_loss -0.6658 +2023-10-16 13:57:32.357549: Pseudo dice [0.9709, 0.8515, 0.905, 0.8648, 0.8305, 0.6363] +2023-10-16 13:57:32.357636: Epoch time: 346.97 s +2023-10-16 13:57:33.598728: +2023-10-16 13:57:33.598853: Epoch 671 +2023-10-16 13:57:33.598966: Current learning rate: 0.00368 +2023-10-16 14:03:20.613798: train_loss -0.7601 +2023-10-16 14:03:20.613941: val_loss -0.6489 +2023-10-16 14:03:20.614055: Pseudo dice [0.975, 0.8824, 0.899, 0.8721, 0.8874, 0.65] +2023-10-16 14:03:20.614146: Epoch time: 347.02 s +2023-10-16 14:03:21.852867: +2023-10-16 14:03:21.853044: Epoch 672 +2023-10-16 14:03:21.853190: Current learning rate: 0.00367 +2023-10-16 14:09:08.883238: train_loss -0.7634 +2023-10-16 14:09:08.883381: val_loss -0.706 +2023-10-16 14:09:08.883495: Pseudo dice [0.9726, 0.8749, 0.9215, 0.864, 0.9013, 0.6659] +2023-10-16 14:09:08.883586: Epoch time: 347.03 s +2023-10-16 14:09:10.117944: +2023-10-16 14:09:10.118047: Epoch 673 +2023-10-16 14:09:10.118160: Current learning rate: 0.00366 +2023-10-16 14:14:57.067062: train_loss -0.7912 +2023-10-16 14:14:57.067231: val_loss -0.6643 +2023-10-16 14:14:57.067334: Pseudo dice [0.9719, 0.8677, 0.9038, 0.8649, 0.879, 0.6476] +2023-10-16 14:14:57.067421: Epoch time: 346.95 s +2023-10-16 14:14:58.304619: +2023-10-16 14:14:58.304735: Epoch 674 +2023-10-16 14:14:58.304837: Current learning rate: 0.00365 +2023-10-16 14:20:45.279370: train_loss -0.7615 +2023-10-16 14:20:45.279514: val_loss -0.6747 +2023-10-16 14:20:45.279631: Pseudo dice [0.9713, 0.8788, 0.9164, 0.88, 0.9019, 0.6342] +2023-10-16 14:20:45.279728: Epoch time: 346.98 s +2023-10-16 14:20:46.698153: +2023-10-16 14:20:46.698259: Epoch 675 +2023-10-16 14:20:46.698378: Current learning rate: 0.00364 +2023-10-16 14:26:33.759420: train_loss -0.7831 +2023-10-16 14:26:33.759569: val_loss -0.6407 +2023-10-16 14:26:33.759697: Pseudo dice [0.972, 0.8794, 0.8897, 0.905, 0.8675, 0.6662] +2023-10-16 14:26:33.759787: Epoch time: 347.06 s +2023-10-16 14:26:35.012318: +2023-10-16 14:26:35.012434: Epoch 676 +2023-10-16 14:26:35.012539: Current learning rate: 0.00363 +2023-10-16 14:32:22.167936: train_loss -0.76 +2023-10-16 14:32:22.168096: val_loss -0.6501 +2023-10-16 14:32:22.168233: Pseudo dice [0.9728, 0.874, 0.8996, 0.8746, 0.8343, 0.639] +2023-10-16 14:32:22.168343: Epoch time: 347.16 s +2023-10-16 14:32:23.423537: +2023-10-16 14:32:23.423660: Epoch 677 +2023-10-16 14:32:23.423763: Current learning rate: 0.00362 +2023-10-16 14:38:10.591294: train_loss -0.773 +2023-10-16 14:38:10.591457: val_loss -0.6392 +2023-10-16 14:38:10.591580: Pseudo dice [0.9732, 0.8753, 0.8913, 0.8908, 0.8786, 0.6712] +2023-10-16 14:38:10.591680: Epoch time: 347.17 s +2023-10-16 14:38:11.866211: +2023-10-16 14:38:11.866316: Epoch 678 +2023-10-16 14:38:11.866436: Current learning rate: 0.00361 +2023-10-16 14:43:58.890313: train_loss -0.7868 +2023-10-16 14:43:58.890458: val_loss -0.6914 +2023-10-16 14:43:58.890584: Pseudo dice [0.9726, 0.8752, 0.9006, 0.8883, 0.8705, 0.6923] +2023-10-16 14:43:58.890673: Epoch time: 347.02 s +2023-10-16 14:44:00.139863: +2023-10-16 14:44:00.139964: Epoch 679 +2023-10-16 14:44:00.140081: Current learning rate: 0.0036 +2023-10-16 14:49:47.252280: train_loss -0.7583 +2023-10-16 14:49:47.252435: val_loss -0.6891 +2023-10-16 14:49:47.252540: Pseudo dice [0.9735, 0.8717, 0.9093, 0.8589, 0.9095, 0.6843] +2023-10-16 14:49:47.252628: Epoch time: 347.11 s +2023-10-16 14:49:48.496274: +2023-10-16 14:49:48.496376: Epoch 680 +2023-10-16 14:49:48.496490: Current learning rate: 0.00359 +2023-10-16 14:55:35.546312: train_loss -0.7443 +2023-10-16 14:55:35.546449: val_loss -0.6779 +2023-10-16 14:55:35.546575: Pseudo dice [0.972, 0.8794, 0.906, 0.8671, 0.8294, 0.6668] +2023-10-16 14:55:35.546662: Epoch time: 347.05 s +2023-10-16 14:55:36.785294: +2023-10-16 14:55:36.785476: Epoch 681 +2023-10-16 14:55:36.785625: Current learning rate: 0.00358 +2023-10-16 15:01:23.742646: train_loss -0.7571 +2023-10-16 15:01:23.742798: val_loss -0.7238 +2023-10-16 15:01:23.742901: Pseudo dice [0.9724, 0.8781, 0.9204, 0.8934, 0.8916, 0.6764] +2023-10-16 15:01:23.742988: Epoch time: 346.96 s +2023-10-16 15:01:25.167395: +2023-10-16 15:01:25.167524: Epoch 682 +2023-10-16 15:01:25.167639: Current learning rate: 0.00357 +2023-10-16 15:07:12.223466: train_loss -0.782 +2023-10-16 15:07:12.223630: val_loss -0.6878 +2023-10-16 15:07:12.223733: Pseudo dice [0.9723, 0.8679, 0.9029, 0.8468, 0.8487, 0.6375] +2023-10-16 15:07:12.223820: Epoch time: 347.06 s +2023-10-16 15:07:13.468253: +2023-10-16 15:07:13.468360: Epoch 683 +2023-10-16 15:07:13.468473: Current learning rate: 0.00356 +2023-10-16 15:13:00.568304: train_loss -0.7683 +2023-10-16 15:13:00.568466: val_loss -0.7197 +2023-10-16 15:13:00.568593: Pseudo dice [0.9729, 0.888, 0.9113, 0.9009, 0.9092, 0.7284] +2023-10-16 15:13:00.568692: Epoch time: 347.1 s +2023-10-16 15:13:01.855891: +2023-10-16 15:13:01.855999: Epoch 684 +2023-10-16 15:13:01.856115: Current learning rate: 0.00355 +2023-10-16 15:18:48.879060: train_loss -0.7536 +2023-10-16 15:18:48.879210: val_loss -0.6543 +2023-10-16 15:18:48.879330: Pseudo dice [0.9701, 0.8749, 0.9048, 0.851, 0.9098, 0.6551] +2023-10-16 15:18:48.879416: Epoch time: 347.02 s +2023-10-16 15:18:50.125212: +2023-10-16 15:18:50.125395: Epoch 685 +2023-10-16 15:18:50.125538: Current learning rate: 0.00354 +2023-10-16 15:24:37.183900: train_loss -0.7768 +2023-10-16 15:24:37.184041: val_loss -0.7177 +2023-10-16 15:24:37.184159: Pseudo dice [0.9731, 0.8867, 0.9078, 0.8753, 0.8575, 0.6952] +2023-10-16 15:24:37.184245: Epoch time: 347.06 s +2023-10-16 15:24:38.425323: +2023-10-16 15:24:38.425500: Epoch 686 +2023-10-16 15:24:38.425670: Current learning rate: 0.00353 +2023-10-16 15:30:25.450144: train_loss -0.7616 +2023-10-16 15:30:25.450284: val_loss -0.6811 +2023-10-16 15:30:25.450403: Pseudo dice [0.9726, 0.8773, 0.9077, 0.9011, 0.8914, 0.6842] +2023-10-16 15:30:25.450490: Epoch time: 347.03 s +2023-10-16 15:30:26.691565: +2023-10-16 15:30:26.691684: Epoch 687 +2023-10-16 15:30:26.691785: Current learning rate: 0.00352 +2023-10-16 15:36:13.898718: train_loss -0.746 +2023-10-16 15:36:13.898895: val_loss -0.6756 +2023-10-16 15:36:13.899005: Pseudo dice [0.9733, 0.8742, 0.9142, 0.8642, 0.8651, 0.6719] +2023-10-16 15:36:13.899095: Epoch time: 347.21 s +2023-10-16 15:36:15.144222: +2023-10-16 15:36:15.144325: Epoch 688 +2023-10-16 15:36:15.144439: Current learning rate: 0.00351 +2023-10-16 15:42:02.385429: train_loss -0.755 +2023-10-16 15:42:02.385591: val_loss -0.6726 +2023-10-16 15:42:02.385713: Pseudo dice [0.9694, 0.8737, 0.9169, 0.8593, 0.8885, 0.6376] +2023-10-16 15:42:02.385812: Epoch time: 347.24 s +2023-10-16 15:42:03.798766: +2023-10-16 15:42:03.799072: Epoch 689 +2023-10-16 15:42:03.799268: Current learning rate: 0.0035 +2023-10-16 15:47:50.885578: train_loss -0.7973 +2023-10-16 15:47:50.885720: val_loss -0.7042 +2023-10-16 15:47:50.885838: Pseudo dice [0.9577, 0.8764, 0.9045, 0.9065, 0.9145, 0.6357] +2023-10-16 15:47:50.885922: Epoch time: 347.09 s +2023-10-16 15:47:52.128037: +2023-10-16 15:47:52.128145: Epoch 690 +2023-10-16 15:47:52.128262: Current learning rate: 0.00349 +2023-10-16 15:53:39.254061: train_loss -0.7614 +2023-10-16 15:53:39.254211: val_loss -0.7025 +2023-10-16 15:53:39.254324: Pseudo dice [0.9724, 0.8791, 0.9141, 0.853, 0.8714, 0.6621] +2023-10-16 15:53:39.254413: Epoch time: 347.13 s +2023-10-16 15:53:40.498606: +2023-10-16 15:53:40.498716: Epoch 691 +2023-10-16 15:53:40.498834: Current learning rate: 0.00348 +2023-10-16 15:59:27.492378: train_loss -0.8142 +2023-10-16 15:59:27.492520: val_loss -0.6976 +2023-10-16 15:59:27.492631: Pseudo dice [0.9733, 0.8819, 0.9122, 0.8859, 0.9243, 0.6735] +2023-10-16 15:59:27.492722: Epoch time: 346.99 s +2023-10-16 15:59:28.744697: +2023-10-16 15:59:28.744801: Epoch 692 +2023-10-16 15:59:28.744918: Current learning rate: 0.00346 +2023-10-16 16:05:15.697318: train_loss -0.7856 +2023-10-16 16:05:15.697461: val_loss -0.695 +2023-10-16 16:05:15.697591: Pseudo dice [0.9567, 0.8851, 0.9146, 0.869, 0.8951, 0.6365] +2023-10-16 16:05:15.697677: Epoch time: 346.95 s +2023-10-16 16:05:16.930659: +2023-10-16 16:05:16.930859: Epoch 693 +2023-10-16 16:05:16.930986: Current learning rate: 0.00345 +2023-10-16 16:11:03.948810: train_loss -0.7754 +2023-10-16 16:11:03.948950: val_loss -0.718 +2023-10-16 16:11:03.949069: Pseudo dice [0.9714, 0.8689, 0.8983, 0.8828, 0.8668, 0.6395] +2023-10-16 16:11:03.949156: Epoch time: 347.02 s +2023-10-16 16:11:05.185005: +2023-10-16 16:11:05.185197: Epoch 694 +2023-10-16 16:11:05.185301: Current learning rate: 0.00344 +2023-10-16 16:16:52.270051: train_loss -0.7802 +2023-10-16 16:16:52.270246: val_loss -0.628 +2023-10-16 16:16:52.270350: Pseudo dice [0.97, 0.8711, 0.9118, 0.9183, 0.8963, 0.622] +2023-10-16 16:16:52.270437: Epoch time: 347.09 s +2023-10-16 16:16:53.686908: +2023-10-16 16:16:53.687034: Epoch 695 +2023-10-16 16:16:53.687136: Current learning rate: 0.00343 +2023-10-16 16:22:40.835341: train_loss -0.7637 +2023-10-16 16:22:40.835490: val_loss -0.6938 +2023-10-16 16:22:40.835609: Pseudo dice [0.9748, 0.8836, 0.8923, 0.8937, 0.9091, 0.6934] +2023-10-16 16:22:40.835694: Epoch time: 347.15 s +2023-10-16 16:22:42.072929: +2023-10-16 16:22:42.073036: Epoch 696 +2023-10-16 16:22:42.073146: Current learning rate: 0.00342 +2023-10-16 16:28:29.189310: train_loss -0.7534 +2023-10-16 16:28:29.189445: val_loss -0.673 +2023-10-16 16:28:29.189578: Pseudo dice [0.9704, 0.8706, 0.8972, 0.9129, 0.9055, 0.697] +2023-10-16 16:28:29.189677: Epoch time: 347.12 s +2023-10-16 16:28:30.433985: +2023-10-16 16:28:30.434089: Epoch 697 +2023-10-16 16:28:30.434215: Current learning rate: 0.00341 +2023-10-16 16:34:17.422712: train_loss -0.7803 +2023-10-16 16:34:17.422876: val_loss -0.6951 +2023-10-16 16:34:17.422993: Pseudo dice [0.9715, 0.8701, 0.9001, 0.8812, 0.886, 0.6462] +2023-10-16 16:34:17.423078: Epoch time: 346.99 s +2023-10-16 16:34:18.685070: +2023-10-16 16:34:18.685174: Epoch 698 +2023-10-16 16:34:18.685283: Current learning rate: 0.0034 +2023-10-16 16:40:05.730069: train_loss -0.7805 +2023-10-16 16:40:05.730230: val_loss -0.625 +2023-10-16 16:40:05.730333: Pseudo dice [0.9709, 0.8713, 0.9143, 0.803, 0.9116, 0.6199] +2023-10-16 16:40:05.730420: Epoch time: 347.05 s +2023-10-16 16:40:06.970120: +2023-10-16 16:40:06.970280: Epoch 699 +2023-10-16 16:40:06.970449: Current learning rate: 0.00339 +2023-10-16 16:45:54.016073: train_loss -0.7806 +2023-10-16 16:45:54.016217: val_loss -0.7162 +2023-10-16 16:45:54.016339: Pseudo dice [0.9718, 0.8703, 0.9138, 0.8974, 0.8785, 0.6422] +2023-10-16 16:45:54.016433: Epoch time: 347.05 s +2023-10-16 16:45:56.997331: +2023-10-16 16:45:56.997594: Epoch 700 +2023-10-16 16:45:56.997781: Current learning rate: 0.00338 +2023-10-16 16:51:44.045771: train_loss -0.7833 +2023-10-16 16:51:44.045931: val_loss -0.6933 +2023-10-16 16:51:44.046034: Pseudo dice [0.9723, 0.865, 0.9224, 0.8523, 0.8763, 0.6381] +2023-10-16 16:51:44.046121: Epoch time: 347.05 s +2023-10-16 16:51:45.448343: +2023-10-16 16:51:45.448452: Epoch 701 +2023-10-16 16:51:45.448575: Current learning rate: 0.00337 +2023-10-16 16:57:32.561426: train_loss -0.7571 +2023-10-16 16:57:32.561591: val_loss -0.6655 +2023-10-16 16:57:32.561729: Pseudo dice [0.9711, 0.8732, 0.9, 0.8923, 0.9004, 0.6384] +2023-10-16 16:57:32.561820: Epoch time: 347.11 s +2023-10-16 16:57:33.801441: +2023-10-16 16:57:33.801694: Epoch 702 +2023-10-16 16:57:33.801955: Current learning rate: 0.00336 +2023-10-16 17:03:20.859094: train_loss -0.7672 +2023-10-16 17:03:20.859270: val_loss -0.7034 +2023-10-16 17:03:20.859374: Pseudo dice [0.9721, 0.8795, 0.898, 0.9042, 0.9137, 0.6659] +2023-10-16 17:03:20.859462: Epoch time: 347.06 s +2023-10-16 17:03:22.101246: +2023-10-16 17:03:22.101372: Epoch 703 +2023-10-16 17:03:22.101486: Current learning rate: 0.00335 +2023-10-16 17:09:09.126226: train_loss -0.783 +2023-10-16 17:09:09.126378: val_loss -0.6845 +2023-10-16 17:09:09.126510: Pseudo dice [0.9726, 0.8791, 0.9095, 0.8866, 0.8608, 0.6806] +2023-10-16 17:09:09.126611: Epoch time: 347.03 s +2023-10-16 17:09:10.386247: +2023-10-16 17:09:10.386356: Epoch 704 +2023-10-16 17:09:10.386464: Current learning rate: 0.00334 +2023-10-16 17:14:57.403064: train_loss -0.7716 +2023-10-16 17:14:57.403202: val_loss -0.698 +2023-10-16 17:14:57.403321: Pseudo dice [0.9718, 0.8811, 0.9088, 0.9002, 0.8809, 0.6618] +2023-10-16 17:14:57.403420: Epoch time: 347.02 s +2023-10-16 17:14:58.645834: +2023-10-16 17:14:58.646003: Epoch 705 +2023-10-16 17:14:58.646159: Current learning rate: 0.00333 +2023-10-16 17:20:45.704206: train_loss -0.7503 +2023-10-16 17:20:45.704348: val_loss -0.6847 +2023-10-16 17:20:45.704459: Pseudo dice [0.9698, 0.873, 0.8917, 0.8889, 0.8997, 0.6743] +2023-10-16 17:20:45.704549: Epoch time: 347.06 s +2023-10-16 17:20:46.957640: +2023-10-16 17:20:46.957807: Epoch 706 +2023-10-16 17:20:46.958007: Current learning rate: 0.00332 +2023-10-16 17:26:33.958879: train_loss -0.7752 +2023-10-16 17:26:33.959020: val_loss -0.7045 +2023-10-16 17:26:33.959134: Pseudo dice [0.9722, 0.8739, 0.9183, 0.8454, 0.9018, 0.6716] +2023-10-16 17:26:33.959222: Epoch time: 347.0 s +2023-10-16 17:26:35.206671: +2023-10-16 17:26:35.206788: Epoch 707 +2023-10-16 17:26:35.206922: Current learning rate: 0.00331 +2023-10-16 17:32:22.138206: train_loss -0.7669 +2023-10-16 17:32:22.138348: val_loss -0.7071 +2023-10-16 17:32:22.138467: Pseudo dice [0.9745, 0.8816, 0.8954, 0.8867, 0.8916, 0.6833] +2023-10-16 17:32:22.138563: Epoch time: 346.93 s +2023-10-16 17:32:23.563481: +2023-10-16 17:32:23.563607: Epoch 708 +2023-10-16 17:32:23.563741: Current learning rate: 0.0033 +2023-10-16 17:38:10.497249: train_loss -0.7673 +2023-10-16 17:38:10.497402: val_loss -0.7156 +2023-10-16 17:38:10.497540: Pseudo dice [0.9729, 0.8705, 0.9078, 0.8799, 0.8892, 0.6529] +2023-10-16 17:38:10.497638: Epoch time: 346.93 s +2023-10-16 17:38:11.745688: +2023-10-16 17:38:11.745798: Epoch 709 +2023-10-16 17:38:11.745906: Current learning rate: 0.00329 +2023-10-16 17:43:58.635768: train_loss -0.7809 +2023-10-16 17:43:58.635956: val_loss -0.6491 +2023-10-16 17:43:58.636094: Pseudo dice [0.9702, 0.8701, 0.8895, 0.8521, 0.8554, 0.6343] +2023-10-16 17:43:58.636192: Epoch time: 346.89 s +2023-10-16 17:43:59.878851: +2023-10-16 17:43:59.878956: Epoch 710 +2023-10-16 17:43:59.879080: Current learning rate: 0.00328 +2023-10-16 17:49:46.728398: train_loss -0.7643 +2023-10-16 17:49:46.728543: val_loss -0.7095 +2023-10-16 17:49:46.728656: Pseudo dice [0.9692, 0.8689, 0.9171, 0.8931, 0.9119, 0.6732] +2023-10-16 17:49:46.728746: Epoch time: 346.85 s +2023-10-16 17:49:47.973462: +2023-10-16 17:49:47.973571: Epoch 711 +2023-10-16 17:49:47.973685: Current learning rate: 0.00327 +2023-10-16 17:55:34.924966: train_loss -0.7672 +2023-10-16 17:55:34.925110: val_loss -0.7094 +2023-10-16 17:55:34.925245: Pseudo dice [0.9737, 0.8833, 0.9068, 0.917, 0.8682, 0.6801] +2023-10-16 17:55:34.925343: Epoch time: 346.95 s +2023-10-16 17:55:36.177106: +2023-10-16 17:55:36.177264: Epoch 712 +2023-10-16 17:55:36.177436: Current learning rate: 0.00326 +2023-10-16 18:01:22.954619: train_loss -0.7593 +2023-10-16 18:01:22.954769: val_loss -0.6383 +2023-10-16 18:01:22.954875: Pseudo dice [0.9728, 0.8743, 0.9015, 0.8736, 0.8608, 0.6368] +2023-10-16 18:01:22.954967: Epoch time: 346.78 s +2023-10-16 18:01:24.252601: +2023-10-16 18:01:24.252719: Epoch 713 +2023-10-16 18:01:24.252821: Current learning rate: 0.00325 +2023-10-16 18:07:11.209710: train_loss -0.7669 +2023-10-16 18:07:11.209862: val_loss -0.6773 +2023-10-16 18:07:11.209980: Pseudo dice [0.9705, 0.876, 0.9022, 0.9134, 0.895, 0.6792] +2023-10-16 18:07:11.210068: Epoch time: 346.96 s +2023-10-16 18:07:12.472027: +2023-10-16 18:07:12.472130: Epoch 714 +2023-10-16 18:07:12.472245: Current learning rate: 0.00324 +2023-10-16 18:12:59.421642: train_loss -0.7703 +2023-10-16 18:12:59.421783: val_loss -0.6806 +2023-10-16 18:12:59.421922: Pseudo dice [0.9727, 0.875, 0.9039, 0.8918, 0.8685, 0.6262] +2023-10-16 18:12:59.422021: Epoch time: 346.95 s +2023-10-16 18:13:00.847863: +2023-10-16 18:13:00.847979: Epoch 715 +2023-10-16 18:13:00.848104: Current learning rate: 0.00323 +2023-10-16 18:18:47.766829: train_loss -0.7554 +2023-10-16 18:18:47.766988: val_loss -0.629 +2023-10-16 18:18:47.767092: Pseudo dice [0.9728, 0.8666, 0.8996, 0.8588, 0.8688, 0.6596] +2023-10-16 18:18:47.767176: Epoch time: 346.92 s +2023-10-16 18:18:49.017198: +2023-10-16 18:18:49.017307: Epoch 716 +2023-10-16 18:18:49.017418: Current learning rate: 0.00322 +2023-10-16 18:24:35.870587: train_loss -0.7733 +2023-10-16 18:24:35.870733: val_loss -0.6543 +2023-10-16 18:24:35.870835: Pseudo dice [0.9709, 0.8719, 0.9111, 0.8689, 0.9133, 0.6523] +2023-10-16 18:24:35.870922: Epoch time: 346.85 s +2023-10-16 18:24:37.124043: +2023-10-16 18:24:37.124146: Epoch 717 +2023-10-16 18:24:37.124261: Current learning rate: 0.00321 +2023-10-16 18:30:23.957323: train_loss -0.77 +2023-10-16 18:30:23.957465: val_loss -0.6474 +2023-10-16 18:30:23.957584: Pseudo dice [0.9735, 0.8619, 0.9143, 0.8353, 0.8954, 0.6592] +2023-10-16 18:30:23.957669: Epoch time: 346.83 s +2023-10-16 18:30:25.211150: +2023-10-16 18:30:25.211277: Epoch 718 +2023-10-16 18:30:25.211446: Current learning rate: 0.0032 +2023-10-16 18:36:12.018801: train_loss -0.7782 +2023-10-16 18:36:12.018966: val_loss -0.6482 +2023-10-16 18:36:12.019079: Pseudo dice [0.9726, 0.8791, 0.8996, 0.884, 0.9021, 0.7036] +2023-10-16 18:36:12.019172: Epoch time: 346.81 s +2023-10-16 18:36:13.263700: +2023-10-16 18:36:13.263804: Epoch 719 +2023-10-16 18:36:13.263930: Current learning rate: 0.00319 +2023-10-16 18:42:00.188163: train_loss -0.7888 +2023-10-16 18:42:00.188304: val_loss -0.7018 +2023-10-16 18:42:00.188424: Pseudo dice [0.9712, 0.87, 0.904, 0.8874, 0.8894, 0.638] +2023-10-16 18:42:00.188511: Epoch time: 346.93 s +2023-10-16 18:42:01.446100: +2023-10-16 18:42:01.446201: Epoch 720 +2023-10-16 18:42:01.446325: Current learning rate: 0.00318 +2023-10-16 18:47:48.306199: train_loss -0.7908 +2023-10-16 18:47:48.306342: val_loss -0.715 +2023-10-16 18:47:48.306471: Pseudo dice [0.9722, 0.8798, 0.9101, 0.8619, 0.8633, 0.6799] +2023-10-16 18:47:48.306579: Epoch time: 346.86 s +2023-10-16 18:47:49.570007: +2023-10-16 18:47:49.570175: Epoch 721 +2023-10-16 18:47:49.570375: Current learning rate: 0.00317 +2023-10-16 18:53:36.487244: train_loss -0.7834 +2023-10-16 18:53:36.487411: val_loss -0.7024 +2023-10-16 18:53:36.487536: Pseudo dice [0.9739, 0.8812, 0.9163, 0.8823, 0.9016, 0.6657] +2023-10-16 18:53:36.487633: Epoch time: 346.92 s +2023-10-16 18:53:37.910301: +2023-10-16 18:53:37.910411: Epoch 722 +2023-10-16 18:53:37.910543: Current learning rate: 0.00316 +2023-10-16 18:59:24.792783: train_loss -0.7423 +2023-10-16 18:59:24.792937: val_loss -0.5908 +2023-10-16 18:59:24.793062: Pseudo dice [0.9718, 0.8649, 0.9003, 0.8871, 0.9164, 0.6585] +2023-10-16 18:59:24.793158: Epoch time: 346.88 s +2023-10-16 18:59:26.048174: +2023-10-16 18:59:26.048283: Epoch 723 +2023-10-16 18:59:26.048398: Current learning rate: 0.00315 +2023-10-16 19:05:12.939356: train_loss -0.7433 +2023-10-16 19:05:12.939518: val_loss -0.6831 +2023-10-16 19:05:12.939631: Pseudo dice [0.9692, 0.8628, 0.9104, 0.8846, 0.901, 0.6258] +2023-10-16 19:05:12.939718: Epoch time: 346.89 s +2023-10-16 19:05:14.202779: +2023-10-16 19:05:14.202989: Epoch 724 +2023-10-16 19:05:14.203168: Current learning rate: 0.00314 +2023-10-16 19:11:01.255048: train_loss -0.7776 +2023-10-16 19:11:01.255198: val_loss -0.6905 +2023-10-16 19:11:01.255312: Pseudo dice [0.9727, 0.8823, 0.9072, 0.9037, 0.8895, 0.6804] +2023-10-16 19:11:01.255404: Epoch time: 347.05 s +2023-10-16 19:11:02.504785: +2023-10-16 19:11:02.504904: Epoch 725 +2023-10-16 19:11:02.505020: Current learning rate: 0.00313 +2023-10-16 19:16:49.462324: train_loss -0.7808 +2023-10-16 19:16:49.462468: val_loss -0.6713 +2023-10-16 19:16:49.462594: Pseudo dice [0.971, 0.8769, 0.9221, 0.8643, 0.8723, 0.6858] +2023-10-16 19:16:49.462681: Epoch time: 346.96 s +2023-10-16 19:16:50.711224: +2023-10-16 19:16:50.711414: Epoch 726 +2023-10-16 19:16:50.711580: Current learning rate: 0.00312 +2023-10-16 19:22:37.729152: train_loss -0.7868 +2023-10-16 19:22:37.729310: val_loss -0.6498 +2023-10-16 19:22:37.729414: Pseudo dice [0.9713, 0.8733, 0.8978, 0.8517, 0.8767, 0.6395] +2023-10-16 19:22:37.729499: Epoch time: 347.02 s +2023-10-16 19:22:38.988612: +2023-10-16 19:22:38.988800: Epoch 727 +2023-10-16 19:22:38.988944: Current learning rate: 0.00311 +2023-10-16 19:28:25.938247: train_loss -0.7688 +2023-10-16 19:28:25.938400: val_loss -0.6864 +2023-10-16 19:28:25.938513: Pseudo dice [0.967, 0.8752, 0.8958, 0.8807, 0.8955, 0.6826] +2023-10-16 19:28:25.938601: Epoch time: 346.95 s +2023-10-16 19:28:27.357462: +2023-10-16 19:28:27.357578: Epoch 728 +2023-10-16 19:28:27.357693: Current learning rate: 0.0031 +2023-10-16 19:34:14.377090: train_loss -0.7517 +2023-10-16 19:34:14.377239: val_loss -0.6426 +2023-10-16 19:34:14.377358: Pseudo dice [0.9716, 0.8628, 0.8938, 0.875, 0.8832, 0.6524] +2023-10-16 19:34:14.377446: Epoch time: 347.02 s +2023-10-16 19:34:15.624686: +2023-10-16 19:34:15.624795: Epoch 729 +2023-10-16 19:34:15.624912: Current learning rate: 0.00309 +2023-10-16 19:40:02.607368: train_loss -0.7796 +2023-10-16 19:40:02.607537: val_loss -0.6619 +2023-10-16 19:40:02.607661: Pseudo dice [0.9724, 0.8814, 0.8943, 0.8721, 0.8429, 0.678] +2023-10-16 19:40:02.607759: Epoch time: 346.98 s +2023-10-16 19:40:03.855425: +2023-10-16 19:40:03.855531: Epoch 730 +2023-10-16 19:40:03.855642: Current learning rate: 0.00308 +2023-10-16 19:45:50.807545: train_loss -0.782 +2023-10-16 19:45:50.807684: val_loss -0.6797 +2023-10-16 19:45:50.807798: Pseudo dice [0.9722, 0.8782, 0.8965, 0.8921, 0.8947, 0.6896] +2023-10-16 19:45:50.807889: Epoch time: 346.95 s +2023-10-16 19:45:52.052246: +2023-10-16 19:45:52.052373: Epoch 731 +2023-10-16 19:45:52.052481: Current learning rate: 0.00307 +2023-10-16 19:51:39.053820: train_loss -0.7904 +2023-10-16 19:51:39.053964: val_loss -0.6774 +2023-10-16 19:51:39.054094: Pseudo dice [0.9718, 0.8854, 0.9172, 0.8836, 0.8633, 0.6705] +2023-10-16 19:51:39.054178: Epoch time: 347.0 s +2023-10-16 19:51:40.560286: +2023-10-16 19:51:40.560477: Epoch 732 +2023-10-16 19:51:40.560627: Current learning rate: 0.00306 +2023-10-16 19:57:27.542472: train_loss -0.7769 +2023-10-16 19:57:27.542644: val_loss -0.7191 +2023-10-16 19:57:27.542748: Pseudo dice [0.9723, 0.8758, 0.9074, 0.8964, 0.8692, 0.6413] +2023-10-16 19:57:27.542835: Epoch time: 346.98 s +2023-10-16 19:57:28.820989: +2023-10-16 19:57:28.821165: Epoch 733 +2023-10-16 19:57:28.821347: Current learning rate: 0.00305 +2023-10-16 20:03:15.776599: train_loss -0.7644 +2023-10-16 20:03:15.776749: val_loss -0.6748 +2023-10-16 20:03:15.776865: Pseudo dice [0.9678, 0.8599, 0.9078, 0.8708, 0.9094, 0.6484] +2023-10-16 20:03:15.776958: Epoch time: 346.96 s +2023-10-16 20:03:17.026584: +2023-10-16 20:03:17.026691: Epoch 734 +2023-10-16 20:03:17.026812: Current learning rate: 0.00304 +2023-10-16 20:09:03.869975: train_loss -0.7608 +2023-10-16 20:09:03.870114: val_loss -0.6832 +2023-10-16 20:09:03.870232: Pseudo dice [0.9709, 0.8705, 0.9131, 0.9043, 0.9058, 0.6642] +2023-10-16 20:09:03.870320: Epoch time: 346.84 s +2023-10-16 20:09:05.310963: +2023-10-16 20:09:05.311097: Epoch 735 +2023-10-16 20:09:05.311204: Current learning rate: 0.00303 +2023-10-16 20:14:52.073126: train_loss -0.7843 +2023-10-16 20:14:52.073279: val_loss -0.6834 +2023-10-16 20:14:52.073405: Pseudo dice [0.971, 0.8622, 0.9047, 0.905, 0.8859, 0.6482] +2023-10-16 20:14:52.073491: Epoch time: 346.76 s +2023-10-16 20:14:53.319446: +2023-10-16 20:14:53.319551: Epoch 736 +2023-10-16 20:14:53.319667: Current learning rate: 0.00302 +2023-10-16 20:20:40.196349: train_loss -0.7921 +2023-10-16 20:20:40.196598: val_loss -0.6307 +2023-10-16 20:20:40.196746: Pseudo dice [0.971, 0.8672, 0.9097, 0.8688, 0.9309, 0.6475] +2023-10-16 20:20:40.196844: Epoch time: 346.88 s +2023-10-16 20:20:41.443801: +2023-10-16 20:20:41.443908: Epoch 737 +2023-10-16 20:20:41.444032: Current learning rate: 0.00301 +2023-10-16 20:26:28.367692: train_loss -0.7729 +2023-10-16 20:26:28.367858: val_loss -0.6829 +2023-10-16 20:26:28.367961: Pseudo dice [0.9629, 0.881, 0.9185, 0.8668, 0.8715, 0.6689] +2023-10-16 20:26:28.368046: Epoch time: 346.92 s +2023-10-16 20:26:29.612815: +2023-10-16 20:26:29.612922: Epoch 738 +2023-10-16 20:26:29.613036: Current learning rate: 0.003 +2023-10-16 20:32:16.500695: train_loss -0.8012 +2023-10-16 20:32:16.500844: val_loss -0.6784 +2023-10-16 20:32:16.500956: Pseudo dice [0.9708, 0.8696, 0.9136, 0.8691, 0.8652, 0.6666] +2023-10-16 20:32:16.501048: Epoch time: 346.89 s +2023-10-16 20:32:17.760465: +2023-10-16 20:32:17.760568: Epoch 739 +2023-10-16 20:32:17.760695: Current learning rate: 0.00299 +2023-10-16 20:38:04.592379: train_loss -0.7674 +2023-10-16 20:38:04.592530: val_loss -0.6908 +2023-10-16 20:38:04.592652: Pseudo dice [0.9724, 0.8714, 0.902, 0.8697, 0.8764, 0.6592] +2023-10-16 20:38:04.592737: Epoch time: 346.83 s +2023-10-16 20:38:05.837263: +2023-10-16 20:38:05.837362: Epoch 740 +2023-10-16 20:38:05.837475: Current learning rate: 0.00297 +2023-10-16 20:43:52.660144: train_loss -0.7516 +2023-10-16 20:43:52.660285: val_loss -0.6461 +2023-10-16 20:43:52.660398: Pseudo dice [0.9706, 0.87, 0.8958, 0.8801, 0.8641, 0.7011] +2023-10-16 20:43:52.660489: Epoch time: 346.82 s +2023-10-16 20:43:53.907556: +2023-10-16 20:43:53.907661: Epoch 741 +2023-10-16 20:43:53.907781: Current learning rate: 0.00296 +2023-10-16 20:49:40.771184: train_loss -0.7764 +2023-10-16 20:49:40.771330: val_loss -0.6637 +2023-10-16 20:49:40.771452: Pseudo dice [0.9724, 0.8785, 0.9024, 0.8671, 0.8626, 0.6238] +2023-10-16 20:49:40.771538: Epoch time: 346.86 s +2023-10-16 20:49:42.199558: +2023-10-16 20:49:42.199669: Epoch 742 +2023-10-16 20:49:42.199785: Current learning rate: 0.00295 +2023-10-16 20:55:29.083561: train_loss -0.7812 +2023-10-16 20:55:29.083701: val_loss -0.6515 +2023-10-16 20:55:29.083814: Pseudo dice [0.9702, 0.8539, 0.9124, 0.8706, 0.8343, 0.6397] +2023-10-16 20:55:29.083905: Epoch time: 346.88 s +2023-10-16 20:55:30.326373: +2023-10-16 20:55:30.326590: Epoch 743 +2023-10-16 20:55:30.326766: Current learning rate: 0.00294 +2023-10-16 21:01:17.272231: train_loss -0.7812 +2023-10-16 21:01:17.272433: val_loss -0.6915 +2023-10-16 21:01:17.272554: Pseudo dice [0.9715, 0.8707, 0.8996, 0.8269, 0.8827, 0.6523] +2023-10-16 21:01:17.272651: Epoch time: 346.95 s +2023-10-16 21:01:18.527537: +2023-10-16 21:01:18.527642: Epoch 744 +2023-10-16 21:01:18.527767: Current learning rate: 0.00293 +2023-10-16 21:07:05.379056: train_loss -0.7603 +2023-10-16 21:07:05.379226: val_loss -0.6876 +2023-10-16 21:07:05.379349: Pseudo dice [0.9726, 0.8639, 0.9083, 0.8809, 0.8898, 0.6921] +2023-10-16 21:07:05.379447: Epoch time: 346.85 s +2023-10-16 21:07:06.626625: +2023-10-16 21:07:06.626747: Epoch 745 +2023-10-16 21:07:06.626876: Current learning rate: 0.00292 +2023-10-16 21:12:53.451590: train_loss -0.7823 +2023-10-16 21:12:53.451732: val_loss -0.6655 +2023-10-16 21:12:53.451844: Pseudo dice [0.9683, 0.8804, 0.899, 0.8178, 0.8839, 0.6644] +2023-10-16 21:12:53.451934: Epoch time: 346.83 s +2023-10-16 21:12:54.696416: +2023-10-16 21:12:54.696591: Epoch 746 +2023-10-16 21:12:54.696789: Current learning rate: 0.00291 +2023-10-16 21:18:41.674343: train_loss -0.7672 +2023-10-16 21:18:41.674484: val_loss -0.678 +2023-10-16 21:18:41.674613: Pseudo dice [0.9738, 0.8722, 0.909, 0.9068, 0.9027, 0.6562] +2023-10-16 21:18:41.674699: Epoch time: 346.98 s +2023-10-16 21:18:42.962177: +2023-10-16 21:18:42.962278: Epoch 747 +2023-10-16 21:18:42.962391: Current learning rate: 0.0029 +2023-10-16 21:24:29.754190: train_loss -0.7823 +2023-10-16 21:24:29.754351: val_loss -0.6516 +2023-10-16 21:24:29.754453: Pseudo dice [0.9742, 0.8777, 0.9024, 0.8644, 0.8886, 0.684] +2023-10-16 21:24:29.754547: Epoch time: 346.79 s +2023-10-16 21:24:31.000975: +2023-10-16 21:24:31.001076: Epoch 748 +2023-10-16 21:24:31.001190: Current learning rate: 0.00289 +2023-10-16 21:30:18.081588: train_loss -0.7763 +2023-10-16 21:30:18.081762: val_loss -0.6687 +2023-10-16 21:30:18.081884: Pseudo dice [0.9736, 0.8657, 0.9113, 0.8781, 0.8645, 0.6676] +2023-10-16 21:30:18.081988: Epoch time: 347.08 s +2023-10-16 21:30:19.327024: +2023-10-16 21:30:19.327136: Epoch 749 +2023-10-16 21:30:19.327245: Current learning rate: 0.00288 +2023-10-16 21:36:06.313214: train_loss -0.769 +2023-10-16 21:36:06.313378: val_loss -0.6887 +2023-10-16 21:36:06.313504: Pseudo dice [0.9696, 0.8651, 0.9186, 0.8477, 0.8635, 0.6643] +2023-10-16 21:36:06.313602: Epoch time: 346.99 s +2023-10-16 21:36:09.322969: +2023-10-16 21:36:09.323163: Epoch 750 +2023-10-16 21:36:09.323311: Current learning rate: 0.00287 +2023-10-16 21:41:56.237420: train_loss -0.7798 +2023-10-16 21:41:56.237570: val_loss -0.6466 +2023-10-16 21:41:56.237676: Pseudo dice [0.9738, 0.8774, 0.8983, 0.8788, 0.8519, 0.6585] +2023-10-16 21:41:56.237762: Epoch time: 346.92 s +2023-10-16 21:41:57.492184: +2023-10-16 21:41:57.492290: Epoch 751 +2023-10-16 21:41:57.492398: Current learning rate: 0.00286 +2023-10-16 21:47:44.454530: train_loss -0.7814 +2023-10-16 21:47:44.454681: val_loss -0.6659 +2023-10-16 21:47:44.454795: Pseudo dice [0.9697, 0.8644, 0.9237, 0.9063, 0.8612, 0.6375] +2023-10-16 21:47:44.454881: Epoch time: 346.96 s +2023-10-16 21:47:45.699300: +2023-10-16 21:47:45.699405: Epoch 752 +2023-10-16 21:47:45.699520: Current learning rate: 0.00285 +2023-10-16 21:53:32.651409: train_loss -0.805 +2023-10-16 21:53:32.651662: val_loss -0.6821 +2023-10-16 21:53:32.651787: Pseudo dice [0.9717, 0.8596, 0.9134, 0.8739, 0.8912, 0.6436] +2023-10-16 21:53:32.651883: Epoch time: 346.95 s +2023-10-16 21:53:33.894313: +2023-10-16 21:53:33.894414: Epoch 753 +2023-10-16 21:53:33.894536: Current learning rate: 0.00284 +2023-10-16 21:59:20.734253: train_loss -0.7614 +2023-10-16 21:59:20.734413: val_loss -0.7219 +2023-10-16 21:59:20.734549: Pseudo dice [0.9706, 0.8697, 0.9083, 0.8744, 0.8853, 0.6663] +2023-10-16 21:59:20.734730: Epoch time: 346.84 s +2023-10-16 21:59:21.983121: +2023-10-16 21:59:21.983289: Epoch 754 +2023-10-16 21:59:21.983458: Current learning rate: 0.00283 +2023-10-16 22:05:08.729062: train_loss -0.7599 +2023-10-16 22:05:08.729207: val_loss -0.6608 +2023-10-16 22:05:08.729323: Pseudo dice [0.9738, 0.8761, 0.9012, 0.9026, 0.9033, 0.6759] +2023-10-16 22:05:08.729420: Epoch time: 346.75 s +2023-10-16 22:05:10.152111: +2023-10-16 22:05:10.152219: Epoch 755 +2023-10-16 22:05:10.152333: Current learning rate: 0.00282 +2023-10-16 22:10:56.905413: train_loss -0.7884 +2023-10-16 22:10:56.905557: val_loss -0.7104 +2023-10-16 22:10:56.905695: Pseudo dice [0.9683, 0.8676, 0.9096, 0.8374, 0.8668, 0.6558] +2023-10-16 22:10:56.905791: Epoch time: 346.75 s +2023-10-16 22:10:58.149326: +2023-10-16 22:10:58.149428: Epoch 756 +2023-10-16 22:10:58.149547: Current learning rate: 0.00281 +2023-10-16 22:16:44.896919: train_loss -0.7781 +2023-10-16 22:16:44.897083: val_loss -0.7192 +2023-10-16 22:16:44.897204: Pseudo dice [0.9706, 0.8774, 0.9037, 0.8773, 0.9042, 0.677] +2023-10-16 22:16:44.897301: Epoch time: 346.75 s +2023-10-16 22:16:46.143886: +2023-10-16 22:16:46.144010: Epoch 757 +2023-10-16 22:16:46.144125: Current learning rate: 0.0028 +2023-10-16 22:22:33.012759: train_loss -0.7888 +2023-10-16 22:22:33.012903: val_loss -0.6814 +2023-10-16 22:22:33.013015: Pseudo dice [0.9729, 0.8683, 0.9104, 0.8744, 0.8768, 0.7198] +2023-10-16 22:22:33.013106: Epoch time: 346.87 s +2023-10-16 22:22:34.259762: +2023-10-16 22:22:34.259865: Epoch 758 +2023-10-16 22:22:34.259990: Current learning rate: 0.00279 +2023-10-16 22:28:21.064841: train_loss -0.7361 +2023-10-16 22:28:21.064984: val_loss -0.6864 +2023-10-16 22:28:21.065100: Pseudo dice [0.9648, 0.8821, 0.9176, 0.8619, 0.8589, 0.6633] +2023-10-16 22:28:21.065188: Epoch time: 346.81 s +2023-10-16 22:28:22.319604: +2023-10-16 22:28:22.319710: Epoch 759 +2023-10-16 22:28:22.319818: Current learning rate: 0.00278 +2023-10-16 22:34:09.133929: train_loss -0.7883 +2023-10-16 22:34:09.134073: val_loss -0.6856 +2023-10-16 22:34:09.134192: Pseudo dice [0.9726, 0.8688, 0.9055, 0.8427, 0.8812, 0.6554] +2023-10-16 22:34:09.134277: Epoch time: 346.82 s +2023-10-16 22:34:10.381863: +2023-10-16 22:34:10.382043: Epoch 760 +2023-10-16 22:34:10.382210: Current learning rate: 0.00277 +2023-10-16 22:39:57.259468: train_loss -0.7825 +2023-10-16 22:39:57.259617: val_loss -0.6793 +2023-10-16 22:39:57.259732: Pseudo dice [0.9727, 0.8735, 0.9076, 0.8929, 0.8783, 0.6945] +2023-10-16 22:39:57.259821: Epoch time: 346.88 s +2023-10-16 22:39:58.531965: +2023-10-16 22:39:58.532070: Epoch 761 +2023-10-16 22:39:58.532183: Current learning rate: 0.00276 +2023-10-16 22:45:45.485026: train_loss -0.7712 +2023-10-16 22:45:45.485170: val_loss -0.6874 +2023-10-16 22:45:45.485283: Pseudo dice [0.9723, 0.8793, 0.9192, 0.8844, 0.9144, 0.6516] +2023-10-16 22:45:45.485373: Epoch time: 346.95 s +2023-10-16 22:45:46.926359: +2023-10-16 22:45:46.926492: Epoch 762 +2023-10-16 22:45:46.926620: Current learning rate: 0.00275 +2023-10-16 22:51:33.977682: train_loss -0.7857 +2023-10-16 22:51:33.977825: val_loss -0.6443 +2023-10-16 22:51:33.977939: Pseudo dice [0.9727, 0.8767, 0.8949, 0.8151, 0.8862, 0.6678] +2023-10-16 22:51:33.978030: Epoch time: 347.05 s +2023-10-16 22:51:35.253114: +2023-10-16 22:51:35.253222: Epoch 763 +2023-10-16 22:51:35.253348: Current learning rate: 0.00274 +2023-10-16 22:57:22.227417: train_loss -0.7944 +2023-10-16 22:57:22.227562: val_loss -0.6669 +2023-10-16 22:57:22.227674: Pseudo dice [0.9687, 0.8757, 0.9031, 0.8526, 0.863, 0.6839] +2023-10-16 22:57:22.227766: Epoch time: 346.98 s +2023-10-16 22:57:23.495120: +2023-10-16 22:57:23.495229: Epoch 764 +2023-10-16 22:57:23.495349: Current learning rate: 0.00273 +2023-10-16 23:03:10.324488: train_loss -0.7739 +2023-10-16 23:03:10.324644: val_loss -0.6643 +2023-10-16 23:03:10.324748: Pseudo dice [0.9693, 0.8623, 0.9022, 0.8465, 0.868, 0.6316] +2023-10-16 23:03:10.324842: Epoch time: 346.83 s +2023-10-16 23:03:11.590431: +2023-10-16 23:03:11.590558: Epoch 765 +2023-10-16 23:03:11.590662: Current learning rate: 0.00272 +2023-10-16 23:08:58.534127: train_loss -0.7627 +2023-10-16 23:08:58.534281: val_loss -0.6885 +2023-10-16 23:08:58.534384: Pseudo dice [0.9727, 0.8742, 0.8991, 0.8921, 0.8761, 0.721] +2023-10-16 23:08:58.534471: Epoch time: 346.94 s +2023-10-16 23:08:59.799573: +2023-10-16 23:08:59.799681: Epoch 766 +2023-10-16 23:08:59.799800: Current learning rate: 0.00271 +2023-10-16 23:14:46.783532: train_loss -0.7838 +2023-10-16 23:14:46.783687: val_loss -0.6636 +2023-10-16 23:14:46.783791: Pseudo dice [0.9722, 0.8678, 0.9032, 0.8832, 0.8708, 0.6741] +2023-10-16 23:14:46.783879: Epoch time: 346.98 s +2023-10-16 23:14:48.045738: +2023-10-16 23:14:48.045854: Epoch 767 +2023-10-16 23:14:48.045959: Current learning rate: 0.0027 +2023-10-16 23:20:34.960293: train_loss -0.7738 +2023-10-16 23:20:34.960457: val_loss -0.6693 +2023-10-16 23:20:34.960583: Pseudo dice [0.9721, 0.8671, 0.9128, 0.8691, 0.88, 0.6513] +2023-10-16 23:20:34.960680: Epoch time: 346.92 s +2023-10-16 23:20:36.431780: +2023-10-16 23:20:36.431914: Epoch 768 +2023-10-16 23:20:36.432017: Current learning rate: 0.00268 +2023-10-16 23:26:23.355504: train_loss -0.7621 +2023-10-16 23:26:23.355654: val_loss -0.6347 +2023-10-16 23:26:23.355771: Pseudo dice [0.9736, 0.8705, 0.8865, 0.8463, 0.8879, 0.6394] +2023-10-16 23:26:23.355858: Epoch time: 346.92 s +2023-10-16 23:26:24.643907: +2023-10-16 23:26:24.644099: Epoch 769 +2023-10-16 23:26:24.644284: Current learning rate: 0.00267 +2023-10-16 23:32:11.696220: train_loss -0.772 +2023-10-16 23:32:11.696359: val_loss -0.7176 +2023-10-16 23:32:11.696473: Pseudo dice [0.9737, 0.8809, 0.9153, 0.8991, 0.8772, 0.673] +2023-10-16 23:32:11.696564: Epoch time: 347.05 s +2023-10-16 23:32:12.978057: +2023-10-16 23:32:12.978164: Epoch 770 +2023-10-16 23:32:12.978276: Current learning rate: 0.00266 +2023-10-16 23:37:59.855270: train_loss -0.7751 +2023-10-16 23:37:59.855424: val_loss -0.6703 +2023-10-16 23:37:59.855527: Pseudo dice [0.9715, 0.8624, 0.904, 0.8745, 0.8474, 0.6303] +2023-10-16 23:37:59.855613: Epoch time: 346.88 s +2023-10-16 23:38:01.119642: +2023-10-16 23:38:01.119751: Epoch 771 +2023-10-16 23:38:01.119865: Current learning rate: 0.00265 +2023-10-16 23:43:48.086070: train_loss -0.7857 +2023-10-16 23:43:48.086206: val_loss -0.7146 +2023-10-16 23:43:48.086318: Pseudo dice [0.973, 0.8639, 0.899, 0.8673, 0.8598, 0.6267] +2023-10-16 23:43:48.086406: Epoch time: 346.97 s +2023-10-16 23:43:49.371821: +2023-10-16 23:43:49.372040: Epoch 772 +2023-10-16 23:43:49.372244: Current learning rate: 0.00264 +2023-10-16 23:49:36.241262: train_loss -0.7812 +2023-10-16 23:49:36.241413: val_loss -0.6925 +2023-10-16 23:49:36.241530: Pseudo dice [0.9729, 0.8709, 0.9125, 0.8367, 0.8972, 0.6604] +2023-10-16 23:49:36.241616: Epoch time: 346.87 s +2023-10-16 23:49:37.501805: +2023-10-16 23:49:37.501961: Epoch 773 +2023-10-16 23:49:37.502124: Current learning rate: 0.00263 diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/plans.json b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/plans.json new file mode 100755 index 0000000000000000000000000000000000000000..44786cad16bc9aeeaff049f3da4748f66d5922e3 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/plans.json @@ -0,0 +1,454 @@ +{ + "dataset_name": "Dataset720_TSPrime", + "plans_name": "nnUNetPlans", + "original_median_spacing_after_transp": [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + "original_median_shape_after_transp": [ + 241, + 512, + 512 + ], + "image_reader_writer": "SimpleITKIO", + "transpose_forward": [ + 0, + 1, + 2 + ], + "transpose_backward": [ + 0, + 1, + 2 + ], + "configurations": { + "2d": { + "data_identifier": "nnUNetPlans_2d", + "preprocessor_name": "DefaultPreprocessor", + "batch_size": 12, + "patch_size": [ + 512, + 512 + ], + "median_image_size_in_voxels": [ + 512.0, + 512.0 + ], + "spacing": [ + 1.269531011581421, + 1.269531011581421 + ], + "normalization_schemes": [ + "CTNormalization" + ], + "use_mask_for_norm": [ + false + ], + "UNet_class_name": "PlainConvUNet", + "UNet_base_num_features": 32, + "n_conv_per_stage_encoder": [ + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_conv_per_stage_decoder": [ + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "num_pool_per_axis": [ + 7, + 7 + ], + "pool_op_kernel_sizes": [ + [ + 1, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ] + ], + "conv_kernel_sizes": [ + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ] + ], + "unet_max_num_features": 512, + "resampling_fn_data": "resample_data_or_seg_to_shape", + "resampling_fn_seg": "resample_data_or_seg_to_shape", + "resampling_fn_data_kwargs": { + "is_seg": false, + "order": 3, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_seg_kwargs": { + "is_seg": true, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_probabilities": "resample_data_or_seg_to_shape", + "resampling_fn_probabilities_kwargs": { + "is_seg": false, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "batch_dice": true + }, + "3d_lowres": { + "data_identifier": "nnUNetPlans_3d_lowres", + "preprocessor_name": "DefaultPreprocessor", + "batch_size": 2, + "patch_size": [ + 80, + 192, + 160 + ], + "median_image_size_in_voxels": [ + 130, + 275, + 275 + ], + "spacing": [ + 4.650736429273743, + 2.361701649461784, + 2.361701649461784 + ], + "normalization_schemes": [ + "CTNormalization" + ], + "use_mask_for_norm": [ + false + ], + "UNet_class_name": "PlainConvUNet", + "UNet_base_num_features": 32, + "n_conv_per_stage_encoder": [ + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_conv_per_stage_decoder": [ + 2, + 2, + 2, + 2, + 2 + ], + "num_pool_per_axis": [ + 4, + 5, + 5 + ], + "pool_op_kernel_sizes": [ + [ + 1, + 1, + 1 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 1, + 2, + 2 + ] + ], + "conv_kernel_sizes": [ + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ] + ], + "unet_max_num_features": 320, + "resampling_fn_data": "resample_data_or_seg_to_shape", + "resampling_fn_seg": "resample_data_or_seg_to_shape", + "resampling_fn_data_kwargs": { + "is_seg": false, + "order": 3, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_seg_kwargs": { + "is_seg": true, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_probabilities": "resample_data_or_seg_to_shape", + "resampling_fn_probabilities_kwargs": { + "is_seg": false, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "batch_dice": false, + "next_stage": "3d_cascade_fullres" + }, + "3d_fullres": { + "data_identifier": "nnUNetPlans_3d_fullres", + "preprocessor_name": "DefaultPreprocessor", + "batch_size": 2, + "patch_size": [ + 80, + 192, + 160 + ], + "median_image_size_in_voxels": [ + 241.0, + 512.0, + 512.0 + ], + "spacing": [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + "normalization_schemes": [ + "CTNormalization" + ], + "use_mask_for_norm": [ + false + ], + "UNet_class_name": "PlainConvUNet", + "UNet_base_num_features": 32, + "n_conv_per_stage_encoder": [ + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_conv_per_stage_decoder": [ + 2, + 2, + 2, + 2, + 2 + ], + "num_pool_per_axis": [ + 4, + 5, + 5 + ], + "pool_op_kernel_sizes": [ + [ + 1, + 1, + 1 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 1, + 2, + 2 + ] + ], + "conv_kernel_sizes": [ + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ] + ], + "unet_max_num_features": 320, + "resampling_fn_data": "resample_data_or_seg_to_shape", + "resampling_fn_seg": "resample_data_or_seg_to_shape", + "resampling_fn_data_kwargs": { + "is_seg": false, + "order": 3, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_seg_kwargs": { + "is_seg": true, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_probabilities": "resample_data_or_seg_to_shape", + "resampling_fn_probabilities_kwargs": { + "is_seg": false, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "batch_dice": true + }, + "3d_cascade_fullres": { + "inherits_from": "3d_fullres", + "previous_stage": "3d_lowres" + } + }, + "experiment_planner_used": "ExperimentPlanner", + "label_manager": "LabelManager", + "foreground_intensity_properties_per_channel": { + "0": { + "max": 1620.0, + "mean": -38.229164123535156, + "median": -54.0, + "min": -1000.0, + "percentile_00_5": -941.0, + "percentile_99_5": 897.0, + "std": 192.37086486816406 + } + } +} \ No newline at end of file diff --git a/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/postprocessing.pkl b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/postprocessing.pkl new file mode 100755 index 0000000000000000000000000000000000000000..b556f3192bb68cd700b2fad89677ec98d8a49102 --- /dev/null +++ b/Dataset720_TSPrime/nnUNetTrainerNoMirroring__nnUNetPlans__3d_fullres/postprocessing.pkl @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97b914ebfa1828ffde547b0262473ade65698112999ae10d07f85d2586cf6e9b +size 388 diff --git a/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/dataset.json b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/dataset.json new file mode 100755 index 0000000000000000000000000000000000000000..ab22bc55c8f1f114d3b75f9e73af14b62b78c40a --- /dev/null +++ b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/dataset.json @@ -0,0 +1,12 @@ +{ + "channel_names": { + "0": "CT" + }, + "labels": { + "background": 0, + "Ctvp": 1 + }, + "numTraining": 60, + "file_ending": ".nii.gz", + "numTest": 0 +} \ No newline at end of file diff --git a/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/dataset_fingerprint.json b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/dataset_fingerprint.json new file mode 100755 index 0000000000000000000000000000000000000000..9557c950a20cad758d7d3453933ca7f72242a4da --- /dev/null +++ b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/dataset_fingerprint.json @@ -0,0 +1,618 @@ +{ + "foreground_intensity_properties_per_channel": { + "0": { + "max": 882.0, + "mean": 45.35713577270508, + "median": 48.0, + "min": -118.0, + "percentile_00_5": -48.0, + "percentile_99_5": 103.0, + "std": 26.203161239624023 + } + }, + "median_relative_size_after_cropping": 1.0, + "shapes_after_crop": [ + [ + 230, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 260, + 512, + 512 + ], + [ + 215, + 512, + 512 + ], + [ + 260, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 265, + 512, + 512 + ], + [ + 229, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 243, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 245, + 512, + 512 + ], + [ + 235, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 242, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 255, + 512, + 512 + ], + [ + 246, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 245, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 249, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 244, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 235, + 512, + 512 + ], + [ + 260, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 190, + 512, + 512 + ], + [ + 255, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 255, + 512, + 512 + ], + [ + 236, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 245, + 512, + 512 + ], + [ + 241, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 210, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 266, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 230, + 512, + 512 + ], + [ + 280, + 512, + 512 + ], + [ + 260, + 512, + 512 + ], + [ + 245, + 512, + 512 + ], + [ + 220, + 512, + 512 + ], + [ + 240, + 512, + 512 + ], + [ + 250, + 512, + 512 + ], + [ + 226, + 512, + 512 + ], + [ + 240, + 512, + 512 + ] + ], + "spacings": [ + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ] + ] +} \ No newline at end of file diff --git a/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/checkpoint_best.pth b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/checkpoint_best.pth new file mode 100755 index 0000000000000000000000000000000000000000..cebb30af7b7843969945f4d0348e207a57e8c0f8 --- /dev/null +++ b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/checkpoint_best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8acdeaa823073099ac98e716c20fcc574b5749f0a21978eb2ad91e2002f6fca1 +size 246436257 diff --git a/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/checkpoint_latest.pth b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/checkpoint_latest.pth new file mode 100755 index 0000000000000000000000000000000000000000..b93ce1af76e4fe171a142c2f5ac5570fd964dc89 --- /dev/null +++ b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/checkpoint_latest.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d79a28d029c7b6774bd1284c0d57ea77d1b77579ee5d60364fa8b94dce336363 +size 246425813 diff --git a/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/debug.json b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/debug.json new file mode 100755 index 0000000000000000000000000000000000000000..ce0a1b8357dc0120bb056e8bfc56a7657a4c28da --- /dev/null +++ b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/debug.json @@ -0,0 +1,52 @@ +{ + "_best_ema": "None", + "batch_size": "2", + "configuration_manager": "{'data_identifier': 'nnUNetPlans_3d_lowres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [130, 275, 275], 'spacing': [4.650736429273743, 2.361701649461784, 2.361701649461784], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': False, 'next_stage': '3d_cascade_fullres'}", + "configuration_name": "3d_lowres", + "cudnn_version": 8500, + "current_epoch": "0", + "dataloader_train": "", + "dataloader_train.generator": "", + "dataloader_train.num_processes": "12", + "dataloader_train.transform": "Compose ( [SpatialTransform( independent_scale_for_each_axis = False, p_rot_per_sample = 0.2, p_scale_per_sample = 0.2, p_el_per_sample = 0, data_key = 'data', label_key = 'seg', patch_size = [80, 192, 160], patch_center_dist_from_border = None, do_elastic_deform = False, alpha = (0, 0), sigma = (0, 0), do_rotation = True, angle_x = (-0.5235987755982988, 0.5235987755982988), angle_y = (-0.5235987755982988, 0.5235987755982988), angle_z = (-0.5235987755982988, 0.5235987755982988), do_scale = True, scale = (0.7, 1.4), border_mode_data = 'constant', border_cval_data = 0, order_data = 3, border_mode_seg = 'constant', border_cval_seg = -1, order_seg = 1, random_crop = False, p_rot_per_axis = 1, p_independent_scale_per_axis = 1 ), GaussianNoiseTransform( p_per_sample = 0.1, data_key = 'data', noise_variance = (0, 0.1), p_per_channel = 1, per_channel = False ), GaussianBlurTransform( p_per_sample = 0.2, different_sigma_per_channel = True, p_per_channel = 0.5, data_key = 'data', blur_sigma = (0.5, 1.0), different_sigma_per_axis = False, p_isotropic = 0 ), BrightnessMultiplicativeTransform( p_per_sample = 0.15, data_key = 'data', multiplier_range = (0.75, 1.25), per_channel = True ), ContrastAugmentationTransform( p_per_sample = 0.15, data_key = 'data', contrast_range = (0.75, 1.25), preserve_range = True, per_channel = True, p_per_channel = 1 ), SimulateLowResolutionTransform( order_upsample = 3, order_downsample = 0, channels = None, per_channel = True, p_per_channel = 0.5, p_per_sample = 0.25, data_key = 'data', zoom_range = (0.5, 1), ignore_axes = None ), GammaTransform( p_per_sample = 0.1, retain_stats = True, per_channel = True, data_key = 'data', gamma_range = (0.7, 1.5), invert_image = True ), GammaTransform( p_per_sample = 0.3, retain_stats = True, per_channel = True, data_key = 'data', gamma_range = (0.7, 1.5), invert_image = False ), MirrorTransform( p_per_sample = 1, data_key = 'data', label_key = 'seg', axes = (0, 1, 2) ), RemoveLabelTransform( output_key = 'seg', input_key = 'seg', replace_with = 0, remove_label = -1 ), RenameTransform( delete_old = True, out_key = 'target', in_key = 'seg' ), DownsampleSegForDSTransform2( axes = None, output_key = 'target', input_key = 'target', order = 0, ds_scales = [[1.0, 1.0, 1.0], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]] ), NumpyToTensor( keys = ['data', 'target'], cast_to = 'float' )] )", + "dataloader_val": "", + "dataloader_val.generator": "", + "dataloader_val.num_processes": "6", + "dataloader_val.transform": "Compose ( [RemoveLabelTransform( output_key = 'seg', input_key = 'seg', replace_with = 0, remove_label = -1 ), RenameTransform( delete_old = True, out_key = 'target', in_key = 'seg' ), DownsampleSegForDSTransform2( axes = None, output_key = 'target', input_key = 'target', order = 0, ds_scales = [[1.0, 1.0, 1.0], [0.5, 0.5, 0.5], [0.25, 0.25, 0.25], [0.125, 0.125, 0.125], [0.0625, 0.0625, 0.0625]] ), NumpyToTensor( keys = ['data', 'target'], cast_to = 'float' )] )", + "dataset_json": "{'channel_names': {'0': 'CT'}, 'labels': {'background': 0, 'Ctvp': 1}, 'numTraining': 60, 'file_ending': '.nii.gz', 'numTest': 0}", + "device": "cuda:0", + "disable_checkpointing": "False", + "fold": "0", + "folder_with_segs_from_previous_stage": "None", + "gpu_name": "NVIDIA GeForce GTX 1080 Ti", + "grad_scaler": "", + "hostname": "vipadmin-Z10PE-D16-WS", + "inference_allowed_mirroring_axes": "(0, 1, 2)", + "initial_lr": "0.01", + "is_cascaded": "False", + "is_ddp": "False", + "label_manager": "", + "local_rank": "0", + "log_file": "./data/nnUNet_results/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/training_log_2023_11_5_22_05_41.txt", + "logger": "", + "loss": "DeepSupervisionWrapper(\n (loss): DC_and_CE_loss(\n (ce): RobustCrossEntropyLoss()\n (dc): MemoryEfficientSoftDiceLoss()\n )\n)", + "lr_scheduler": "", + "my_init_kwargs": "{'plans': {'dataset_name': 'Dataset721_TSPrimeCTVP', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [241, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'configurations': {'2d': {'data_identifier': 'nnUNetPlans_2d', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 12, 'patch_size': [512, 512], 'median_image_size_in_voxels': [512.0, 512.0], 'spacing': [1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2, 2, 2], 'num_pool_per_axis': [7, 7], 'pool_op_kernel_sizes': [[1, 1], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2]], 'conv_kernel_sizes': [[3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3]], 'unet_max_num_features': 512, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True}, '3d_lowres': {'data_identifier': 'nnUNetPlans_3d_lowres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [130, 275, 275], 'spacing': [4.650736429273743, 2.361701649461784, 2.361701649461784], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': False, 'next_stage': '3d_cascade_fullres'}, '3d_fullres': {'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [241.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True}, '3d_cascade_fullres': {'inherits_from': '3d_fullres', 'previous_stage': '3d_lowres'}}, 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 882.0, 'mean': 45.35713577270508, 'median': 48.0, 'min': -118.0, 'percentile_00_5': -48.0, 'percentile_99_5': 103.0, 'std': 26.203161239624023}}}, 'configuration': '3d_lowres', 'fold': 0, 'dataset_json': {'channel_names': {'0': 'CT'}, 'labels': {'background': 0, 'Ctvp': 1}, 'numTraining': 60, 'file_ending': '.nii.gz', 'numTest': 0}, 'unpack_dataset': True, 'device': device(type='cuda')}", + "network": "PlainConvUNet", + "num_epochs": "1000", + "num_input_channels": "1", + "num_iterations_per_epoch": "250", + "num_val_iterations_per_epoch": "50", + "optimizer": "SGD (\nParameter Group 0\n dampening: 0\n differentiable: False\n foreach: None\n initial_lr: 0.01\n lr: 0.01\n maximize: False\n momentum: 0.99\n nesterov: True\n weight_decay: 3e-05\n)", + "output_folder": "./data/nnUNet_results/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0", + "output_folder_base": "./data/nnUNet_results/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres", + "oversample_foreground_percent": "0.33", + "plans_manager": "{'dataset_name': 'Dataset721_TSPrimeCTVP', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [241, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'configurations': {'2d': {'data_identifier': 'nnUNetPlans_2d', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 12, 'patch_size': [512, 512], 'median_image_size_in_voxels': [512.0, 512.0], 'spacing': [1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2, 2, 2], 'num_pool_per_axis': [7, 7], 'pool_op_kernel_sizes': [[1, 1], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2], [2, 2]], 'conv_kernel_sizes': [[3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3], [3, 3]], 'unet_max_num_features': 512, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True}, '3d_lowres': {'data_identifier': 'nnUNetPlans_3d_lowres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [130, 275, 275], 'spacing': [4.650736429273743, 2.361701649461784, 2.361701649461784], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': False, 'next_stage': '3d_cascade_fullres'}, '3d_fullres': {'data_identifier': 'nnUNetPlans_3d_fullres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [241.0, 512.0, 512.0], 'spacing': [2.5, 1.269531011581421, 1.269531011581421], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': True}, '3d_cascade_fullres': {'inherits_from': '3d_fullres', 'previous_stage': '3d_lowres'}}, 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 882.0, 'mean': 45.35713577270508, 'median': 48.0, 'min': -118.0, 'percentile_00_5': -48.0, 'percentile_99_5': 103.0, 'std': 26.203161239624023}}}", + "preprocessed_dataset_folder": "./data/nnUNet_preprocessed/Dataset721_TSPrimeCTVP/nnUNetPlans_3d_lowres", + "preprocessed_dataset_folder_base": "./data/nnUNet_preprocessed/Dataset721_TSPrimeCTVP", + "save_every": "50", + "torch_version": "2.0.1+cu117", + "unpack_dataset": "True", + "was_initialized": "True", + "weight_decay": "3e-05" +} \ No newline at end of file diff --git a/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/network_architecture b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/network_architecture new file mode 100755 index 0000000000000000000000000000000000000000..e8dd9af306f1958d25af867d5c2090c7ffb8a8b1 --- /dev/null +++ b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/network_architecture @@ -0,0 +1,171 @@ +digraph { + graph [bgcolor="#FFFFFF" color="#000000" fontcolor="#000000" fontname=Times fontsize=10 margin="0,0" pad="1.0,0.5" rankdir=LR] + node [color="#000000" fillcolor="#E8E8E8" fontcolor="#000000" fontname=Times fontsize=10 margin="0,0" shape=box style=filled] + edge [color="#000000" fontcolor="#000000" fontname=Times fontsize=10 style=solid] + "/outputs/109" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/110" [label=<
InstanceNormalization
>] + "/outputs/111" [label=<
LeakyRelu
>] + "/outputs/112" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/113" [label=<
InstanceNormalization
>] + "/outputs/114" [label=<
LeakyRelu
>] + "/outputs/115" [label=<
Conv, kernel_size: [3, 3, 3], stride: [2, 2, 2]
>] + "/outputs/116" [label=<
InstanceNormalization
>] + "/outputs/117" [label=<
LeakyRelu
>] + "/outputs/118" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/119" [label=<
InstanceNormalization
>] + "/outputs/120" [label=<
LeakyRelu
>] + "/outputs/121" [label=<
Conv, kernel_size: [3, 3, 3], stride: [2, 2, 2]
>] + "/outputs/122" [label=<
InstanceNormalization
>] + "/outputs/123" [label=<
LeakyRelu
>] + "/outputs/124" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/125" [label=<
InstanceNormalization
>] + "/outputs/126" [label=<
LeakyRelu
>] + "/outputs/127" [label=<
Conv, kernel_size: [3, 3, 3], stride: [2, 2, 2]
>] + "/outputs/128" [label=<
InstanceNormalization
>] + "/outputs/129" [label=<
LeakyRelu
>] + "/outputs/130" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/131" [label=<
InstanceNormalization
>] + "/outputs/132" [label=<
LeakyRelu
>] + "/outputs/133" [label=<
Conv, kernel_size: [3, 3, 3], stride: [2, 2, 2]
>] + "/outputs/134" [label=<
InstanceNormalization
>] + "/outputs/135" [label=<
LeakyRelu
>] + "/outputs/136" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/137" [label=<
InstanceNormalization
>] + "/outputs/138" [label=<
LeakyRelu
>] + "/outputs/139" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 2, 2]
>] + "/outputs/140" [label=<
InstanceNormalization
>] + "/outputs/141" [label=<
LeakyRelu
>] + "/outputs/142" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/143" [label=<
InstanceNormalization
>] + "/outputs/144" [label=<
LeakyRelu
>] + "/outputs/145" [label=<
ConvTranspose, kernel_size: [1, 2, 2], stride: [1, 2, 2]
>] + "/outputs/146" [label=<
Concat
>] + "/outputs/147" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/148" [label=<
InstanceNormalization
>] + "/outputs/149" [label=<
LeakyRelu
>] + "/outputs/150" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/151" [label=<
InstanceNormalization
>] + "/outputs/152" [label=<
LeakyRelu
>] + "/outputs/153" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/154" [label=<
ConvTranspose, kernel_size: [2, 2, 2], stride: [2, 2, 2]
>] + "/outputs/155" [label=<
Concat
>] + "/outputs/156" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/157" [label=<
InstanceNormalization
>] + "/outputs/158" [label=<
LeakyRelu
>] + "/outputs/159" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/160" [label=<
InstanceNormalization
>] + "/outputs/161" [label=<
LeakyRelu
>] + "/outputs/162" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/163" [label=<
ConvTranspose, kernel_size: [2, 2, 2], stride: [2, 2, 2]
>] + "/outputs/164" [label=<
Concat
>] + "/outputs/165" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/166" [label=<
InstanceNormalization
>] + "/outputs/167" [label=<
LeakyRelu
>] + "/outputs/168" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/169" [label=<
InstanceNormalization
>] + "/outputs/170" [label=<
LeakyRelu
>] + "/outputs/171" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/172" [label=<
ConvTranspose, kernel_size: [2, 2, 2], stride: [2, 2, 2]
>] + "/outputs/173" [label=<
Concat
>] + "/outputs/174" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/175" [label=<
InstanceNormalization
>] + "/outputs/176" [label=<
LeakyRelu
>] + "/outputs/177" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/178" [label=<
InstanceNormalization
>] + "/outputs/179" [label=<
LeakyRelu
>] + "/outputs/180" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/181" [label=<
ConvTranspose, kernel_size: [2, 2, 2], stride: [2, 2, 2]
>] + "/outputs/182" [label=<
Concat
>] + "/outputs/183" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/184" [label=<
InstanceNormalization
>] + "/outputs/185" [label=<
LeakyRelu
>] + "/outputs/186" [label=<
Conv, kernel_size: [3, 3, 3], stride: [1, 1, 1]
>] + "/outputs/187" [label=<
InstanceNormalization
>] + "/outputs/188" [label=<
LeakyRelu
>] + "/outputs/189" [label=<
Conv, kernel_size: [1, 1, 1], stride: [1, 1, 1]
>] + "/outputs/109" -> "/outputs/110" [label="1x32x80x192x160"] + "/outputs/110" -> "/outputs/111" [label="1x32x80x192x160"] + "/outputs/111" -> "/outputs/112" [label="1x32x80x192x160"] + "/outputs/112" -> "/outputs/113" [label="1x32x80x192x160"] + "/outputs/113" -> "/outputs/114" [label="1x32x80x192x160"] + "/outputs/114" -> "/outputs/115" [label="1x32x80x192x160"] + "/outputs/114" -> "/outputs/182" [label="1x32x80x192x160"] + "/outputs/115" -> "/outputs/116" [label="1x64x40x96x80"] + "/outputs/116" -> "/outputs/117" [label="1x64x40x96x80"] + "/outputs/117" -> "/outputs/118" [label="1x64x40x96x80"] + "/outputs/118" -> "/outputs/119" [label="1x64x40x96x80"] + "/outputs/119" -> "/outputs/120" [label="1x64x40x96x80"] + "/outputs/120" -> "/outputs/121" [label="1x64x40x96x80"] + "/outputs/120" -> "/outputs/173" [label="1x64x40x96x80"] + "/outputs/121" -> "/outputs/122" [label="1x128x20x48x40"] + "/outputs/122" -> "/outputs/123" [label="1x128x20x48x40"] + "/outputs/123" -> "/outputs/124" [label="1x128x20x48x40"] + "/outputs/124" -> "/outputs/125" [label="1x128x20x48x40"] + "/outputs/125" -> "/outputs/126" [label="1x128x20x48x40"] + "/outputs/126" -> "/outputs/127" [label="1x128x20x48x40"] + "/outputs/126" -> "/outputs/164" [label="1x128x20x48x40"] + "/outputs/127" -> "/outputs/128" [label="1x256x10x24x20"] + "/outputs/128" -> "/outputs/129" [label="1x256x10x24x20"] + "/outputs/129" -> "/outputs/130" [label="1x256x10x24x20"] + "/outputs/130" -> "/outputs/131" [label="1x256x10x24x20"] + "/outputs/131" -> "/outputs/132" [label="1x256x10x24x20"] + "/outputs/132" -> "/outputs/133" [label="1x256x10x24x20"] + "/outputs/132" -> "/outputs/155" [label="1x256x10x24x20"] + "/outputs/133" -> "/outputs/134" [label="1x320x5x12x10"] + "/outputs/134" -> "/outputs/135" [label="1x320x5x12x10"] + "/outputs/135" -> "/outputs/136" [label="1x320x5x12x10"] + "/outputs/136" -> "/outputs/137" [label="1x320x5x12x10"] + "/outputs/137" -> "/outputs/138" [label="1x320x5x12x10"] + "/outputs/138" -> "/outputs/139" [label="1x320x5x12x10"] + "/outputs/138" -> "/outputs/146" [label="1x320x5x12x10"] + "/outputs/139" -> "/outputs/140" [label="1x320x5x6x5"] + "/outputs/140" -> "/outputs/141" [label="1x320x5x6x5"] + "/outputs/141" -> "/outputs/142" [label="1x320x5x6x5"] + "/outputs/142" -> "/outputs/143" [label="1x320x5x6x5"] + "/outputs/143" -> "/outputs/144" [label="1x320x5x6x5"] + "/outputs/144" -> "/outputs/145" [label="1x320x5x6x5"] + "/outputs/145" -> "/outputs/146" [label="1x320x5x12x10"] + "/outputs/146" -> "/outputs/147" [label="1x640x5x12x10"] + "/outputs/147" -> "/outputs/148" [label="1x320x5x12x10"] + "/outputs/148" -> "/outputs/149" [label="1x320x5x12x10"] + "/outputs/149" -> "/outputs/150" [label="1x320x5x12x10"] + "/outputs/150" -> "/outputs/151" [label="1x320x5x12x10"] + "/outputs/151" -> "/outputs/152" [label="1x320x5x12x10"] + "/outputs/152" -> "/outputs/153" [label="1x320x5x12x10"] + "/outputs/152" -> "/outputs/154" [label="1x320x5x12x10"] + "/outputs/154" -> "/outputs/155" [label="1x256x10x24x20"] + "/outputs/155" -> "/outputs/156" [label="1x512x10x24x20"] + "/outputs/156" -> "/outputs/157" [label="1x256x10x24x20"] + "/outputs/157" -> "/outputs/158" [label="1x256x10x24x20"] + "/outputs/158" -> "/outputs/159" [label="1x256x10x24x20"] + "/outputs/159" -> "/outputs/160" [label="1x256x10x24x20"] + "/outputs/160" -> "/outputs/161" [label="1x256x10x24x20"] + "/outputs/161" -> "/outputs/162" [label="1x256x10x24x20"] + "/outputs/161" -> "/outputs/163" [label="1x256x10x24x20"] + "/outputs/163" -> "/outputs/164" [label="1x128x20x48x40"] + "/outputs/164" -> "/outputs/165" [label="1x256x20x48x40"] + "/outputs/165" -> "/outputs/166" [label="1x128x20x48x40"] + "/outputs/166" -> "/outputs/167" [label="1x128x20x48x40"] + "/outputs/167" -> "/outputs/168" [label="1x128x20x48x40"] + "/outputs/168" -> "/outputs/169" [label="1x128x20x48x40"] + "/outputs/169" -> "/outputs/170" [label="1x128x20x48x40"] + "/outputs/170" -> "/outputs/171" [label="1x128x20x48x40"] + "/outputs/170" -> "/outputs/172" [label="1x128x20x48x40"] + "/outputs/172" -> "/outputs/173" [label="1x64x40x96x80"] + "/outputs/173" -> "/outputs/174" [label="1x128x40x96x80"] + "/outputs/174" -> "/outputs/175" [label="1x64x40x96x80"] + "/outputs/175" -> "/outputs/176" [label="1x64x40x96x80"] + "/outputs/176" -> "/outputs/177" [label="1x64x40x96x80"] + "/outputs/177" -> "/outputs/178" [label="1x64x40x96x80"] + "/outputs/178" -> "/outputs/179" [label="1x64x40x96x80"] + "/outputs/179" -> "/outputs/180" [label="1x64x40x96x80"] + "/outputs/179" -> "/outputs/181" [label="1x64x40x96x80"] + "/outputs/181" -> "/outputs/182" [label="1x32x80x192x160"] + "/outputs/182" -> "/outputs/183" [label="1x64x80x192x160"] + "/outputs/183" -> "/outputs/184" [label="1x32x80x192x160"] + "/outputs/184" -> "/outputs/185" [label="1x32x80x192x160"] + "/outputs/185" -> "/outputs/186" [label="1x32x80x192x160"] + "/outputs/186" -> "/outputs/187" [label="1x32x80x192x160"] + "/outputs/187" -> "/outputs/188" [label="1x32x80x192x160"] + "/outputs/188" -> "/outputs/189" [label="1x32x80x192x160"] +} diff --git a/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/progress.png b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/progress.png new file mode 100755 index 0000000000000000000000000000000000000000..fb38f449fe66ae9488487e4c4a6395362313a4e5 Binary files /dev/null and b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/progress.png differ diff --git a/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/training_log_2023_11_5_22_05_41.txt b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/training_log_2023_11_5_22_05_41.txt new file mode 100755 index 0000000000000000000000000000000000000000..61d851c9dfecd7983be95b024eaac4f399372a60 --- /dev/null +++ b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/fold_0/training_log_2023_11_5_22_05_41.txt @@ -0,0 +1,660 @@ + +####################################################################### +Please cite the following paper when using nnU-Net: +Isensee, F., Jaeger, P. F., Kohl, S. A., Petersen, J., & Maier-Hein, K. H. (2021). nnU-Net: a self-configuring method for deep learning-based biomedical image segmentation. Nature methods, 18(2), 203-211. +####################################################################### + + +This is the configuration used by this training: +Configuration name: 3d_lowres + {'data_identifier': 'nnUNetPlans_3d_lowres', 'preprocessor_name': 'DefaultPreprocessor', 'batch_size': 2, 'patch_size': [80, 192, 160], 'median_image_size_in_voxels': [130, 275, 275], 'spacing': [4.650736429273743, 2.361701649461784, 2.361701649461784], 'normalization_schemes': ['CTNormalization'], 'use_mask_for_norm': [False], 'UNet_class_name': 'PlainConvUNet', 'UNet_base_num_features': 32, 'n_conv_per_stage_encoder': [2, 2, 2, 2, 2, 2], 'n_conv_per_stage_decoder': [2, 2, 2, 2, 2], 'num_pool_per_axis': [4, 5, 5], 'pool_op_kernel_sizes': [[1, 1, 1], [2, 2, 2], [2, 2, 2], [2, 2, 2], [2, 2, 2], [1, 2, 2]], 'conv_kernel_sizes': [[3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3], [3, 3, 3]], 'unet_max_num_features': 320, 'resampling_fn_data': 'resample_data_or_seg_to_shape', 'resampling_fn_seg': 'resample_data_or_seg_to_shape', 'resampling_fn_data_kwargs': {'is_seg': False, 'order': 3, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_seg_kwargs': {'is_seg': True, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'resampling_fn_probabilities': 'resample_data_or_seg_to_shape', 'resampling_fn_probabilities_kwargs': {'is_seg': False, 'order': 1, 'order_z': 0, 'force_separate_z': None}, 'batch_dice': False, 'next_stage': '3d_cascade_fullres'} + +These are the global plan.json settings: + {'dataset_name': 'Dataset721_TSPrimeCTVP', 'plans_name': 'nnUNetPlans', 'original_median_spacing_after_transp': [2.5, 1.269531011581421, 1.269531011581421], 'original_median_shape_after_transp': [241, 512, 512], 'image_reader_writer': 'SimpleITKIO', 'transpose_forward': [0, 1, 2], 'transpose_backward': [0, 1, 2], 'experiment_planner_used': 'ExperimentPlanner', 'label_manager': 'LabelManager', 'foreground_intensity_properties_per_channel': {'0': {'max': 882.0, 'mean': 45.35713577270508, 'median': 48.0, 'min': -118.0, 'percentile_00_5': -48.0, 'percentile_99_5': 103.0, 'std': 26.203161239624023}}} + +2023-11-05 22:05:43.766089: unpacking dataset... +2023-11-05 22:05:52.983822: unpacking done... +2023-11-05 22:05:53.001199: do_dummy_2d_data_aug: False +2023-11-05 22:05:53.001824: Using splits from existing split file: ./data/nnUNet_preprocessed/Dataset721_TSPrimeCTVP/splits_final.json +2023-11-05 22:05:53.022583: The split file contains 5 splits. +2023-11-05 22:05:53.022662: Desired fold for training: 0 +2023-11-05 22:05:53.022732: This split has 48 training and 12 validation cases. +2023-11-05 22:06:32.735970: Unable to plot network architecture: +2023-11-05 22:06:32.736120: failed to execute PosixPath('dot'), make sure the Graphviz executables are on your systems' PATH +2023-11-05 22:06:32.837145: +2023-11-05 22:06:32.837218: Epoch 0 +2023-11-05 22:06:32.837328: Current learning rate: 0.01 +2023-11-05 22:20:54.774893: train_loss 0.0022 +2023-11-05 22:20:54.775081: val_loss -0.1688 +2023-11-05 22:20:54.775169: Pseudo dice [0.0] +2023-11-05 22:20:54.775265: Epoch time: 861.94 s +2023-11-05 22:20:54.775342: Yayy! New best EMA pseudo Dice: 0.0 +2023-11-05 22:20:56.371249: +2023-11-05 22:20:56.371469: Epoch 1 +2023-11-05 22:20:56.371657: Current learning rate: 0.00999 +2023-11-05 22:31:50.078054: train_loss -0.4984 +2023-11-05 22:31:50.078262: val_loss -0.6578 +2023-11-05 22:31:50.078343: Pseudo dice [0.7269] +2023-11-05 22:31:50.078430: Epoch time: 653.71 s +2023-11-05 22:31:50.078506: Yayy! New best EMA pseudo Dice: 0.0727 +2023-11-05 22:31:53.171840: +2023-11-05 22:31:53.172024: Epoch 2 +2023-11-05 22:31:53.172153: Current learning rate: 0.00998 +2023-11-05 22:42:48.347788: train_loss -0.6775 +2023-11-05 22:42:48.347950: val_loss -0.7266 +2023-11-05 22:42:48.348027: Pseudo dice [0.7937] +2023-11-05 22:42:48.348120: Epoch time: 655.18 s +2023-11-05 22:42:48.348189: Yayy! New best EMA pseudo Dice: 0.1448 +2023-11-05 22:42:51.579526: +2023-11-05 22:42:51.579650: Epoch 3 +2023-11-05 22:42:51.579753: Current learning rate: 0.00997 +2023-11-05 22:53:46.176134: train_loss -0.7185 +2023-11-05 22:53:46.176269: val_loss -0.7461 +2023-11-05 22:53:46.176361: Pseudo dice [0.8043] +2023-11-05 22:53:46.176449: Epoch time: 654.6 s +2023-11-05 22:53:46.176519: Yayy! New best EMA pseudo Dice: 0.2107 +2023-11-05 22:53:49.545828: +2023-11-05 22:53:49.545945: Epoch 4 +2023-11-05 22:53:49.546050: Current learning rate: 0.00996 +2023-11-05 23:04:43.303561: train_loss -0.7317 +2023-11-05 23:04:43.303715: val_loss -0.7646 +2023-11-05 23:04:43.303797: Pseudo dice [0.8278] +2023-11-05 23:04:43.303883: Epoch time: 653.76 s +2023-11-05 23:04:43.303957: Yayy! New best EMA pseudo Dice: 0.2725 +2023-11-05 23:04:46.916903: +2023-11-05 23:04:46.917109: Epoch 5 +2023-11-05 23:04:46.917217: Current learning rate: 0.00995 +2023-11-05 23:15:41.269564: train_loss -0.7454 +2023-11-05 23:15:41.269719: val_loss -0.776 +2023-11-05 23:15:41.269796: Pseudo dice [0.8309] +2023-11-05 23:15:41.269877: Epoch time: 654.35 s +2023-11-05 23:15:41.269946: Yayy! New best EMA pseudo Dice: 0.3283 +2023-11-05 23:15:44.430620: +2023-11-05 23:15:44.430745: Epoch 6 +2023-11-05 23:15:44.430851: Current learning rate: 0.00995 +2023-11-05 23:26:37.331294: train_loss -0.7739 +2023-11-05 23:26:37.331445: val_loss -0.7867 +2023-11-05 23:26:37.331522: Pseudo dice [0.8319] +2023-11-05 23:26:37.331612: Epoch time: 652.9 s +2023-11-05 23:26:37.331681: Yayy! New best EMA pseudo Dice: 0.3787 +2023-11-05 23:26:40.667429: +2023-11-05 23:26:40.667527: Epoch 7 +2023-11-05 23:26:40.667621: Current learning rate: 0.00994 +2023-11-05 23:37:36.057433: train_loss -0.7721 +2023-11-05 23:37:36.057588: val_loss -0.7854 +2023-11-05 23:37:36.057666: Pseudo dice [0.8414] +2023-11-05 23:37:36.057748: Epoch time: 655.39 s +2023-11-05 23:37:36.057818: Yayy! New best EMA pseudo Dice: 0.4249 +2023-11-05 23:37:39.249048: +2023-11-05 23:37:39.249249: Epoch 8 +2023-11-05 23:37:39.249468: Current learning rate: 0.00993 +2023-11-05 23:48:34.520747: train_loss -0.7639 +2023-11-05 23:48:34.520913: val_loss -0.7863 +2023-11-05 23:48:34.520990: Pseudo dice [0.8342] +2023-11-05 23:48:34.521071: Epoch time: 655.27 s +2023-11-05 23:48:34.521140: Yayy! New best EMA pseudo Dice: 0.4659 +2023-11-05 23:48:37.934005: +2023-11-05 23:48:37.934186: Epoch 9 +2023-11-05 23:48:37.934292: Current learning rate: 0.00992 +2023-11-05 23:59:31.744097: train_loss -0.7754 +2023-11-05 23:59:31.744260: val_loss -0.7949 +2023-11-05 23:59:31.744335: Pseudo dice [0.83] +2023-11-05 23:59:31.744416: Epoch time: 653.81 s +2023-11-05 23:59:31.744484: Yayy! New best EMA pseudo Dice: 0.5023 +2023-11-05 23:59:35.221768: +2023-11-05 23:59:35.221899: Epoch 10 +2023-11-05 23:59:35.222059: Current learning rate: 0.00991 +2023-11-06 00:10:30.542706: train_loss -0.7745 +2023-11-06 00:10:30.542849: val_loss -0.7849 +2023-11-06 00:10:30.542925: Pseudo dice [0.8183] +2023-11-06 00:10:30.543005: Epoch time: 655.32 s +2023-11-06 00:10:30.543073: Yayy! New best EMA pseudo Dice: 0.5339 +2023-11-06 00:10:33.828026: +2023-11-06 00:10:33.828140: Epoch 11 +2023-11-06 00:10:33.828271: Current learning rate: 0.0099 +2023-11-06 00:21:29.170071: train_loss -0.7769 +2023-11-06 00:21:29.170231: val_loss -0.7886 +2023-11-06 00:21:29.170306: Pseudo dice [0.8436] +2023-11-06 00:21:29.170388: Epoch time: 655.34 s +2023-11-06 00:21:29.170456: Yayy! New best EMA pseudo Dice: 0.5648 +2023-11-06 00:21:32.330414: +2023-11-06 00:21:32.330636: Epoch 12 +2023-11-06 00:21:32.330763: Current learning rate: 0.00989 +2023-11-06 00:32:31.568352: train_loss -0.7846 +2023-11-06 00:32:31.568504: val_loss -0.7826 +2023-11-06 00:32:31.568580: Pseudo dice [0.8258] +2023-11-06 00:32:31.568660: Epoch time: 659.24 s +2023-11-06 00:32:31.568729: Yayy! New best EMA pseudo Dice: 0.5909 +2023-11-06 00:32:34.977341: +2023-11-06 00:32:34.977602: Epoch 13 +2023-11-06 00:32:34.977876: Current learning rate: 0.00988 +2023-11-06 00:43:36.160741: train_loss -0.7869 +2023-11-06 00:43:36.160885: val_loss -0.7834 +2023-11-06 00:43:36.160981: Pseudo dice [0.8317] +2023-11-06 00:43:36.161071: Epoch time: 661.18 s +2023-11-06 00:43:36.161148: Yayy! New best EMA pseudo Dice: 0.615 +2023-11-06 00:43:42.027902: +2023-11-06 00:43:42.028009: Epoch 14 +2023-11-06 00:43:42.028111: Current learning rate: 0.00987 +2023-11-06 00:54:42.362827: train_loss -0.785 +2023-11-06 00:54:42.362983: val_loss -0.7905 +2023-11-06 00:54:42.363072: Pseudo dice [0.8346] +2023-11-06 00:54:42.363162: Epoch time: 660.34 s +2023-11-06 00:54:42.363242: Yayy! New best EMA pseudo Dice: 0.637 +2023-11-06 00:54:45.816739: +2023-11-06 00:54:45.816881: Epoch 15 +2023-11-06 00:54:45.816998: Current learning rate: 0.00986 +2023-11-06 01:05:39.600759: train_loss -0.7845 +2023-11-06 01:05:39.600898: val_loss -0.7838 +2023-11-06 01:05:39.600987: Pseudo dice [0.8295] +2023-11-06 01:05:39.601068: Epoch time: 653.78 s +2023-11-06 01:05:39.601136: Yayy! New best EMA pseudo Dice: 0.6562 +2023-11-06 01:05:43.176803: +2023-11-06 01:05:43.176908: Epoch 16 +2023-11-06 01:05:43.177009: Current learning rate: 0.00986 +2023-11-06 01:16:50.574224: train_loss -0.7986 +2023-11-06 01:16:50.574384: val_loss -0.8016 +2023-11-06 01:16:50.574479: Pseudo dice [0.845] +2023-11-06 01:16:50.574569: Epoch time: 667.4 s +2023-11-06 01:16:50.574645: Yayy! New best EMA pseudo Dice: 0.6751 +2023-11-06 01:16:53.886550: +2023-11-06 01:16:53.886660: Epoch 17 +2023-11-06 01:16:53.886781: Current learning rate: 0.00985 +2023-11-06 01:27:54.560711: train_loss -0.7931 +2023-11-06 01:27:54.560856: val_loss -0.81 +2023-11-06 01:27:54.560950: Pseudo dice [0.8518] +2023-11-06 01:27:54.561041: Epoch time: 660.67 s +2023-11-06 01:27:54.561118: Yayy! New best EMA pseudo Dice: 0.6928 +2023-11-06 01:27:57.874895: +2023-11-06 01:27:57.875184: Epoch 18 +2023-11-06 01:27:57.875363: Current learning rate: 0.00984 +2023-11-06 01:38:55.045314: train_loss -0.7949 +2023-11-06 01:38:55.045470: val_loss -0.7958 +2023-11-06 01:38:55.045547: Pseudo dice [0.828] +2023-11-06 01:38:55.045628: Epoch time: 657.17 s +2023-11-06 01:38:55.045697: Yayy! New best EMA pseudo Dice: 0.7063 +2023-11-06 01:38:58.180024: +2023-11-06 01:38:58.180334: Epoch 19 +2023-11-06 01:38:58.180515: Current learning rate: 0.00983 +2023-11-06 01:47:52.886719: train_loss -0.7978 +2023-11-06 01:47:52.886871: val_loss -0.8033 +2023-11-06 01:47:52.886948: Pseudo dice [0.8408] +2023-11-06 01:47:52.887028: Epoch time: 534.71 s +2023-11-06 01:47:52.887096: Yayy! New best EMA pseudo Dice: 0.7197 +2023-11-06 01:47:56.292678: +2023-11-06 01:47:56.292835: Epoch 20 +2023-11-06 01:47:56.292980: Current learning rate: 0.00982 +2023-11-06 01:55:44.553798: train_loss -0.8072 +2023-11-06 01:55:44.553949: val_loss -0.8016 +2023-11-06 01:55:44.554031: Pseudo dice [0.8365] +2023-11-06 01:55:44.554116: Epoch time: 468.26 s +2023-11-06 01:55:44.554189: Yayy! New best EMA pseudo Dice: 0.7314 +2023-11-06 01:55:47.659638: +2023-11-06 01:55:47.659833: Epoch 21 +2023-11-06 01:55:47.660119: Current learning rate: 0.00981 +2023-11-06 02:03:35.627137: train_loss -0.8073 +2023-11-06 02:03:35.627289: val_loss -0.809 +2023-11-06 02:03:35.627386: Pseudo dice [0.8394] +2023-11-06 02:03:35.627477: Epoch time: 467.97 s +2023-11-06 02:03:35.627555: Yayy! New best EMA pseudo Dice: 0.7422 +2023-11-06 02:03:38.898279: +2023-11-06 02:03:38.898384: Epoch 22 +2023-11-06 02:03:38.898483: Current learning rate: 0.0098 +2023-11-06 02:13:19.098450: train_loss -0.8012 +2023-11-06 02:13:19.098597: val_loss -0.794 +2023-11-06 02:13:19.098699: Pseudo dice [0.8268] +2023-11-06 02:13:19.098793: Epoch time: 580.2 s +2023-11-06 02:13:19.098870: Yayy! New best EMA pseudo Dice: 0.7507 +2023-11-06 02:13:22.567191: +2023-11-06 02:13:22.567326: Epoch 23 +2023-11-06 02:13:22.567432: Current learning rate: 0.00979 +2023-11-06 02:24:37.768249: train_loss -0.8039 +2023-11-06 02:24:37.768389: val_loss -0.8143 +2023-11-06 02:24:37.768477: Pseudo dice [0.8576] +2023-11-06 02:24:37.768565: Epoch time: 675.2 s +2023-11-06 02:24:37.768634: Yayy! New best EMA pseudo Dice: 0.7614 +2023-11-06 02:24:41.063595: +2023-11-06 02:24:41.063719: Epoch 24 +2023-11-06 02:24:41.063822: Current learning rate: 0.00978 +2023-11-06 02:35:56.216298: train_loss -0.8141 +2023-11-06 02:35:56.216446: val_loss -0.8112 +2023-11-06 02:35:56.216523: Pseudo dice [0.8314] +2023-11-06 02:35:56.216605: Epoch time: 675.15 s +2023-11-06 02:35:56.216681: Yayy! New best EMA pseudo Dice: 0.7684 +2023-11-06 02:35:59.473060: +2023-11-06 02:35:59.473267: Epoch 25 +2023-11-06 02:35:59.473394: Current learning rate: 0.00977 +2023-11-06 02:47:14.841282: train_loss -0.8238 +2023-11-06 02:47:14.841446: val_loss -0.8127 +2023-11-06 02:47:14.841526: Pseudo dice [0.8517] +2023-11-06 02:47:14.841607: Epoch time: 675.37 s +2023-11-06 02:47:14.841675: Yayy! New best EMA pseudo Dice: 0.7767 +2023-11-06 02:47:18.199337: +2023-11-06 02:47:18.199439: Epoch 26 +2023-11-06 02:47:18.199549: Current learning rate: 0.00977 +2023-11-06 02:58:33.463852: train_loss -0.8094 +2023-11-06 02:58:33.464010: val_loss -0.8178 +2023-11-06 02:58:33.464087: Pseudo dice [0.8539] +2023-11-06 02:58:33.464169: Epoch time: 675.27 s +2023-11-06 02:58:33.464237: Yayy! New best EMA pseudo Dice: 0.7844 +2023-11-06 02:58:36.681627: +2023-11-06 02:58:36.681730: Epoch 27 +2023-11-06 02:58:36.681832: Current learning rate: 0.00976 +2023-11-06 03:09:51.961360: train_loss -0.8114 +2023-11-06 03:09:51.961498: val_loss -0.7815 +2023-11-06 03:09:51.961591: Pseudo dice [0.8219] +2023-11-06 03:09:51.961673: Epoch time: 675.28 s +2023-11-06 03:09:51.961741: Yayy! New best EMA pseudo Dice: 0.7882 +2023-11-06 03:09:55.427147: +2023-11-06 03:09:55.427268: Epoch 28 +2023-11-06 03:09:55.427380: Current learning rate: 0.00975 +2023-11-06 03:21:10.383088: train_loss -0.7997 +2023-11-06 03:21:10.383252: val_loss -0.7981 +2023-11-06 03:21:10.383328: Pseudo dice [0.8349] +2023-11-06 03:21:10.383410: Epoch time: 674.96 s +2023-11-06 03:21:10.383477: Yayy! New best EMA pseudo Dice: 0.7928 +2023-11-06 03:21:13.641899: +2023-11-06 03:21:13.641999: Epoch 29 +2023-11-06 03:21:13.642121: Current learning rate: 0.00974 +2023-11-06 03:32:28.908478: train_loss -0.806 +2023-11-06 03:32:28.908639: val_loss -0.8071 +2023-11-06 03:32:28.908733: Pseudo dice [0.8445] +2023-11-06 03:32:28.908826: Epoch time: 675.27 s +2023-11-06 03:32:28.908903: Yayy! New best EMA pseudo Dice: 0.798 +2023-11-06 03:32:32.713846: +2023-11-06 03:32:32.714119: Epoch 30 +2023-11-06 03:32:32.714329: Current learning rate: 0.00973 +2023-11-06 03:43:47.583081: train_loss -0.8078 +2023-11-06 03:43:47.583235: val_loss -0.7946 +2023-11-06 03:43:47.583393: Pseudo dice [0.8358] +2023-11-06 03:43:47.583498: Epoch time: 674.87 s +2023-11-06 03:43:47.583567: Yayy! New best EMA pseudo Dice: 0.8018 +2023-11-06 03:43:50.914172: +2023-11-06 03:43:50.914282: Epoch 31 +2023-11-06 03:43:50.914385: Current learning rate: 0.00972 +2023-11-06 03:55:05.869875: train_loss -0.8134 +2023-11-06 03:55:05.870045: val_loss -0.8119 +2023-11-06 03:55:05.870121: Pseudo dice [0.8536] +2023-11-06 03:55:05.870202: Epoch time: 674.96 s +2023-11-06 03:55:05.870270: Yayy! New best EMA pseudo Dice: 0.807 +2023-11-06 03:55:09.261483: +2023-11-06 03:55:09.261622: Epoch 32 +2023-11-06 03:55:09.261738: Current learning rate: 0.00971 +2023-11-06 04:06:24.365536: train_loss -0.814 +2023-11-06 04:06:24.365684: val_loss -0.799 +2023-11-06 04:06:24.365768: Pseudo dice [0.8439] +2023-11-06 04:06:24.365851: Epoch time: 675.1 s +2023-11-06 04:06:24.365922: Yayy! New best EMA pseudo Dice: 0.8107 +2023-11-06 04:06:27.632730: +2023-11-06 04:06:27.632976: Epoch 33 +2023-11-06 04:06:27.633147: Current learning rate: 0.0097 +2023-11-06 04:17:42.891809: train_loss -0.8178 +2023-11-06 04:17:42.891958: val_loss -0.8109 +2023-11-06 04:17:42.892051: Pseudo dice [0.8469] +2023-11-06 04:17:42.892142: Epoch time: 675.26 s +2023-11-06 04:17:42.892221: Yayy! New best EMA pseudo Dice: 0.8143 +2023-11-06 04:17:46.135388: +2023-11-06 04:17:46.135488: Epoch 34 +2023-11-06 04:17:46.135598: Current learning rate: 0.00969 +2023-11-06 04:29:01.376284: train_loss -0.817 +2023-11-06 04:29:01.376450: val_loss -0.8104 +2023-11-06 04:29:01.376533: Pseudo dice [0.855] +2023-11-06 04:29:01.376614: Epoch time: 675.24 s +2023-11-06 04:29:01.376683: Yayy! New best EMA pseudo Dice: 0.8184 +2023-11-06 04:29:04.748322: +2023-11-06 04:29:04.748513: Epoch 35 +2023-11-06 04:29:04.748706: Current learning rate: 0.00968 +2023-11-06 04:40:19.958354: train_loss -0.8086 +2023-11-06 04:40:19.958521: val_loss -0.8051 +2023-11-06 04:40:19.958597: Pseudo dice [0.8335] +2023-11-06 04:40:19.958687: Epoch time: 675.21 s +2023-11-06 04:40:19.958761: Yayy! New best EMA pseudo Dice: 0.8199 +2023-11-06 04:40:23.530188: +2023-11-06 04:40:23.530292: Epoch 36 +2023-11-06 04:40:23.530401: Current learning rate: 0.00968 +2023-11-06 04:51:38.573648: train_loss -0.8159 +2023-11-06 04:51:38.573810: val_loss -0.8143 +2023-11-06 04:51:38.573904: Pseudo dice [0.8454] +2023-11-06 04:51:38.573995: Epoch time: 675.04 s +2023-11-06 04:51:38.574072: Yayy! New best EMA pseudo Dice: 0.8224 +2023-11-06 04:51:42.123522: +2023-11-06 04:51:42.123701: Epoch 37 +2023-11-06 04:51:42.123864: Current learning rate: 0.00967 +2023-11-06 05:02:57.064787: train_loss -0.8144 +2023-11-06 05:02:57.064952: val_loss -0.81 +2023-11-06 05:02:57.065049: Pseudo dice [0.8435] +2023-11-06 05:02:57.065142: Epoch time: 674.94 s +2023-11-06 05:02:57.065221: Yayy! New best EMA pseudo Dice: 0.8245 +2023-11-06 05:03:00.421975: +2023-11-06 05:03:00.422125: Epoch 38 +2023-11-06 05:03:00.422287: Current learning rate: 0.00966 +2023-11-06 05:14:15.585457: train_loss -0.8254 +2023-11-06 05:14:15.585616: val_loss -0.7961 +2023-11-06 05:14:15.585695: Pseudo dice [0.83] +2023-11-06 05:14:15.585777: Epoch time: 675.16 s +2023-11-06 05:14:15.585849: Yayy! New best EMA pseudo Dice: 0.8251 +2023-11-06 05:14:19.062620: +2023-11-06 05:14:19.062906: Epoch 39 +2023-11-06 05:14:19.063134: Current learning rate: 0.00965 +2023-11-06 05:25:34.054948: train_loss -0.8194 +2023-11-06 05:25:34.055121: val_loss -0.8239 +2023-11-06 05:25:34.055197: Pseudo dice [0.8627] +2023-11-06 05:25:34.055279: Epoch time: 674.99 s +2023-11-06 05:25:34.055348: Yayy! New best EMA pseudo Dice: 0.8288 +2023-11-06 05:25:37.355520: +2023-11-06 05:25:37.355628: Epoch 40 +2023-11-06 05:25:37.355733: Current learning rate: 0.00964 +2023-11-06 05:36:52.323830: train_loss -0.8123 +2023-11-06 05:36:52.323998: val_loss -0.8085 +2023-11-06 05:36:52.324074: Pseudo dice [0.8389] +2023-11-06 05:36:52.324154: Epoch time: 674.97 s +2023-11-06 05:36:52.324223: Yayy! New best EMA pseudo Dice: 0.8298 +2023-11-06 05:36:55.576262: +2023-11-06 05:36:55.576443: Epoch 41 +2023-11-06 05:36:55.576548: Current learning rate: 0.00963 +2023-11-06 05:48:11.067998: train_loss -0.8271 +2023-11-06 05:48:11.068160: val_loss -0.8201 +2023-11-06 05:48:11.068256: Pseudo dice [0.857] +2023-11-06 05:48:11.068349: Epoch time: 675.49 s +2023-11-06 05:48:11.068427: Yayy! New best EMA pseudo Dice: 0.8326 +2023-11-06 05:48:14.317959: +2023-11-06 05:48:14.318098: Epoch 42 +2023-11-06 05:48:14.318215: Current learning rate: 0.00962 +2023-11-06 05:59:29.383212: train_loss -0.8256 +2023-11-06 05:59:29.383366: val_loss -0.8221 +2023-11-06 05:59:29.383451: Pseudo dice [0.8575] +2023-11-06 05:59:29.383550: Epoch time: 675.07 s +2023-11-06 05:59:29.383627: Yayy! New best EMA pseudo Dice: 0.835 +2023-11-06 05:59:32.576050: +2023-11-06 05:59:32.576236: Epoch 43 +2023-11-06 05:59:32.576383: Current learning rate: 0.00961 +2023-11-06 06:10:47.765534: train_loss -0.8276 +2023-11-06 06:10:47.765698: val_loss -0.8066 +2023-11-06 06:10:47.765776: Pseudo dice [0.8416] +2023-11-06 06:10:47.765857: Epoch time: 675.19 s +2023-11-06 06:10:47.765927: Yayy! New best EMA pseudo Dice: 0.8357 +2023-11-06 06:10:51.414655: +2023-11-06 06:10:51.414837: Epoch 44 +2023-11-06 06:10:51.415005: Current learning rate: 0.0096 +2023-11-06 06:22:06.945473: train_loss -0.8151 +2023-11-06 06:22:06.945662: val_loss -0.8149 +2023-11-06 06:22:06.945739: Pseudo dice [0.842] +2023-11-06 06:22:06.945817: Epoch time: 675.53 s +2023-11-06 06:22:06.945896: Yayy! New best EMA pseudo Dice: 0.8363 +2023-11-06 06:22:10.103697: +2023-11-06 06:22:10.103803: Epoch 45 +2023-11-06 06:22:10.103918: Current learning rate: 0.00959 +2023-11-06 06:33:25.378408: train_loss -0.8281 +2023-11-06 06:33:25.378564: val_loss -0.8194 +2023-11-06 06:33:25.378639: Pseudo dice [0.8511] +2023-11-06 06:33:25.378727: Epoch time: 675.28 s +2023-11-06 06:33:25.378796: Yayy! New best EMA pseudo Dice: 0.8378 +2023-11-06 06:33:28.743157: +2023-11-06 06:33:28.743263: Epoch 46 +2023-11-06 06:33:28.743382: Current learning rate: 0.00959 +2023-11-06 06:44:43.864779: train_loss -0.8172 +2023-11-06 06:44:43.864925: val_loss -0.8076 +2023-11-06 06:44:43.865001: Pseudo dice [0.8374] +2023-11-06 06:44:43.865081: Epoch time: 675.12 s +2023-11-06 06:44:45.082992: +2023-11-06 06:44:45.083101: Epoch 47 +2023-11-06 06:44:45.083211: Current learning rate: 0.00958 +2023-11-06 06:55:59.903310: train_loss -0.8225 +2023-11-06 06:55:59.903490: val_loss -0.8182 +2023-11-06 06:55:59.903566: Pseudo dice [0.8473] +2023-11-06 06:55:59.903647: Epoch time: 674.82 s +2023-11-06 06:55:59.903717: Yayy! New best EMA pseudo Dice: 0.8387 +2023-11-06 06:56:03.120151: +2023-11-06 06:56:03.120266: Epoch 48 +2023-11-06 06:56:03.120366: Current learning rate: 0.00957 +2023-11-06 07:07:18.367775: train_loss -0.8177 +2023-11-06 07:07:18.367926: val_loss -0.809 +2023-11-06 07:07:18.368003: Pseudo dice [0.834] +2023-11-06 07:07:18.368084: Epoch time: 675.25 s +2023-11-06 07:07:19.602919: +2023-11-06 07:07:19.603036: Epoch 49 +2023-11-06 07:07:19.603153: Current learning rate: 0.00956 +2023-11-06 07:18:34.630416: train_loss -0.8252 +2023-11-06 07:18:34.630569: val_loss -0.8214 +2023-11-06 07:18:34.630645: Pseudo dice [0.8596] +2023-11-06 07:18:34.630736: Epoch time: 675.03 s +2023-11-06 07:18:34.913779: Yayy! New best EMA pseudo Dice: 0.8404 +2023-11-06 07:18:38.290022: +2023-11-06 07:18:38.290306: Epoch 50 +2023-11-06 07:18:38.290472: Current learning rate: 0.00955 +2023-11-06 07:29:52.160251: train_loss -0.8324 +2023-11-06 07:29:52.160417: val_loss -0.8225 +2023-11-06 07:29:52.160571: Pseudo dice [0.8528] +2023-11-06 07:29:52.160689: Epoch time: 673.87 s +2023-11-06 07:29:52.160758: Yayy! New best EMA pseudo Dice: 0.8416 +2023-11-06 07:29:55.579350: +2023-11-06 07:29:55.579564: Epoch 51 +2023-11-06 07:29:55.579664: Current learning rate: 0.00954 +2023-11-06 07:41:10.809812: train_loss -0.8262 +2023-11-06 07:41:10.809976: val_loss -0.8195 +2023-11-06 07:41:10.810052: Pseudo dice [0.8512] +2023-11-06 07:41:10.810133: Epoch time: 675.23 s +2023-11-06 07:41:10.810200: Yayy! New best EMA pseudo Dice: 0.8426 +2023-11-06 07:41:14.117213: +2023-11-06 07:41:14.117320: Epoch 52 +2023-11-06 07:41:14.117436: Current learning rate: 0.00953 +2023-11-06 07:52:29.440799: train_loss -0.8204 +2023-11-06 07:52:29.440960: val_loss -0.8119 +2023-11-06 07:52:29.441036: Pseudo dice [0.8433] +2023-11-06 07:52:29.441117: Epoch time: 675.32 s +2023-11-06 07:52:29.441186: Yayy! New best EMA pseudo Dice: 0.8427 +2023-11-06 07:52:32.941524: +2023-11-06 07:52:32.941636: Epoch 53 +2023-11-06 07:52:32.941733: Current learning rate: 0.00952 +2023-11-06 08:03:48.102913: train_loss -0.8276 +2023-11-06 08:03:48.103097: val_loss -0.8167 +2023-11-06 08:03:48.103188: Pseudo dice [0.861] +2023-11-06 08:03:48.103277: Epoch time: 675.16 s +2023-11-06 08:03:48.103354: Yayy! New best EMA pseudo Dice: 0.8445 +2023-11-06 08:03:51.334111: +2023-11-06 08:03:51.334226: Epoch 54 +2023-11-06 08:03:51.334339: Current learning rate: 0.00951 +2023-11-06 08:15:06.513220: train_loss -0.8313 +2023-11-06 08:15:06.513460: val_loss -0.8093 +2023-11-06 08:15:06.513551: Pseudo dice [0.8479] +2023-11-06 08:15:06.513641: Epoch time: 675.18 s +2023-11-06 08:15:06.513718: Yayy! New best EMA pseudo Dice: 0.8448 +2023-11-06 08:15:09.747582: +2023-11-06 08:15:09.747695: Epoch 55 +2023-11-06 08:15:09.747810: Current learning rate: 0.0095 +2023-11-06 08:26:25.128954: train_loss -0.8234 +2023-11-06 08:26:25.129091: val_loss -0.8087 +2023-11-06 08:26:25.129179: Pseudo dice [0.8479] +2023-11-06 08:26:25.129266: Epoch time: 675.38 s +2023-11-06 08:26:25.129333: Yayy! New best EMA pseudo Dice: 0.8451 +2023-11-06 08:26:28.348276: +2023-11-06 08:26:28.348374: Epoch 56 +2023-11-06 08:26:28.348496: Current learning rate: 0.00949 +2023-11-06 08:37:43.825908: train_loss -0.8124 +2023-11-06 08:37:43.826058: val_loss -0.8055 +2023-11-06 08:37:43.826153: Pseudo dice [0.8376] +2023-11-06 08:37:43.826247: Epoch time: 675.48 s +2023-11-06 08:37:45.064820: +2023-11-06 08:37:45.064925: Epoch 57 +2023-11-06 08:37:45.065036: Current learning rate: 0.00949 +2023-11-06 08:49:00.095835: train_loss -0.8276 +2023-11-06 08:49:00.096008: val_loss -0.8191 +2023-11-06 08:49:00.096088: Pseudo dice [0.8536] +2023-11-06 08:49:00.096174: Epoch time: 675.03 s +2023-11-06 08:49:00.096246: Yayy! New best EMA pseudo Dice: 0.8453 +2023-11-06 08:49:03.525604: +2023-11-06 08:49:03.525789: Epoch 58 +2023-11-06 08:49:03.525964: Current learning rate: 0.00948 +2023-11-06 09:00:18.875708: train_loss -0.8285 +2023-11-06 09:00:18.875870: val_loss -0.8237 +2023-11-06 09:00:18.875946: Pseudo dice [0.8581] +2023-11-06 09:00:18.876027: Epoch time: 675.35 s +2023-11-06 09:00:18.876095: Yayy! New best EMA pseudo Dice: 0.8466 +2023-11-06 09:00:22.098460: +2023-11-06 09:00:22.098590: Epoch 59 +2023-11-06 09:00:22.098714: Current learning rate: 0.00947 +2023-11-06 09:11:37.533084: train_loss -0.8272 +2023-11-06 09:11:37.533242: val_loss -0.791 +2023-11-06 09:11:37.533318: Pseudo dice [0.8158] +2023-11-06 09:11:37.533400: Epoch time: 675.44 s +2023-11-06 09:11:38.800722: +2023-11-06 09:11:38.800860: Epoch 60 +2023-11-06 09:11:38.800962: Current learning rate: 0.00946 +2023-11-06 09:22:53.990348: train_loss -0.8271 +2023-11-06 09:22:53.990512: val_loss -0.8294 +2023-11-06 09:22:53.990592: Pseudo dice [0.8748] +2023-11-06 09:22:53.990684: Epoch time: 675.19 s +2023-11-06 09:22:53.990766: Yayy! New best EMA pseudo Dice: 0.8466 +2023-11-06 09:22:57.349074: +2023-11-06 09:22:57.349289: Epoch 61 +2023-11-06 09:22:57.349463: Current learning rate: 0.00945 +2023-11-06 09:34:12.470989: train_loss -0.8302 +2023-11-06 09:34:12.471159: val_loss -0.8152 +2023-11-06 09:34:12.471240: Pseudo dice [0.8533] +2023-11-06 09:34:12.471326: Epoch time: 675.12 s +2023-11-06 09:34:12.471398: Yayy! New best EMA pseudo Dice: 0.8473 +2023-11-06 09:34:15.608224: +2023-11-06 09:34:15.608330: Epoch 62 +2023-11-06 09:34:15.608428: Current learning rate: 0.00944 +2023-11-06 09:45:30.830228: train_loss -0.8108 +2023-11-06 09:45:30.830392: val_loss -0.8124 +2023-11-06 09:45:30.830467: Pseudo dice [0.8537] +2023-11-06 09:45:30.830546: Epoch time: 675.22 s +2023-11-06 09:45:30.830615: Yayy! New best EMA pseudo Dice: 0.8479 +2023-11-06 09:45:33.973119: +2023-11-06 09:45:33.973341: Epoch 63 +2023-11-06 09:45:33.973521: Current learning rate: 0.00943 +2023-11-06 09:56:48.860524: train_loss -0.8232 +2023-11-06 09:56:48.860660: val_loss -0.7923 +2023-11-06 09:56:48.860746: Pseudo dice [0.8356] +2023-11-06 09:56:48.860831: Epoch time: 674.89 s +2023-11-06 09:56:50.298269: +2023-11-06 09:56:50.298393: Epoch 64 +2023-11-06 09:56:50.298507: Current learning rate: 0.00942 +2023-11-06 10:08:05.669342: train_loss -0.8194 +2023-11-06 10:08:05.669511: val_loss -0.8066 +2023-11-06 10:08:05.669608: Pseudo dice [0.8408] +2023-11-06 10:08:05.669699: Epoch time: 675.37 s +2023-11-06 10:08:06.933493: +2023-11-06 10:08:06.933608: Epoch 65 +2023-11-06 10:08:06.933718: Current learning rate: 0.00941 +2023-11-06 10:19:22.076491: train_loss -0.8295 +2023-11-06 10:19:22.076648: val_loss -0.8209 +2023-11-06 10:19:22.076744: Pseudo dice [0.8535] +2023-11-06 10:19:22.076838: Epoch time: 675.14 s +2023-11-06 10:19:23.367105: +2023-11-06 10:19:23.367233: Epoch 66 +2023-11-06 10:19:23.367350: Current learning rate: 0.0094 +2023-11-06 10:30:38.725112: train_loss -0.825 +2023-11-06 10:30:38.725279: val_loss -0.8273 +2023-11-06 10:30:38.725372: Pseudo dice [0.8589] +2023-11-06 10:30:38.725462: Epoch time: 675.36 s +2023-11-06 10:30:38.725539: Yayy! New best EMA pseudo Dice: 0.8481 +2023-11-06 10:30:42.055256: +2023-11-06 10:30:42.055468: Epoch 67 +2023-11-06 10:30:42.055651: Current learning rate: 0.00939 +2023-11-06 10:41:57.480700: train_loss -0.8361 +2023-11-06 10:41:57.480846: val_loss -0.8078 +2023-11-06 10:41:57.480921: Pseudo dice [0.8426] +2023-11-06 10:41:57.481000: Epoch time: 675.43 s +2023-11-06 10:41:58.767543: +2023-11-06 10:41:58.767738: Epoch 68 +2023-11-06 10:41:58.767924: Current learning rate: 0.00939 +2023-11-06 10:53:14.281489: train_loss -0.835 +2023-11-06 10:53:14.281637: val_loss -0.8196 +2023-11-06 10:53:14.281713: Pseudo dice [0.8561] +2023-11-06 10:53:14.281793: Epoch time: 675.51 s +2023-11-06 10:53:14.281860: Yayy! New best EMA pseudo Dice: 0.8484 +2023-11-06 10:53:17.731063: +2023-11-06 10:53:17.731177: Epoch 69 +2023-11-06 10:53:17.731296: Current learning rate: 0.00938 +2023-11-06 11:04:32.854343: train_loss -0.8306 +2023-11-06 11:04:32.854506: val_loss -0.8266 +2023-11-06 11:04:32.854598: Pseudo dice [0.8572] +2023-11-06 11:04:32.854696: Epoch time: 675.12 s +2023-11-06 11:04:32.854775: Yayy! New best EMA pseudo Dice: 0.8493 +2023-11-06 11:04:36.198146: +2023-11-06 11:04:36.198331: Epoch 70 +2023-11-06 11:04:36.198503: Current learning rate: 0.00937 +2023-11-06 11:15:51.137854: train_loss -0.8385 +2023-11-06 11:15:51.138000: val_loss -0.8133 +2023-11-06 11:15:51.138093: Pseudo dice [0.8431] +2023-11-06 11:15:51.138183: Epoch time: 674.94 s +2023-11-06 11:15:52.630492: +2023-11-06 11:15:52.630608: Epoch 71 +2023-11-06 11:15:52.630735: Current learning rate: 0.00936 +2023-11-06 11:27:07.819234: train_loss -0.8339 +2023-11-06 11:27:07.819484: val_loss -0.8214 +2023-11-06 11:27:07.819611: Pseudo dice [0.8487] +2023-11-06 11:27:07.819704: Epoch time: 675.19 s +2023-11-06 11:27:09.109598: +2023-11-06 11:27:09.109764: Epoch 72 +2023-11-06 11:27:09.109937: Current learning rate: 0.00935 +2023-11-06 11:38:24.488279: train_loss -0.84 +2023-11-06 11:38:24.488451: val_loss -0.8316 +2023-11-06 11:38:24.488527: Pseudo dice [0.8615] +2023-11-06 11:38:24.488606: Epoch time: 675.38 s +2023-11-06 11:38:24.488675: Yayy! New best EMA pseudo Dice: 0.8499 +2023-11-06 11:38:27.703335: +2023-11-06 11:38:27.703459: Epoch 73 +2023-11-06 11:38:27.703562: Current learning rate: 0.00934 +2023-11-06 11:49:43.036126: train_loss -0.8416 +2023-11-06 11:49:43.036298: val_loss -0.8257 +2023-11-06 11:49:43.036396: Pseudo dice [0.8603] +2023-11-06 11:49:43.036489: Epoch time: 675.33 s +2023-11-06 11:49:43.036568: Yayy! New best EMA pseudo Dice: 0.851 +2023-11-06 11:49:46.209569: +2023-11-06 11:49:46.209720: Epoch 74 +2023-11-06 11:49:46.209849: Current learning rate: 0.00933 +2023-11-06 12:01:01.503134: train_loss -0.8418 +2023-11-06 12:01:01.503297: val_loss -0.8301 +2023-11-06 12:01:01.503382: Pseudo dice [0.8594] +2023-11-06 12:01:01.503466: Epoch time: 675.29 s +2023-11-06 12:01:01.503535: Yayy! New best EMA pseudo Dice: 0.8518 +2023-11-06 12:01:04.783809: +2023-11-06 12:01:04.783910: Epoch 75 +2023-11-06 12:01:04.784027: Current learning rate: 0.00932 +2023-11-06 12:12:19.949506: train_loss -0.8463 +2023-11-06 12:12:19.949670: val_loss -0.8123 +2023-11-06 12:12:19.949746: Pseudo dice [0.8351] +2023-11-06 12:12:19.949828: Epoch time: 675.17 s +2023-11-06 12:12:21.242239: +2023-11-06 12:12:21.242361: Epoch 76 +2023-11-06 12:12:21.242465: Current learning rate: 0.00931 +2023-11-06 12:23:36.276375: train_loss -0.8364 +2023-11-06 12:23:36.276535: val_loss -0.8107 +2023-11-06 12:23:36.276609: Pseudo dice [0.8454] +2023-11-06 12:23:36.276692: Epoch time: 675.03 s +2023-11-06 12:23:37.558540: +2023-11-06 12:23:37.558640: Epoch 77 +2023-11-06 12:23:37.558762: Current learning rate: 0.0093 +2023-11-06 12:34:52.820284: train_loss -0.8417 +2023-11-06 12:34:52.820433: val_loss -0.8144 +2023-11-06 12:34:52.820513: Pseudo dice [0.8607] +2023-11-06 12:34:52.820607: Epoch time: 675.26 s +2023-11-06 12:34:54.330649: +2023-11-06 12:34:54.330824: Epoch 78 +2023-11-06 12:34:54.330933: Current learning rate: 0.0093 +2023-11-06 12:46:09.541902: train_loss -0.8413 +2023-11-06 12:46:09.542046: val_loss -0.8263 +2023-11-06 12:46:09.542131: Pseudo dice [0.8589] +2023-11-06 12:46:09.542212: Epoch time: 675.21 s +2023-11-06 12:46:10.856498: +2023-11-06 12:46:10.856613: Epoch 79 +2023-11-06 12:46:10.856715: Current learning rate: 0.00929 +2023-11-06 12:57:25.954721: train_loss -0.8449 +2023-11-06 12:57:25.954871: val_loss -0.8188 +2023-11-06 12:57:25.954963: Pseudo dice [0.8579] +2023-11-06 12:57:25.955053: Epoch time: 675.1 s +2023-11-06 12:57:25.955130: Yayy! New best EMA pseudo Dice: 0.8522 +2023-11-06 12:57:29.184287: +2023-11-06 12:57:29.184399: Epoch 80 +2023-11-06 12:57:29.184514: Current learning rate: 0.00928 +2023-11-06 13:08:44.293826: train_loss -0.831 +2023-11-06 13:08:44.293987: val_loss -0.825 +2023-11-06 13:08:44.294063: Pseudo dice [0.8536] +2023-11-06 13:08:44.294145: Epoch time: 675.11 s +2023-11-06 13:08:44.294213: Yayy! New best EMA pseudo Dice: 0.8523 +2023-11-06 13:08:47.574882: +2023-11-06 13:08:47.574989: Epoch 81 +2023-11-06 13:08:47.575088: Current learning rate: 0.00927 diff --git a/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/plans.json b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/plans.json new file mode 100755 index 0000000000000000000000000000000000000000..40aa5ba1694eff82b85f4309a1544a263ddcca69 --- /dev/null +++ b/Dataset721_TSPrimeCTVP/nnUNetTrainer__nnUNetPlans__3d_lowres/plans.json @@ -0,0 +1,454 @@ +{ + "dataset_name": "Dataset721_TSPrimeCTVP", + "plans_name": "nnUNetPlans", + "original_median_spacing_after_transp": [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + "original_median_shape_after_transp": [ + 241, + 512, + 512 + ], + "image_reader_writer": "SimpleITKIO", + "transpose_forward": [ + 0, + 1, + 2 + ], + "transpose_backward": [ + 0, + 1, + 2 + ], + "configurations": { + "2d": { + "data_identifier": "nnUNetPlans_2d", + "preprocessor_name": "DefaultPreprocessor", + "batch_size": 12, + "patch_size": [ + 512, + 512 + ], + "median_image_size_in_voxels": [ + 512.0, + 512.0 + ], + "spacing": [ + 1.269531011581421, + 1.269531011581421 + ], + "normalization_schemes": [ + "CTNormalization" + ], + "use_mask_for_norm": [ + false + ], + "UNet_class_name": "PlainConvUNet", + "UNet_base_num_features": 32, + "n_conv_per_stage_encoder": [ + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_conv_per_stage_decoder": [ + 2, + 2, + 2, + 2, + 2, + 2, + 2 + ], + "num_pool_per_axis": [ + 7, + 7 + ], + "pool_op_kernel_sizes": [ + [ + 1, + 1 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ], + [ + 2, + 2 + ] + ], + "conv_kernel_sizes": [ + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ], + [ + 3, + 3 + ] + ], + "unet_max_num_features": 512, + "resampling_fn_data": "resample_data_or_seg_to_shape", + "resampling_fn_seg": "resample_data_or_seg_to_shape", + "resampling_fn_data_kwargs": { + "is_seg": false, + "order": 3, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_seg_kwargs": { + "is_seg": true, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_probabilities": "resample_data_or_seg_to_shape", + "resampling_fn_probabilities_kwargs": { + "is_seg": false, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "batch_dice": true + }, + "3d_lowres": { + "data_identifier": "nnUNetPlans_3d_lowres", + "preprocessor_name": "DefaultPreprocessor", + "batch_size": 2, + "patch_size": [ + 80, + 192, + 160 + ], + "median_image_size_in_voxels": [ + 130, + 275, + 275 + ], + "spacing": [ + 4.650736429273743, + 2.361701649461784, + 2.361701649461784 + ], + "normalization_schemes": [ + "CTNormalization" + ], + "use_mask_for_norm": [ + false + ], + "UNet_class_name": "PlainConvUNet", + "UNet_base_num_features": 32, + "n_conv_per_stage_encoder": [ + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_conv_per_stage_decoder": [ + 2, + 2, + 2, + 2, + 2 + ], + "num_pool_per_axis": [ + 4, + 5, + 5 + ], + "pool_op_kernel_sizes": [ + [ + 1, + 1, + 1 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 1, + 2, + 2 + ] + ], + "conv_kernel_sizes": [ + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ] + ], + "unet_max_num_features": 320, + "resampling_fn_data": "resample_data_or_seg_to_shape", + "resampling_fn_seg": "resample_data_or_seg_to_shape", + "resampling_fn_data_kwargs": { + "is_seg": false, + "order": 3, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_seg_kwargs": { + "is_seg": true, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_probabilities": "resample_data_or_seg_to_shape", + "resampling_fn_probabilities_kwargs": { + "is_seg": false, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "batch_dice": false, + "next_stage": "3d_cascade_fullres" + }, + "3d_fullres": { + "data_identifier": "nnUNetPlans_3d_fullres", + "preprocessor_name": "DefaultPreprocessor", + "batch_size": 2, + "patch_size": [ + 80, + 192, + 160 + ], + "median_image_size_in_voxels": [ + 241.0, + 512.0, + 512.0 + ], + "spacing": [ + 2.5, + 1.269531011581421, + 1.269531011581421 + ], + "normalization_schemes": [ + "CTNormalization" + ], + "use_mask_for_norm": [ + false + ], + "UNet_class_name": "PlainConvUNet", + "UNet_base_num_features": 32, + "n_conv_per_stage_encoder": [ + 2, + 2, + 2, + 2, + 2, + 2 + ], + "n_conv_per_stage_decoder": [ + 2, + 2, + 2, + 2, + 2 + ], + "num_pool_per_axis": [ + 4, + 5, + 5 + ], + "pool_op_kernel_sizes": [ + [ + 1, + 1, + 1 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 2, + 2, + 2 + ], + [ + 1, + 2, + 2 + ] + ], + "conv_kernel_sizes": [ + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ], + [ + 3, + 3, + 3 + ] + ], + "unet_max_num_features": 320, + "resampling_fn_data": "resample_data_or_seg_to_shape", + "resampling_fn_seg": "resample_data_or_seg_to_shape", + "resampling_fn_data_kwargs": { + "is_seg": false, + "order": 3, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_seg_kwargs": { + "is_seg": true, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "resampling_fn_probabilities": "resample_data_or_seg_to_shape", + "resampling_fn_probabilities_kwargs": { + "is_seg": false, + "order": 1, + "order_z": 0, + "force_separate_z": null + }, + "batch_dice": true + }, + "3d_cascade_fullres": { + "inherits_from": "3d_fullres", + "previous_stage": "3d_lowres" + } + }, + "experiment_planner_used": "ExperimentPlanner", + "label_manager": "LabelManager", + "foreground_intensity_properties_per_channel": { + "0": { + "max": 882.0, + "mean": 45.35713577270508, + "median": 48.0, + "min": -118.0, + "percentile_00_5": -48.0, + "percentile_99_5": 103.0, + "std": 26.203161239624023 + } + } +} \ No newline at end of file