Upload 6 files

checkpoints/log_LRS2_lip_dprnn_3spk/config.yaml

@@ -0,0 +1,42 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 3
+mix_lst_path: ./data/LRS2/mixture_data_list_3mix.csv
+audio_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS2/audio_clean/
+reference_direc: /mnt/nas_sg/wulanchabu/zexu.pan/datasets/LRS2/mvlrs_v1/
+audio_sr: 16000
+ref_sr: 25
+
+# dataloader
+num_workers: 4
+batch_size: 4         # four GPU training with a total effective batch size of 16
+accu_grad: 0
+effec_batch_size: 4   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 6         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: av_dprnn
+  N: 256
+  L: 40
+  B: 64
+  H: 128
+  K: 100
+  R: 6
+
+# optimizer
+loss_type: sisdr      # "snr", "sisdr", "hybrid"
+init_learning_rate: 0.001
+max_epoch: 150
+clip_grad_norm: 5

checkpoints/log_LRS2_lip_dprnn_3spk/last_best_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce64d2712ed4fa29894c3b1f9c2a8332133100df96342dc098cbf63e30c2211c
+size 94585962

checkpoints/log_LRS2_lip_dprnn_3spk/last_checkpoint.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ba15915d38ca0e4dd092575fa9aecb74f3829f2a1a78f42c284203e9805acdf2
+size 94585962

checkpoints/log_LRS2_lip_dprnn_3spk/log_2024-09-26(16:47:55).txt

@@ -0,0 +1,586 @@
+## Config file
+
+# Log 
+seed: 777
+use_cuda: 1           # 1 for True, 0 for False
+
+# dataset
+speaker_no: 3
+mix_lst_path: ./data/LRS2/mixture_data_list_3mix.csv
+audio_direc: /data4/zexu.pan/datasets/LRS2/audio_clean/
+reference_direc: /data4/zexu.pan/datasets/LRS2/mvlrs_v1/
+audio_sr: 16000
+visual_sr: 25
+
+# dataloader
+num_workers: 4
+batch_size: 4         # four GPU training with a total effective batch size of 16
+accu_grad: 0
+effec_batch_size: 4   # per GPU, only used if accu_grad is set to 1, must be multiple times of batch size
+max_length: 6         # truncate the utterances in dataloader, in seconds 
+
+# network settings
+init_from: None       # 'None' or a log name 'log_2024-07-22(18:12:13)'
+causal: 0             # 1 for True, 0 for False
+network_reference:
+  cue: lip            # lip or speech or gesture or EEG
+  backbone: resnet18  # resnet18 or shufflenetV2 or blazenet64
+  emb_size: 256       # resnet18:256
+network_audio:
+  backbone: dprnn
+  N: 256
+  L: 40
+  B: 64
+  H: 128
+  K: 100
+  R: 6
+
+# optimizer
+loss_type: sisdr      # "snr", "sisdr", "hybrid"
+init_learning_rate: 0.001
+max_epoch: 150
+clip_grad_norm: 5
+W0926 16:47:57.512017 140558052996864 torch/distributed/run.py:779] 
+W0926 16:47:57.512017 140558052996864 torch/distributed/run.py:779] *****************************************
+W0926 16:47:57.512017 140558052996864 torch/distributed/run.py:779] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
+W0926 16:47:57.512017 140558052996864 torch/distributed/run.py:779] *****************************************
+started on checkpoints/log_2024-09-26(16:47:55)
+
+namespace(accu_grad=0, audio_direc='/data4/zexu.pan/datasets/LRS2/audio_clean/', audio_sr=16000, batch_size=4, causal=0, checkpoint_dir='checkpoints/log_2024-09-26(16:47:55)', clip_grad_norm=5.0, config=[<yamlargparse.Path object at 0x7fc53e739c40>], device=device(type='cuda'), distributed=True, effec_batch_size=4, init_from='None', init_learning_rate=0.001, local_rank=0, loss_type='sisdr', max_epoch=150, max_length=6, mix_lst_path='./data/LRS2/mixture_data_list_3mix.csv', network_audio=namespace(B=64, H=128, K=100, L=40, N=256, R=6, backbone='dprnn'), network_reference=namespace(backbone='resnet18', cue='lip', emb_size=256), num_workers=4, reference_direc='/data4/zexu.pan/datasets/LRS2/mvlrs_v1/', seed=777, speaker_no=3, train_from_last_checkpoint=0, use_cuda=1, visual_sr=25, world_size=4)
+network_wrapper(
+  (sep_network): Dprnn(
+    (encoder): Encoder(
+      (conv1d_U): Conv1d(1, 256, kernel_size=(40,), stride=(20,), bias=False)
+    )
+    (separator): rnn(
+      (layer_norm): GroupNorm(1, 256, eps=1e-08, affine=True)
+      (bottleneck_conv1x1): Conv1d(256, 64, kernel_size=(1,), stride=(1,), bias=False)
+      (dual_rnn): ModuleList(
+        (0-5): 6 x Dual_RNN_Block(
+          (intra_rnn): LSTM(64, 128, batch_first=True, bidirectional=True)
+          (inter_rnn): LSTM(64, 128, batch_first=True, bidirectional=True)
+          (intra_norm): GroupNorm(1, 64, eps=1e-08, affine=True)
+          (inter_norm): GroupNorm(1, 64, eps=1e-08, affine=True)
+          (intra_linear): Linear(in_features=256, out_features=64, bias=True)
+          (inter_linear): Linear(in_features=256, out_features=64, bias=True)
+        )
+      )
+      (prelu): PReLU(num_parameters=1)
+      (mask_conv1x1): Conv1d(64, 256, kernel_size=(1,), stride=(1,), bias=False)
+      (av_conv): Conv1d(320, 64, kernel_size=(1,), stride=(1,), bias=False)
+    )
+    (decoder): Decoder(
+      (basis_signals): Linear(in_features=256, out_features=40, bias=False)
+    )
+  )
+  (ref_encoder): Visual_encoder(
+    (v_frontend): VisualFrontend(
+      (frontend3D): Sequential(
+        (0): Conv3d(1, 64, kernel_size=(5, 7, 7), stride=(1, 2, 2), padding=(2, 3, 3), bias=False)
+        (1): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        (2): ReLU()
+        (3): MaxPool3d(kernel_size=(1, 3, 3), stride=(1, 2, 2), padding=(0, 1, 1), dilation=1, ceil_mode=False)
+      )
+      (resnet): ResNet(
+        (layer1): ResNetLayer(
+          (conv1a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)
+          (outbna): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(64, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer2): ResNetLayer(
+          (conv1a): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(128, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer3): ResNetLayer(
+          (conv1a): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(256, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (layer4): ResNetLayer(
+          (conv1a): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)
+          (bn1a): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2a): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (downsample): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)
+          (outbna): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv1b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (bn1b): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+          (conv2b): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)
+          (outbnb): SyncBatchNorm(512, eps=0.001, momentum=0.01, affine=True, track_running_stats=True)
+        )
+        (avgpool): AvgPool2d(kernel_size=(4, 4), stride=(1, 1), padding=0)
+      )
+    )
+    (v_ds): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+    (visual_conv): Sequential(
+      (0): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (1): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (2): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (3): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+      (4): VisualConv1D(
+        (relu_0): ReLU()
+        (norm_0): SyncBatchNorm(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (conv1x1): Conv1d(256, 512, kernel_size=(1,), stride=(1,), bias=False)
+        (relu): ReLU()
+        (norm_1): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (dsconv): Conv1d(512, 512, kernel_size=(3,), stride=(1,), padding=(1,), groups=512)
+        (prelu): PReLU(num_parameters=1)
+        (norm_2): SyncBatchNorm(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+        (pw_conv): Conv1d(512, 256, kernel_size=(1,), stride=(1,), bias=False)
+      )
+    )
+  )
+)
+
+Total number of parameters: 15306950 
+
+
+Total number of trainable parameters: 4121862 
+
+Start new training from scratch
+[rank1]:[W926 16:48:04.919168280 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank0]:[W926 16:48:04.925726508 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank3]:[W926 16:48:04.926579292 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+[rank2]:[W926 16:48:04.929743722 reducer.cpp:1400] Warning: find_unused_parameters=True was specified in DDP constructor, but did not find any unused parameters in the forward pass. This flag results in an extra traversal of the autograd graph every iteration,  which can adversely affect performance. If your model indeed never has any unused parameters in the forward pass, consider turning this flag off. Note that this warning may be a false positive if your model has flow control causing later iterations to have unused parameters. (function operator())
+Train Summary | End of Epoch 1 | Time 716.64s | Train Loss 0.882
+Valid Summary | End of Epoch 1 | Time 34.35s | Valid Loss 0.293
+Test Summary | End of Epoch 1 | Time 13.53s | Test Loss 0.293
+Fund new best model, dict saved
+Train Summary | End of Epoch 2 | Time 711.83s | Train Loss -0.253
+Valid Summary | End of Epoch 2 | Time 32.94s | Valid Loss -0.279
+Test Summary | End of Epoch 2 | Time 13.85s | Test Loss -0.279
+Fund new best model, dict saved
+Train Summary | End of Epoch 3 | Time 707.50s | Train Loss -0.751
+Valid Summary | End of Epoch 3 | Time 32.74s | Valid Loss -0.641
+Test Summary | End of Epoch 3 | Time 13.96s | Test Loss -0.641
+Fund new best model, dict saved
+Train Summary | End of Epoch 4 | Time 715.27s | Train Loss -1.100
+Valid Summary | End of Epoch 4 | Time 35.06s | Valid Loss -1.026
+Test Summary | End of Epoch 4 | Time 14.07s | Test Loss -1.026
+Fund new best model, dict saved
+Train Summary | End of Epoch 5 | Time 721.61s | Train Loss -1.348
+Valid Summary | End of Epoch 5 | Time 34.71s | Valid Loss -1.178
+Test Summary | End of Epoch 5 | Time 13.98s | Test Loss -1.178
+Fund new best model, dict saved
+Train Summary | End of Epoch 6 | Time 725.93s | Train Loss -1.491
+Valid Summary | End of Epoch 6 | Time 39.37s | Valid Loss -1.258
+Test Summary | End of Epoch 6 | Time 14.30s | Test Loss -1.258
+Fund new best model, dict saved
+Train Summary | End of Epoch 7 | Time 746.14s | Train Loss -1.560
+Valid Summary | End of Epoch 7 | Time 41.31s | Valid Loss -1.275
+Test Summary | End of Epoch 7 | Time 14.51s | Test Loss -1.275
+Fund new best model, dict saved
+Train Summary | End of Epoch 8 | Time 743.28s | Train Loss -1.566
+Valid Summary | End of Epoch 8 | Time 43.07s | Valid Loss -1.267
+Test Summary | End of Epoch 8 | Time 14.99s | Test Loss -1.267
+Train Summary | End of Epoch 9 | Time 748.24s | Train Loss -1.517
+Valid Summary | End of Epoch 9 | Time 44.12s | Valid Loss -1.209
+Test Summary | End of Epoch 9 | Time 15.13s | Test Loss -1.209
+Train Summary | End of Epoch 10 | Time 757.55s | Train Loss -1.453
+Valid Summary | End of Epoch 10 | Time 44.53s | Valid Loss -1.245
+Test Summary | End of Epoch 10 | Time 15.24s | Test Loss -1.245
+Train Summary | End of Epoch 11 | Time 751.76s | Train Loss -1.381
+Valid Summary | End of Epoch 11 | Time 45.03s | Valid Loss -0.555
+Test Summary | End of Epoch 11 | Time 15.45s | Test Loss -0.555
+Train Summary | End of Epoch 12 | Time 758.94s | Train Loss -1.248
+Valid Summary | End of Epoch 12 | Time 45.47s | Valid Loss -0.771
+Test Summary | End of Epoch 12 | Time 15.37s | Test Loss -0.771
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000500
+Train Summary | End of Epoch 13 | Time 750.60s | Train Loss -1.963
+Valid Summary | End of Epoch 13 | Time 43.25s | Valid Loss -1.671
+Test Summary | End of Epoch 13 | Time 14.70s | Test Loss -1.671
+Fund new best model, dict saved
+Train Summary | End of Epoch 14 | Time 750.36s | Train Loss -2.132
+Valid Summary | End of Epoch 14 | Time 43.17s | Valid Loss -1.792
+Test Summary | End of Epoch 14 | Time 14.75s | Test Loss -1.792
+Fund new best model, dict saved
+Train Summary | End of Epoch 15 | Time 754.03s | Train Loss -2.248
+Valid Summary | End of Epoch 15 | Time 44.21s | Valid Loss -1.936
+Test Summary | End of Epoch 15 | Time 14.65s | Test Loss -1.936
+Fund new best model, dict saved
+Train Summary | End of Epoch 16 | Time 749.64s | Train Loss -2.343
+Valid Summary | End of Epoch 16 | Time 42.93s | Valid Loss -1.953
+Test Summary | End of Epoch 16 | Time 14.59s | Test Loss -1.953
+Fund new best model, dict saved
+Train Summary | End of Epoch 17 | Time 753.82s | Train Loss -2.431
+Valid Summary | End of Epoch 17 | Time 45.98s | Valid Loss -2.024
+Test Summary | End of Epoch 17 | Time 14.57s | Test Loss -2.024
+Fund new best model, dict saved
+Train Summary | End of Epoch 18 | Time 747.06s | Train Loss -2.527
+Valid Summary | End of Epoch 18 | Time 42.12s | Valid Loss -2.166
+Test Summary | End of Epoch 18 | Time 14.69s | Test Loss -2.166
+Fund new best model, dict saved
+Train Summary | End of Epoch 19 | Time 744.92s | Train Loss -2.622
+Valid Summary | End of Epoch 19 | Time 43.09s | Valid Loss -2.192
+Test Summary | End of Epoch 19 | Time 14.56s | Test Loss -2.192
+Fund new best model, dict saved
+Train Summary | End of Epoch 20 | Time 753.59s | Train Loss -2.714
+Valid Summary | End of Epoch 20 | Time 42.05s | Valid Loss -2.337
+Test Summary | End of Epoch 20 | Time 14.57s | Test Loss -2.337
+Fund new best model, dict saved
+Train Summary | End of Epoch 21 | Time 744.78s | Train Loss -2.809
+Valid Summary | End of Epoch 21 | Time 42.55s | Valid Loss -2.343
+Test Summary | End of Epoch 21 | Time 14.63s | Test Loss -2.343
+Fund new best model, dict saved
+Train Summary | End of Epoch 22 | Time 746.46s | Train Loss -2.913
+Valid Summary | End of Epoch 22 | Time 45.56s | Valid Loss -2.460
+Test Summary | End of Epoch 22 | Time 14.34s | Test Loss -2.460
+Fund new best model, dict saved
+Train Summary | End of Epoch 23 | Time 741.42s | Train Loss -3.008
+Valid Summary | End of Epoch 23 | Time 42.06s | Valid Loss -2.517
+Test Summary | End of Epoch 23 | Time 14.52s | Test Loss -2.517
+Fund new best model, dict saved
+Train Summary | End of Epoch 24 | Time 744.26s | Train Loss -3.111
+Valid Summary | End of Epoch 24 | Time 41.47s | Valid Loss -2.586
+Test Summary | End of Epoch 24 | Time 14.40s | Test Loss -2.586
+Fund new best model, dict saved
+Train Summary | End of Epoch 25 | Time 748.69s | Train Loss -3.215
+Valid Summary | End of Epoch 25 | Time 42.03s | Valid Loss -2.658
+Test Summary | End of Epoch 25 | Time 14.55s | Test Loss -2.658
+Fund new best model, dict saved
+Train Summary | End of Epoch 26 | Time 741.80s | Train Loss -3.316
+Valid Summary | End of Epoch 26 | Time 40.57s | Valid Loss -2.727
+Test Summary | End of Epoch 26 | Time 14.33s | Test Loss -2.727
+Fund new best model, dict saved
+Train Summary | End of Epoch 27 | Time 741.77s | Train Loss -3.413
+Valid Summary | End of Epoch 27 | Time 46.25s | Valid Loss -2.826
+Test Summary | End of Epoch 27 | Time 14.45s | Test Loss -2.826
+Fund new best model, dict saved
+Train Summary | End of Epoch 28 | Time 740.40s | Train Loss -3.512
+Valid Summary | End of Epoch 28 | Time 40.01s | Valid Loss -2.883
+Test Summary | End of Epoch 28 | Time 14.39s | Test Loss -2.883
+Fund new best model, dict saved
+Train Summary | End of Epoch 29 | Time 738.03s | Train Loss -3.609
+Valid Summary | End of Epoch 29 | Time 41.25s | Valid Loss -2.946
+Test Summary | End of Epoch 29 | Time 14.37s | Test Loss -2.946
+Fund new best model, dict saved
+Train Summary | End of Epoch 30 | Time 745.93s | Train Loss -3.708
+Valid Summary | End of Epoch 30 | Time 40.26s | Valid Loss -3.022
+Test Summary | End of Epoch 30 | Time 14.25s | Test Loss -3.022
+Fund new best model, dict saved
+Train Summary | End of Epoch 31 | Time 735.98s | Train Loss -3.806
+Valid Summary | End of Epoch 31 | Time 40.92s | Valid Loss -3.056
+Test Summary | End of Epoch 31 | Time 14.38s | Test Loss -3.056
+Fund new best model, dict saved
+Train Summary | End of Epoch 32 | Time 736.88s | Train Loss -3.900
+Valid Summary | End of Epoch 32 | Time 45.03s | Valid Loss -3.079
+Test Summary | End of Epoch 32 | Time 15.06s | Test Loss -3.079
+Fund new best model, dict saved
+Train Summary | End of Epoch 33 | Time 736.33s | Train Loss -3.993
+Valid Summary | End of Epoch 33 | Time 40.11s | Valid Loss -3.202
+Test Summary | End of Epoch 33 | Time 14.36s | Test Loss -3.202
+Fund new best model, dict saved
+Train Summary | End of Epoch 34 | Time 736.97s | Train Loss -4.086
+Valid Summary | End of Epoch 34 | Time 38.81s | Valid Loss -3.190
+Test Summary | End of Epoch 34 | Time 14.34s | Test Loss -3.190
+Train Summary | End of Epoch 35 | Time 739.99s | Train Loss -4.170
+Valid Summary | End of Epoch 35 | Time 40.32s | Valid Loss -3.270
+Test Summary | End of Epoch 35 | Time 14.23s | Test Loss -3.270
+Fund new best model, dict saved
+Train Summary | End of Epoch 36 | Time 737.00s | Train Loss -4.256
+Valid Summary | End of Epoch 36 | Time 38.84s | Valid Loss -3.336
+Test Summary | End of Epoch 36 | Time 14.31s | Test Loss -3.336
+Fund new best model, dict saved
+Train Summary | End of Epoch 37 | Time 732.92s | Train Loss -4.340
+Valid Summary | End of Epoch 37 | Time 44.71s | Valid Loss -3.257
+Test Summary | End of Epoch 37 | Time 15.17s | Test Loss -3.257
+Train Summary | End of Epoch 38 | Time 734.88s | Train Loss -4.418
+Valid Summary | End of Epoch 38 | Time 38.51s | Valid Loss -3.363
+Test Summary | End of Epoch 38 | Time 14.23s | Test Loss -3.363
+Fund new best model, dict saved
+Train Summary | End of Epoch 39 | Time 731.06s | Train Loss -4.498
+Valid Summary | End of Epoch 39 | Time 39.45s | Valid Loss -3.458
+Test Summary | End of Epoch 39 | Time 14.19s | Test Loss -3.458
+Fund new best model, dict saved
+Train Summary | End of Epoch 40 | Time 740.66s | Train Loss -4.574
+Valid Summary | End of Epoch 40 | Time 38.38s | Valid Loss -3.399
+Test Summary | End of Epoch 40 | Time 14.20s | Test Loss -3.399
+Train Summary | End of Epoch 41 | Time 728.36s | Train Loss -4.654
+Valid Summary | End of Epoch 41 | Time 38.44s | Valid Loss -3.500
+Test Summary | End of Epoch 41 | Time 14.16s | Test Loss -3.500
+Fund new best model, dict saved
+Train Summary | End of Epoch 42 | Time 731.56s | Train Loss -4.722
+Valid Summary | End of Epoch 42 | Time 40.71s | Valid Loss -3.614
+Test Summary | End of Epoch 42 | Time 15.21s | Test Loss -3.614
+Fund new best model, dict saved
+Train Summary | End of Epoch 43 | Time 729.22s | Train Loss -4.798
+Valid Summary | End of Epoch 43 | Time 38.44s | Valid Loss -3.693
+Test Summary | End of Epoch 43 | Time 14.09s | Test Loss -3.693
+Fund new best model, dict saved
+Train Summary | End of Epoch 44 | Time 730.00s | Train Loss -4.879
+Valid Summary | End of Epoch 44 | Time 37.59s | Valid Loss -3.678
+Test Summary | End of Epoch 44 | Time 14.19s | Test Loss -3.678
+Train Summary | End of Epoch 45 | Time 734.15s | Train Loss -4.964
+Valid Summary | End of Epoch 45 | Time 38.64s | Valid Loss -3.800
+Test Summary | End of Epoch 45 | Time 14.08s | Test Loss -3.800
+Fund new best model, dict saved
+Train Summary | End of Epoch 46 | Time 729.50s | Train Loss -5.072
+Valid Summary | End of Epoch 46 | Time 37.15s | Valid Loss -3.734
+Test Summary | End of Epoch 46 | Time 14.16s | Test Loss -3.734
+Train Summary | End of Epoch 47 | Time 725.78s | Train Loss -5.185
+Valid Summary | End of Epoch 47 | Time 37.57s | Valid Loss -3.958
+Test Summary | End of Epoch 47 | Time 15.28s | Test Loss -3.958
+Fund new best model, dict saved
+Train Summary | End of Epoch 48 | Time 731.21s | Train Loss -5.281
+Valid Summary | End of Epoch 48 | Time 36.71s | Valid Loss -4.040
+Test Summary | End of Epoch 48 | Time 14.12s | Test Loss -4.040
+Fund new best model, dict saved
+Train Summary | End of Epoch 49 | Time 725.42s | Train Loss -5.375
+Valid Summary | End of Epoch 49 | Time 36.82s | Valid Loss -4.125
+Test Summary | End of Epoch 49 | Time 14.23s | Test Loss -4.125
+Fund new best model, dict saved
+Train Summary | End of Epoch 50 | Time 734.09s | Train Loss -5.473
+Valid Summary | End of Epoch 50 | Time 37.13s | Valid Loss -4.210
+Test Summary | End of Epoch 50 | Time 14.18s | Test Loss -4.210
+Fund new best model, dict saved
+Train Summary | End of Epoch 51 | Time 725.50s | Train Loss -5.559
+Valid Summary | End of Epoch 51 | Time 36.97s | Valid Loss -4.359
+Test Summary | End of Epoch 51 | Time 14.22s | Test Loss -4.359
+Fund new best model, dict saved
+Train Summary | End of Epoch 52 | Time 726.31s | Train Loss -5.645
+Valid Summary | End of Epoch 52 | Time 36.74s | Valid Loss -4.412
+Test Summary | End of Epoch 52 | Time 14.16s | Test Loss -4.412
+Fund new best model, dict saved
+Train Summary | End of Epoch 53 | Time 730.08s | Train Loss -5.729
+Valid Summary | End of Epoch 53 | Time 36.28s | Valid Loss -4.457
+Test Summary | End of Epoch 53 | Time 14.22s | Test Loss -4.457
+Fund new best model, dict saved
+Train Summary | End of Epoch 54 | Time 724.64s | Train Loss -5.815
+Valid Summary | End of Epoch 54 | Time 36.31s | Valid Loss -4.554
+Test Summary | End of Epoch 54 | Time 14.27s | Test Loss -4.554
+Fund new best model, dict saved
+Train Summary | End of Epoch 55 | Time 725.54s | Train Loss -5.892
+Valid Summary | End of Epoch 55 | Time 39.90s | Valid Loss -4.606
+Test Summary | End of Epoch 55 | Time 14.18s | Test Loss -4.606
+Fund new best model, dict saved
+Train Summary | End of Epoch 56 | Time 726.29s | Train Loss -5.968
+Valid Summary | End of Epoch 56 | Time 36.75s | Valid Loss -4.640
+Test Summary | End of Epoch 56 | Time 14.24s | Test Loss -4.640
+Fund new best model, dict saved
+Train Summary | End of Epoch 57 | Time 722.47s | Train Loss -6.043
+Valid Summary | End of Epoch 57 | Time 36.27s | Valid Loss -4.693
+Test Summary | End of Epoch 57 | Time 14.13s | Test Loss -4.693
+Fund new best model, dict saved
+Train Summary | End of Epoch 58 | Time 730.58s | Train Loss -6.108
+Valid Summary | End of Epoch 58 | Time 36.70s | Valid Loss -4.720
+Test Summary | End of Epoch 58 | Time 14.32s | Test Loss -4.720
+Fund new best model, dict saved
+Train Summary | End of Epoch 59 | Time 721.53s | Train Loss -6.179
+Valid Summary | End of Epoch 59 | Time 36.29s | Valid Loss -4.858
+Test Summary | End of Epoch 59 | Time 14.10s | Test Loss -4.858
+Fund new best model, dict saved
+Train Summary | End of Epoch 60 | Time 723.80s | Train Loss -6.245
+Valid Summary | End of Epoch 60 | Time 38.75s | Valid Loss -4.819
+Test Summary | End of Epoch 60 | Time 15.27s | Test Loss -4.819
+Train Summary | End of Epoch 61 | Time 724.00s | Train Loss -6.307
+Valid Summary | End of Epoch 61 | Time 35.99s | Valid Loss -4.942
+Test Summary | End of Epoch 61 | Time 14.20s | Test Loss -4.942
+Fund new best model, dict saved
+Train Summary | End of Epoch 62 | Time 723.08s | Train Loss -6.362
+Valid Summary | End of Epoch 62 | Time 36.11s | Valid Loss -4.982
+Test Summary | End of Epoch 62 | Time 14.20s | Test Loss -4.982
+Fund new best model, dict saved
+Train Summary | End of Epoch 63 | Time 729.04s | Train Loss -6.421
+Valid Summary | End of Epoch 63 | Time 35.76s | Valid Loss -4.887
+Test Summary | End of Epoch 63 | Time 14.19s | Test Loss -4.887
+Train Summary | End of Epoch 64 | Time 721.26s | Train Loss -6.473
+Valid Summary | End of Epoch 64 | Time 36.40s | Valid Loss -4.988
+Test Summary | End of Epoch 64 | Time 14.05s | Test Loss -4.988
+Fund new best model, dict saved
+Train Summary | End of Epoch 65 | Time 719.98s | Train Loss -6.526
+Valid Summary | End of Epoch 65 | Time 35.59s | Valid Loss -5.094
+Test Summary | End of Epoch 65 | Time 14.18s | Test Loss -5.094
+Fund new best model, dict saved
+Train Summary | End of Epoch 66 | Time 728.57s | Train Loss -6.580
+Valid Summary | End of Epoch 66 | Time 35.75s | Valid Loss -5.008
+Test Summary | End of Epoch 66 | Time 14.19s | Test Loss -5.008
+Train Summary | End of Epoch 67 | Time 696.84s | Train Loss -6.626
+Valid Summary | End of Epoch 67 | Time 32.85s | Valid Loss -5.174
+Test Summary | End of Epoch 67 | Time 13.50s | Test Loss -5.174
+Fund new best model, dict saved
+Train Summary | End of Epoch 68 | Time 661.44s | Train Loss -6.671
+Valid Summary | End of Epoch 68 | Time 32.46s | Valid Loss -5.085
+Test Summary | End of Epoch 68 | Time 13.51s | Test Loss -5.085
+Train Summary | End of Epoch 69 | Time 659.72s | Train Loss -6.716
+Valid Summary | End of Epoch 69 | Time 32.57s | Valid Loss -5.186
+Test Summary | End of Epoch 69 | Time 13.48s | Test Loss -5.186
+Fund new best model, dict saved
+Train Summary | End of Epoch 70 | Time 661.29s | Train Loss -6.760
+Valid Summary | End of Epoch 70 | Time 32.41s | Valid Loss -5.172
+Test Summary | End of Epoch 70 | Time 13.53s | Test Loss -5.172
+Train Summary | End of Epoch 71 | Time 660.51s | Train Loss -6.800
+Valid Summary | End of Epoch 71 | Time 32.54s | Valid Loss -5.214
+Test Summary | End of Epoch 71 | Time 13.52s | Test Loss -5.214
+Fund new best model, dict saved
+Train Summary | End of Epoch 72 | Time 660.09s | Train Loss -6.841
+Valid Summary | End of Epoch 72 | Time 32.48s | Valid Loss -5.223
+Test Summary | End of Epoch 72 | Time 13.52s | Test Loss -5.223
+Fund new best model, dict saved
+Train Summary | End of Epoch 73 | Time 659.66s | Train Loss -6.881
+Valid Summary | End of Epoch 73 | Time 32.48s | Valid Loss -5.157
+Test Summary | End of Epoch 73 | Time 13.48s | Test Loss -5.157
+Train Summary | End of Epoch 74 | Time 659.99s | Train Loss -6.916
+Valid Summary | End of Epoch 74 | Time 32.43s | Valid Loss -5.238
+Test Summary | End of Epoch 74 | Time 13.51s | Test Loss -5.238
+Fund new best model, dict saved
+Train Summary | End of Epoch 75 | Time 659.97s | Train Loss -6.958
+Valid Summary | End of Epoch 75 | Time 32.54s | Valid Loss -5.285
+Test Summary | End of Epoch 75 | Time 13.53s | Test Loss -5.285
+Fund new best model, dict saved
+Train Summary | End of Epoch 76 | Time 659.84s | Train Loss -6.993
+Valid Summary | End of Epoch 76 | Time 32.43s | Valid Loss -5.267
+Test Summary | End of Epoch 76 | Time 13.48s | Test Loss -5.267
+Train Summary | End of Epoch 77 | Time 659.72s | Train Loss -7.029
+Valid Summary | End of Epoch 77 | Time 32.39s | Valid Loss -5.372
+Test Summary | End of Epoch 77 | Time 13.55s | Test Loss -5.372
+Fund new best model, dict saved
+Train Summary | End of Epoch 78 | Time 660.30s | Train Loss -7.060
+Valid Summary | End of Epoch 78 | Time 32.47s | Valid Loss -5.331
+Test Summary | End of Epoch 78 | Time 13.51s | Test Loss -5.331
+Train Summary | End of Epoch 79 | Time 658.86s | Train Loss -7.095
+Valid Summary | End of Epoch 79 | Time 32.43s | Valid Loss -5.444
+Test Summary | End of Epoch 79 | Time 13.50s | Test Loss -5.444
+Fund new best model, dict saved
+Train Summary | End of Epoch 80 | Time 660.69s | Train Loss -7.121
+Valid Summary | End of Epoch 80 | Time 32.68s | Valid Loss -5.283
+Test Summary | End of Epoch 80 | Time 13.51s | Test Loss -5.283
+Train Summary | End of Epoch 81 | Time 663.11s | Train Loss -7.156
+Valid Summary | End of Epoch 81 | Time 32.67s | Valid Loss -5.456
+Test Summary | End of Epoch 81 | Time 13.58s | Test Loss -5.456
+Fund new best model, dict saved
+Train Summary | End of Epoch 82 | Time 662.48s | Train Loss -7.190
+Valid Summary | End of Epoch 82 | Time 32.71s | Valid Loss -5.377
+Test Summary | End of Epoch 82 | Time 13.61s | Test Loss -5.377
+Train Summary | End of Epoch 83 | Time 662.77s | Train Loss -7.214
+Valid Summary | End of Epoch 83 | Time 32.69s | Valid Loss -5.452
+Test Summary | End of Epoch 83 | Time 13.56s | Test Loss -5.452
+Train Summary | End of Epoch 84 | Time 663.27s | Train Loss -7.245
+Valid Summary | End of Epoch 84 | Time 32.61s | Valid Loss -5.478
+Test Summary | End of Epoch 84 | Time 13.62s | Test Loss -5.478
+Fund new best model, dict saved
+Train Summary | End of Epoch 85 | Time 662.36s | Train Loss -7.276
+Valid Summary | End of Epoch 85 | Time 32.70s | Valid Loss -5.561
+Test Summary | End of Epoch 85 | Time 13.63s | Test Loss -5.561
+Fund new best model, dict saved
+Train Summary | End of Epoch 86 | Time 662.53s | Train Loss -7.305
+Valid Summary | End of Epoch 86 | Time 32.65s | Valid Loss -5.490
+Test Summary | End of Epoch 86 | Time 13.56s | Test Loss -5.490
+Train Summary | End of Epoch 87 | Time 662.51s | Train Loss -7.325
+Valid Summary | End of Epoch 87 | Time 32.50s | Valid Loss -5.381
+Test Summary | End of Epoch 87 | Time 13.56s | Test Loss -5.381
+Train Summary | End of Epoch 88 | Time 663.25s | Train Loss -7.356
+Valid Summary | End of Epoch 88 | Time 32.53s | Valid Loss -5.539
+Test Summary | End of Epoch 88 | Time 13.55s | Test Loss -5.539
+Train Summary | End of Epoch 89 | Time 663.66s | Train Loss -7.379
+Valid Summary | End of Epoch 89 | Time 32.62s | Valid Loss -5.483
+Test Summary | End of Epoch 89 | Time 13.60s | Test Loss -5.483
+Train Summary | End of Epoch 90 | Time 663.16s | Train Loss -7.407
+Valid Summary | End of Epoch 90 | Time 32.66s | Valid Loss -5.423
+Test Summary | End of Epoch 90 | Time 13.58s | Test Loss -5.423
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000250
+Train Summary | End of Epoch 91 | Time 662.88s | Train Loss -7.530
+Valid Summary | End of Epoch 91 | Time 32.67s | Valid Loss -5.568
+Test Summary | End of Epoch 91 | Time 13.52s | Test Loss -5.568
+Fund new best model, dict saved
+Train Summary | End of Epoch 92 | Time 662.88s | Train Loss -7.621
+Valid Summary | End of Epoch 92 | Time 32.59s | Valid Loss -5.606
+Test Summary | End of Epoch 92 | Time 13.57s | Test Loss -5.606
+Fund new best model, dict saved
+Train Summary | End of Epoch 93 | Time 663.13s | Train Loss -7.674
+Valid Summary | End of Epoch 93 | Time 32.67s | Valid Loss -5.602
+Test Summary | End of Epoch 93 | Time 13.59s | Test Loss -5.602
+Train Summary | End of Epoch 94 | Time 662.66s | Train Loss -7.717
+Valid Summary | End of Epoch 94 | Time 32.70s | Valid Loss -5.503
+Test Summary | End of Epoch 94 | Time 13.53s | Test Loss -5.503
+Train Summary | End of Epoch 95 | Time 662.53s | Train Loss -7.754
+Valid Summary | End of Epoch 95 | Time 32.52s | Valid Loss -5.597
+Test Summary | End of Epoch 95 | Time 13.56s | Test Loss -5.597
+Train Summary | End of Epoch 96 | Time 663.48s | Train Loss -7.787
+Valid Summary | End of Epoch 96 | Time 32.57s | Valid Loss -5.523
+Test Summary | End of Epoch 96 | Time 13.56s | Test Loss -5.523
+Train Summary | End of Epoch 97 | Time 662.24s | Train Loss -7.820
+Valid Summary | End of Epoch 97 | Time 32.56s | Valid Loss -5.584
+Test Summary | End of Epoch 97 | Time 13.56s | Test Loss -5.584
+reload weights and optimizer from last best checkpoint
+Learning rate adjusted to: 0.000125
+Train Summary | End of Epoch 98 | Time 662.76s | Train Loss -7.760
+Valid Summary | End of Epoch 98 | Time 32.61s | Valid Loss -5.594
+Test Summary | End of Epoch 98 | Time 13.60s | Test Loss -5.594
+Train Summary | End of Epoch 99 | Time 663.16s | Train Loss -7.814
+Valid Summary | End of Epoch 99 | Time 32.72s | Valid Loss -5.579
+Test Summary | End of Epoch 99 | Time 13.60s | Test Loss -5.579
+Train Summary | End of Epoch 100 | Time 663.19s | Train Loss -7.853
+Valid Summary | End of Epoch 100 | Time 32.68s | Valid Loss -5.535
+Test Summary | End of Epoch 100 | Time 13.61s | Test Loss -5.535
+Train Summary | End of Epoch 101 | Time 662.57s | Train Loss -7.885
+Valid Summary | End of Epoch 101 | Time 32.73s | Valid Loss -5.500
+Test Summary | End of Epoch 101 | Time 13.53s | Test Loss -5.500
+Train Summary | End of Epoch 102 | Time 662.99s | Train Loss -7.915
+Valid Summary | End of Epoch 102 | Time 32.59s | Valid Loss -5.485
+Test Summary | End of Epoch 102 | Time 13.60s | Test Loss -5.485
+No imporvement for 10 epochs, early stopping.
+Start evaluation
+Avg SISNR:i tensor([10.5698], device='cuda:0')
+Avg SNRi: 11.085818972940272
+Avg PESQi: 0.3742397728761037
+Avg STOIi: 0.2910296654460408

checkpoints/log_LRS2_lip_dprnn_3spk/tensorboard/events.out.tfevents.1727340482.nls-dev-servers011167134195.na63.49263.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2062c9f886fe54797718bf56b24596d878a6ad5d4a4beb39f67fd590fe9ca120
+size 15036