# python3 -m espnet2.bin.asr_train --use_preprocessor true --bpemodel none --token_type char --token_list data/en_token_list/char/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/cv_mix_clean_reverb_max_16k/wav.scp,speech,kaldi_ark --valid_shape_file exp/asr_stats_raw_en_char/valid/speech_shape --resume true --init_param /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth --ignore_init_mismatch false --fold_length 80000 --output_dir exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only --config conf/tuning/train_sot_asr_conformer.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/asr_stats_raw_en_char/train/feats_stats.npz --train_data_path_and_name_and_type dump/raw/tr_mix_clean_reverb_max_16k_sp/wav.scp,speech,kaldi_ark --train_shape_file exp/asr_stats_raw_en_char/train/speech_shape --fold_length 150 --train_data_path_and_name_and_type dump/raw/tr_mix_clean_reverb_max_16k_sp/text,text,text --train_shape_file exp/asr_stats_raw_en_char/train/text_shape.char --valid_data_path_and_name_and_type dump/raw/cv_mix_clean_reverb_max_16k/text,text,text --valid_shape_file exp/asr_stats_raw_en_char/valid/text_shape.char --ngpu 2 --multiprocessing_distributed True 
# Started at Tue Feb 20 15:47:49 CST 2024
#
/star-home/jinzengrui/lib/miniconda3/envs/espnet/bin/python3 /star-home/jinzengrui/lib/miniconda3/envs/espnet/lib/python3.9/site-packages/espnet-202308-py3.9.egg/espnet2/bin/asr_train.py --use_preprocessor true --bpemodel none --token_type char --token_list data/en_token_list/char/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/cv_mix_clean_reverb_max_16k/wav.scp,speech,kaldi_ark --valid_shape_file exp/asr_stats_raw_en_char/valid/speech_shape --resume true --init_param /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth --ignore_init_mismatch false --fold_length 80000 --output_dir exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only --config conf/tuning/train_sot_asr_conformer.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/asr_stats_raw_en_char/train/feats_stats.npz --train_data_path_and_name_and_type dump/raw/tr_mix_clean_reverb_max_16k_sp/wav.scp,speech,kaldi_ark --train_shape_file exp/asr_stats_raw_en_char/train/speech_shape --fold_length 150 --train_data_path_and_name_and_type dump/raw/tr_mix_clean_reverb_max_16k_sp/text,text,text --train_shape_file exp/asr_stats_raw_en_char/train/text_shape.char --valid_data_path_and_name_and_type dump/raw/cv_mix_clean_reverb_max_16k/text,text,text --valid_shape_file exp/asr_stats_raw_en_char/valid/text_shape.char --ngpu 2 --multiprocessing_distributed True
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:15,999 (distributed_c10d:228) INFO: Added key: store_based_barrier_key:1 to store for rank: 0
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:15,999 (distributed_c10d:262) INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 2 nodes.
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:16,025 (asr:490) INFO: Vocabulary size: 32
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:21,367 (abs_task:1229) INFO: pytorch.version=1.12.1+cu116, cuda.available=True, cudnn.version=8302, cudnn.benchmark=False, cudnn.deterministic=True
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:21,377 (abs_task:1230) INFO: Model structure:
ESPnetASRModel(
  (frontend): DefaultFrontend(
    (stft): Stft(n_fft=512, win_length=512, hop_length=128, center=True, normalized=False, onesided=True)
    (frontend): Frontend()
    (logmel): LogMel(sr=16000, n_fft=512, n_mels=80, fmin=0, fmax=8000.0, htk=False)
  )
  (normalize): GlobalMVN(stats_file=exp/asr_stats_raw_en_char/train/feats_stats.npz, norm_means=True, norm_vars=True)
  (encoder): ConformerEncoder(
    (embed): Conv2dSubsampling(
      (conv): Sequential(
        (0): Conv2d(1, 256, kernel_size=(3, 3), stride=(2, 2))
        (1): ReLU()
        (2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2))
        (3): ReLU()
      )
      (out): Sequential(
        (0): Linear(in_features=4864, out_features=256, bias=True)
        (1): RelPositionalEncoding(
          (dropout): Dropout(p=0.1, inplace=False)
        )
      )
    )
    (encoders): MultiSequential(
      (0): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (1): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (2): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (3): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (4): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (5): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (6): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (7): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (8): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (9): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (10): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (11): EncoderLayer(
        (self_attn): RelPositionMultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (linear_pos): Linear(in_features=256, out_features=256, bias=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (feed_forward_macaron): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): Swish()
        )
        (conv_module): ConvolutionModule(
          (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,))
          (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256)
          (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
          (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,))
          (activation): Swish()
        )
        (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
    )
    (after_norm): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
  )
  (decoder): TransformerDecoder(
    (embed): Sequential(
      (0): Embedding(32, 256)
      (1): PositionalEncoding(
        (dropout): Dropout(p=0.1, inplace=False)
      )
    )
    (after_norm): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
    (output_layer): Linear(in_features=256, out_features=32, bias=True)
    (decoders): MultiSequential(
      (0): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (1): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (2): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (3): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (4): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
      (5): DecoderLayer(
        (self_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (src_attn): MultiHeadedAttention(
          (linear_q): Linear(in_features=256, out_features=256, bias=True)
          (linear_k): Linear(in_features=256, out_features=256, bias=True)
          (linear_v): Linear(in_features=256, out_features=256, bias=True)
          (linear_out): Linear(in_features=256, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
        )
        (feed_forward): PositionwiseFeedForward(
          (w_1): Linear(in_features=256, out_features=2048, bias=True)
          (w_2): Linear(in_features=2048, out_features=256, bias=True)
          (dropout): Dropout(p=0.1, inplace=False)
          (activation): ReLU()
        )
        (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True)
        (dropout): Dropout(p=0.1, inplace=False)
      )
    )
  )
  (criterion_att): LabelSmoothingLoss(
    (criterion): KLDivLoss()
  )
)

Model summary:
    Class Name: ESPnetASRModel
    Total Number of model parameters: 43.00 M
    Number of trainable parameters: 43.00 M (100.0%)
    Size: 172.01 MB
    Type: torch.float32
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:21,377 (abs_task:1233) INFO: Optimizer:
Adam (
Parameter Group 0
    amsgrad: False
    betas: (0.9, 0.999)
    capturable: False
    eps: 1e-08
    foreach: None
    initial_lr: 0.002
    lr: 1e-07
    maximize: False
    weight_decay: 1e-06
)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:21,378 (abs_task:1234) INFO: Scheduler: WarmupLR(warmup_steps=20000)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:21,385 (abs_task:1243) INFO: Saving the configuration in exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/config.yaml
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:21,399 (abs_task:1304) INFO: Loading pretrained params from /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:22,989 (asr:461) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4')
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,001 (abs_task:1614) INFO: [train] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/tr_mix_clean_reverb_max_16k_sp/wav.scp", "type": "kaldi_ark"}
  text: {"path": "dump/raw/tr_mix_clean_reverb_max_16k_sp/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.CommonPreprocessor_multi object at 0x7f0da67a87f0>)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,002 (abs_task:1615) INFO: [train] Batch sampler: NumElementsBatchSampler(N-batch=1037, batch_bins=10000000, sort_in_batch=descending, sort_batch=descending)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,002 (abs_task:1616) INFO: [train] mini-batch sizes summary: N-batch=1037, mean=57.9, min=6, max=125
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,048 (asr:461) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4')
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,096 (abs_task:1614) INFO: [valid] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/cv_mix_clean_reverb_max_16k/wav.scp", "type": "kaldi_ark"}
  text: {"path": "dump/raw/cv_mix_clean_reverb_max_16k/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.CommonPreprocessor_multi object at 0x7f0da5e707f0>)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,096 (abs_task:1615) INFO: [valid] Batch sampler: NumElementsBatchSampler(N-batch=88, batch_bins=10000000, sort_in_batch=descending, sort_batch=descending)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,097 (abs_task:1616) INFO: [valid] mini-batch sizes summary: N-batch=88, mean=56.8, min=5, max=102
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,109 (asr:461) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4')
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,139 (abs_task:1614) INFO: [plot_att] dataset:
ESPnetDataset(
  speech: {"path": "dump/raw/cv_mix_clean_reverb_max_16k/wav.scp", "type": "kaldi_ark"}
  text: {"path": "dump/raw/cv_mix_clean_reverb_max_16k/text", "type": "text"}
  preprocess: <espnet2.train.preprocessor.CommonPreprocessor_multi object at 0x7f0da5e70d60>)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,139 (abs_task:1615) INFO: [plot_att] Batch sampler: UnsortedBatchSampler(N-batch=5000, batch_size=1, key_file=exp/asr_stats_raw_en_char/valid/speech_shape, 
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:24,139 (abs_task:1616) INFO: [plot_att] mini-batch sizes summary: N-batch=3, mean=1.0, min=1, max=1
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] NCCL INFO Bootstrap : Using eth0:10.177.6.147<0>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_0

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_1
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] NCCL INFO NET/IB : No device found.
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] NCCL INFO NET/Socket : Using [0]eth0:10.177.6.147<0>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] NCCL INFO Using network Socket
NCCL version 2.10.3+cuda11.6
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946847 [1] NCCL INFO Bootstrap : Using eth0:10.177.6.147<0>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946847 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946847 [1] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946847 [1] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_0

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946847 [1] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946847 [1] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_1
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946847 [1] NCCL INFO NET/IB : No device found.
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946847 [1] NCCL INFO NET/Socket : Using [0]eth0:10.177.6.147<0>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946847 [1] NCCL INFO Using network Socket
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO Channel 00/02 :    0   1
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946956 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] -1/-1/-1->1->0
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO Channel 01/02 :    0   1
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946956 [1] NCCL INFO Setting affinity for GPU 7 to ff,ffc0000f,fffc0000
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO Setting affinity for GPU 6 to ff,ffc0000f,fffc0000
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946956 [1] NCCL INFO Channel 00 : 1[b5000] -> 0[b4000] via P2P/IPC
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO Channel 00 : 0[b4000] -> 1[b5000] via P2P/IPC
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946956 [1] NCCL INFO Channel 01 : 1[b5000] -> 0[b4000] via P2P/IPC
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO Channel 01 : 0[b4000] -> 1[b5000] via P2P/IPC
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946956 [1] NCCL INFO Connected all rings
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO Connected all rings
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946956 [1] NCCL INFO Connected all trees
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO Connected all trees
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946956 [1] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 8/8/512
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946956 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 8/8/512
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946956 [1] NCCL INFO comm 0x7fccbc002f70 rank 1 nranks 2 cudaDev 1 busId b5000 - Init COMPLETE
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946955 [0] NCCL INFO comm 0x7f0d1c002f70 rank 0 nranks 2 cudaDev 0 busId b4000 - Init COMPLETE
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946846 [0] NCCL INFO Launch mode Parallel
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:48:28,205 (trainer:284) INFO: 1/60epoch started
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:51:08,021 (distributed:995) INFO: Reducer buckets have been rebuilt in this iteration.
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:51:40,341 (trainer:732) INFO: 1epoch:train:1-51batch: iter_time=0.014, forward_time=0.354, loss_att=393.693, acc=0.443, loss=393.693, backward_time=0.300, grad_norm=236.750, clip=100.000, loss_scale=1.000, optim_step_time=0.070, optim0_lr0=7.500e-07, train_time=15.853
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:52:12,947 (trainer:732) INFO: 1epoch:train:52-102batch: iter_time=3.248e-04, forward_time=0.202, loss_att=385.358, acc=0.444, loss=385.358, backward_time=0.281, grad_norm=228.659, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.000e-06, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:52:46,018 (trainer:732) INFO: 1epoch:train:103-153batch: iter_time=3.086e-04, forward_time=0.205, loss_att=367.925, acc=0.444, loss=367.925, backward_time=0.289, grad_norm=216.965, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.300e-06, train_time=2.597
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:53:18,701 (trainer:732) INFO: 1epoch:train:154-204batch: iter_time=3.004e-04, forward_time=0.202, loss_att=357.804, acc=0.448, loss=357.804, backward_time=0.280, grad_norm=198.505, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.600e-06, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:53:50,783 (trainer:732) INFO: 1epoch:train:205-255batch: iter_time=3.404e-04, forward_time=0.201, loss_att=345.146, acc=0.447, loss=345.146, backward_time=0.277, grad_norm=180.826, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=5.850e-06, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:54:23,428 (trainer:732) INFO: 1epoch:train:256-306batch: iter_time=3.048e-04, forward_time=0.202, loss_att=349.776, acc=0.460, loss=349.776, backward_time=0.281, grad_norm=180.818, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.100e-06, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:54:55,942 (trainer:732) INFO: 1epoch:train:307-357batch: iter_time=3.226e-04, forward_time=0.202, loss_att=333.992, acc=0.467, loss=333.992, backward_time=0.280, grad_norm=157.505, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.400e-06, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:55:28,304 (trainer:732) INFO: 1epoch:train:358-408batch: iter_time=3.041e-04, forward_time=0.200, loss_att=302.532, acc=0.470, loss=302.532, backward_time=0.279, grad_norm=128.849, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.700e-06, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:56:00,490 (trainer:732) INFO: 1epoch:train:409-459batch: iter_time=3.305e-04, forward_time=0.201, loss_att=288.926, acc=0.468, loss=288.926, backward_time=0.278, grad_norm=106.797, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.095e-05, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:56:32,918 (trainer:732) INFO: 1epoch:train:460-510batch: iter_time=3.419e-04, forward_time=0.201, loss_att=283.913, acc=0.478, loss=283.913, backward_time=0.279, grad_norm=88.984, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.220e-05, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:57:05,656 (trainer:732) INFO: 1epoch:train:511-561batch: iter_time=3.197e-04, forward_time=0.203, loss_att=276.417, acc=0.492, loss=276.417, backward_time=0.283, grad_norm=67.900, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.350e-05, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:57:38,056 (trainer:732) INFO: 1epoch:train:562-612batch: iter_time=3.291e-04, forward_time=0.201, loss_att=267.773, acc=0.499, loss=267.773, backward_time=0.277, grad_norm=56.014, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.480e-05, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:58:10,251 (trainer:732) INFO: 1epoch:train:613-663batch: iter_time=4.037e-04, forward_time=0.202, loss_att=257.866, acc=0.506, loss=257.866, backward_time=0.279, grad_norm=50.633, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.605e-05, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:58:43,387 (trainer:732) INFO: 1epoch:train:664-714batch: iter_time=3.234e-04, forward_time=0.202, loss_att=257.652, acc=0.521, loss=257.652, backward_time=0.282, grad_norm=48.409, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=1.730e-05, train_time=2.586
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:59:15,622 (trainer:732) INFO: 1epoch:train:715-765batch: iter_time=2.900e-04, forward_time=0.200, loss_att=244.904, acc=0.520, loss=244.904, backward_time=0.277, grad_norm=45.095, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.860e-05, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 15:59:48,441 (trainer:732) INFO: 1epoch:train:766-816batch: iter_time=3.126e-04, forward_time=0.202, loss_att=244.314, acc=0.531, loss=244.314, backward_time=0.281, grad_norm=45.606, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.990e-05, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:00:20,514 (trainer:732) INFO: 1epoch:train:817-867batch: iter_time=3.120e-04, forward_time=0.200, loss_att=243.171, acc=0.535, loss=243.171, backward_time=0.278, grad_norm=42.903, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.115e-05, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:00:52,644 (trainer:732) INFO: 1epoch:train:868-918batch: iter_time=3.135e-04, forward_time=0.200, loss_att=235.275, acc=0.534, loss=235.275, backward_time=0.277, grad_norm=41.711, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.240e-05, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:01:25,062 (trainer:732) INFO: 1epoch:train:919-969batch: iter_time=3.215e-04, forward_time=0.202, loss_att=237.631, acc=0.545, loss=237.631, backward_time=0.280, grad_norm=42.529, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.370e-05, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:01:57,580 (trainer:732) INFO: 1epoch:train:970-1020batch: iter_time=3.011e-04, forward_time=0.201, loss_att=233.713, acc=0.553, loss=233.713, backward_time=0.279, grad_norm=40.699, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.500e-05, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:10:38,177 (trainer:338) INFO: 1epoch results: [train] iter_time=9.823e-04, forward_time=0.209, loss_att=293.507, acc=0.492, loss=293.507, backward_time=0.281, grad_norm=108.960, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.310e-05, train_time=3.163, time=13 minutes and 41.38 seconds, total_count=1037, gpu_max_cached_mem_GB=30.428, [valid] loss_att=224.369, acc=0.578, cer=0.488, wer=0.851, loss=224.369, time=4 minutes and 47.71 seconds, total_count=88, gpu_max_cached_mem_GB=30.428, [att_plot] time=3 minutes and 40.85 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:10:42,445 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:10:42,446 (trainer:272) INFO: 2/60epoch started. Estimated time to finish: 21 hours, 52 minutes and 0.21 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:13:46,352 (trainer:732) INFO: 2epoch:train:1-51batch: iter_time=0.016, forward_time=0.203, loss_att=228.382, acc=0.555, loss=228.382, backward_time=0.278, grad_norm=40.858, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.665e-05, train_time=15.174
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:14:18,362 (trainer:732) INFO: 2epoch:train:52-102batch: iter_time=3.156e-04, forward_time=0.199, loss_att=225.181, acc=0.558, loss=225.181, backward_time=0.277, grad_norm=39.932, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.790e-05, train_time=2.521
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:14:50,803 (trainer:732) INFO: 2epoch:train:103-153batch: iter_time=3.574e-04, forward_time=0.202, loss_att=225.373, acc=0.565, loss=225.373, backward_time=0.278, grad_norm=38.277, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.920e-05, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:15:23,679 (trainer:732) INFO: 2epoch:train:154-204batch: iter_time=3.393e-04, forward_time=0.205, loss_att=228.849, acc=0.572, loss=228.849, backward_time=0.282, grad_norm=40.644, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.050e-05, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:15:55,800 (trainer:732) INFO: 2epoch:train:205-255batch: iter_time=3.456e-04, forward_time=0.200, loss_att=224.546, acc=0.574, loss=224.546, backward_time=0.277, grad_norm=40.234, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.175e-05, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:16:28,808 (trainer:732) INFO: 2epoch:train:256-306batch: iter_time=3.492e-04, forward_time=0.204, loss_att=219.041, acc=0.578, loss=219.041, backward_time=0.283, grad_norm=39.771, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=3.300e-05, train_time=2.578
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:17:01,561 (trainer:732) INFO: 2epoch:train:307-357batch: iter_time=3.709e-04, forward_time=0.203, loss_att=212.967, acc=0.578, loss=212.967, backward_time=0.281, grad_norm=40.052, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.430e-05, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:17:33,990 (trainer:732) INFO: 2epoch:train:358-408batch: iter_time=3.362e-04, forward_time=0.200, loss_att=213.598, acc=0.581, loss=213.598, backward_time=0.276, grad_norm=36.887, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=3.560e-05, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:18:06,615 (trainer:732) INFO: 2epoch:train:409-459batch: iter_time=3.998e-04, forward_time=0.204, loss_att=216.964, acc=0.589, loss=216.964, backward_time=0.282, grad_norm=38.138, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.685e-05, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:18:38,889 (trainer:732) INFO: 2epoch:train:460-510batch: iter_time=3.566e-04, forward_time=0.200, loss_att=209.084, acc=0.590, loss=209.084, backward_time=0.276, grad_norm=37.815, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.810e-05, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:19:11,495 (trainer:732) INFO: 2epoch:train:511-561batch: iter_time=3.656e-04, forward_time=0.202, loss_att=210.242, acc=0.593, loss=210.242, backward_time=0.279, grad_norm=38.372, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.940e-05, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:19:44,480 (trainer:732) INFO: 2epoch:train:562-612batch: iter_time=3.829e-04, forward_time=0.203, loss_att=212.481, acc=0.598, loss=212.481, backward_time=0.283, grad_norm=40.424, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.070e-05, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:20:16,564 (trainer:732) INFO: 2epoch:train:613-663batch: iter_time=5.242e-04, forward_time=0.201, loss_att=205.287, acc=0.595, loss=205.287, backward_time=0.277, grad_norm=36.394, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.195e-05, train_time=2.518
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:20:49,022 (trainer:732) INFO: 2epoch:train:664-714batch: iter_time=3.477e-04, forward_time=0.202, loss_att=206.831, acc=0.606, loss=206.831, backward_time=0.281, grad_norm=37.628, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.320e-05, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:21:21,740 (trainer:732) INFO: 2epoch:train:715-765batch: iter_time=3.316e-04, forward_time=0.202, loss_att=209.262, acc=0.611, loss=209.262, backward_time=0.283, grad_norm=38.969, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.450e-05, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:21:54,312 (trainer:732) INFO: 2epoch:train:766-816batch: iter_time=3.548e-04, forward_time=0.201, loss_att=200.279, acc=0.602, loss=200.279, backward_time=0.279, grad_norm=37.859, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.580e-05, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:22:26,513 (trainer:732) INFO: 2epoch:train:817-867batch: iter_time=3.793e-04, forward_time=0.201, loss_att=195.963, acc=0.603, loss=195.963, backward_time=0.278, grad_norm=37.513, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.705e-05, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:22:58,859 (trainer:732) INFO: 2epoch:train:868-918batch: iter_time=3.809e-04, forward_time=0.201, loss_att=198.451, acc=0.610, loss=198.451, backward_time=0.277, grad_norm=37.638, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.830e-05, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:23:31,138 (trainer:732) INFO: 2epoch:train:919-969batch: iter_time=4.004e-04, forward_time=0.201, loss_att=191.993, acc=0.610, loss=191.993, backward_time=0.276, grad_norm=35.262, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.960e-05, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:24:03,669 (trainer:732) INFO: 2epoch:train:970-1020batch: iter_time=3.362e-04, forward_time=0.202, loss_att=194.838, acc=0.614, loss=194.838, backward_time=0.278, grad_norm=35.346, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.090e-05, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:34:28,281 (trainer:338) INFO: 2epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=211.228, acc=0.589, loss=211.228, backward_time=0.279, grad_norm=38.332, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.900e-05, train_time=3.132, time=13 minutes and 33.22 seconds, total_count=2074, gpu_max_cached_mem_GB=30.428, [valid] loss_att=189.231, acc=0.638, cer=0.429, wer=0.791, loss=189.231, time=4 minutes and 48.97 seconds, total_count=176, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 23.65 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:34:32,043 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:34:32,045 (trainer:272) INFO: 3/60epoch started. Estimated time to finish: 22 hours, 15 minutes and 51.33 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:37:35,197 (trainer:732) INFO: 3epoch:train:1-51batch: iter_time=0.010, forward_time=0.204, loss_att=193.968, acc=0.619, loss=193.968, backward_time=0.280, grad_norm=35.924, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.255e-05, train_time=15.114
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:38:07,769 (trainer:732) INFO: 3epoch:train:52-102batch: iter_time=3.569e-04, forward_time=0.203, loss_att=193.691, acc=0.625, loss=193.691, backward_time=0.280, grad_norm=37.094, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.380e-05, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:38:40,659 (trainer:732) INFO: 3epoch:train:103-153batch: iter_time=3.443e-04, forward_time=0.203, loss_att=194.025, acc=0.630, loss=194.025, backward_time=0.282, grad_norm=38.086, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.510e-05, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:39:13,099 (trainer:732) INFO: 3epoch:train:154-204batch: iter_time=3.186e-04, forward_time=0.201, loss_att=189.074, acc=0.629, loss=189.074, backward_time=0.278, grad_norm=38.404, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.640e-05, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:39:45,768 (trainer:732) INFO: 3epoch:train:205-255batch: iter_time=3.414e-04, forward_time=0.204, loss_att=194.191, acc=0.640, loss=194.191, backward_time=0.285, grad_norm=38.721, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=5.765e-05, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:40:17,453 (trainer:732) INFO: 3epoch:train:256-306batch: iter_time=2.629e-04, forward_time=0.197, loss_att=186.542, acc=0.629, loss=186.542, backward_time=0.274, grad_norm=35.772, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=5.890e-05, train_time=2.496
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:40:49,690 (trainer:732) INFO: 3epoch:train:307-357batch: iter_time=2.741e-04, forward_time=0.199, loss_att=191.965, acc=0.636, loss=191.965, backward_time=0.279, grad_norm=36.905, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=6.020e-05, train_time=2.518
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:41:21,873 (trainer:732) INFO: 3epoch:train:358-408batch: iter_time=2.453e-04, forward_time=0.198, loss_att=188.837, acc=0.641, loss=188.837, backward_time=0.277, grad_norm=37.818, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=6.150e-05, train_time=2.516
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:41:53,737 (trainer:732) INFO: 3epoch:train:409-459batch: iter_time=3.083e-04, forward_time=0.199, loss_att=184.546, acc=0.641, loss=184.546, backward_time=0.276, grad_norm=34.434, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.275e-05, train_time=2.512
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:42:26,022 (trainer:732) INFO: 3epoch:train:460-510batch: iter_time=3.913e-04, forward_time=0.202, loss_att=184.461, acc=0.640, loss=184.461, backward_time=0.276, grad_norm=34.623, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=6.400e-05, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:42:58,688 (trainer:732) INFO: 3epoch:train:511-561batch: iter_time=3.673e-04, forward_time=0.203, loss_att=181.591, acc=0.644, loss=181.591, backward_time=0.280, grad_norm=36.176, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.530e-05, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:43:31,198 (trainer:732) INFO: 3epoch:train:562-612batch: iter_time=3.561e-04, forward_time=0.201, loss_att=180.734, acc=0.647, loss=180.734, backward_time=0.278, grad_norm=34.889, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.660e-05, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:44:03,369 (trainer:732) INFO: 3epoch:train:613-663batch: iter_time=3.521e-04, forward_time=0.201, loss_att=177.336, acc=0.647, loss=177.336, backward_time=0.276, grad_norm=36.451, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.785e-05, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:44:35,865 (trainer:732) INFO: 3epoch:train:664-714batch: iter_time=3.493e-04, forward_time=0.203, loss_att=179.613, acc=0.654, loss=179.613, backward_time=0.280, grad_norm=36.682, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.910e-05, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:45:08,110 (trainer:732) INFO: 3epoch:train:715-765batch: iter_time=3.385e-04, forward_time=0.201, loss_att=170.995, acc=0.651, loss=170.995, backward_time=0.276, grad_norm=33.233, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.040e-05, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:45:40,586 (trainer:732) INFO: 3epoch:train:766-816batch: iter_time=3.555e-04, forward_time=0.202, loss_att=175.146, acc=0.656, loss=175.146, backward_time=0.278, grad_norm=34.433, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.170e-05, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:46:12,597 (trainer:732) INFO: 3epoch:train:817-867batch: iter_time=3.464e-04, forward_time=0.199, loss_att=168.641, acc=0.655, loss=168.641, backward_time=0.275, grad_norm=34.685, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.295e-05, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:46:45,429 (trainer:732) INFO: 3epoch:train:868-918batch: iter_time=3.776e-04, forward_time=0.203, loss_att=175.077, acc=0.668, loss=175.077, backward_time=0.281, grad_norm=35.625, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.420e-05, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:47:18,065 (trainer:732) INFO: 3epoch:train:919-969batch: iter_time=3.510e-04, forward_time=0.203, loss_att=172.617, acc=0.662, loss=172.617, backward_time=0.280, grad_norm=35.964, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.550e-05, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:47:50,408 (trainer:732) INFO: 3epoch:train:970-1020batch: iter_time=2.981e-04, forward_time=0.201, loss_att=167.454, acc=0.657, loss=167.454, backward_time=0.277, grad_norm=35.569, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.680e-05, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:58:08,065 (trainer:338) INFO: 3epoch results: [train] iter_time=8.292e-04, forward_time=0.201, loss_att=181.941, acc=0.644, loss=181.941, backward_time=0.278, grad_norm=36.021, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.490e-05, train_time=3.121, time=13 minutes and 30.1 seconds, total_count=3111, gpu_max_cached_mem_GB=30.428, [valid] loss_att=166.060, acc=0.683, cer=0.387, wer=0.732, loss=166.060, time=4 minutes and 43.71 seconds, total_count=264, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 22.2 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:58:12,106 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 16:58:12,109 (trainer:272) INFO: 4/60epoch started. Estimated time to finish: 22 hours, 4 minutes and 54.16 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:01:15,762 (trainer:732) INFO: 4epoch:train:1-51batch: iter_time=0.011, forward_time=0.204, loss_att=167.669, acc=0.670, loss=167.669, backward_time=0.278, grad_norm=33.939, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.845e-05, train_time=15.152
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:01:48,753 (trainer:732) INFO: 4epoch:train:52-102batch: iter_time=3.461e-04, forward_time=0.204, loss_att=172.923, acc=0.679, loss=172.923, backward_time=0.283, grad_norm=36.991, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.970e-05, train_time=2.589
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:02:20,841 (trainer:732) INFO: 4epoch:train:103-153batch: iter_time=3.566e-04, forward_time=0.199, loss_att=159.126, acc=0.666, loss=159.126, backward_time=0.275, grad_norm=33.727, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.100e-05, train_time=2.519
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:02:53,204 (trainer:732) INFO: 4epoch:train:154-204batch: iter_time=3.133e-04, forward_time=0.201, loss_att=163.338, acc=0.672, loss=163.338, backward_time=0.277, grad_norm=33.112, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=8.230e-05, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:03:25,314 (trainer:732) INFO: 4epoch:train:205-255batch: iter_time=3.711e-04, forward_time=0.200, loss_att=160.017, acc=0.675, loss=160.017, backward_time=0.276, grad_norm=33.160, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.355e-05, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:03:57,664 (trainer:732) INFO: 4epoch:train:256-306batch: iter_time=3.272e-04, forward_time=0.201, loss_att=165.785, acc=0.680, loss=165.785, backward_time=0.278, grad_norm=35.896, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.480e-05, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:04:29,985 (trainer:732) INFO: 4epoch:train:307-357batch: iter_time=3.655e-04, forward_time=0.200, loss_att=163.433, acc=0.682, loss=163.433, backward_time=0.276, grad_norm=34.377, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.610e-05, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:05:02,457 (trainer:732) INFO: 4epoch:train:358-408batch: iter_time=3.429e-04, forward_time=0.200, loss_att=161.496, acc=0.682, loss=161.496, backward_time=0.277, grad_norm=34.540, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.740e-05, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:05:34,872 (trainer:732) INFO: 4epoch:train:409-459batch: iter_time=3.545e-04, forward_time=0.201, loss_att=164.690, acc=0.689, loss=164.690, backward_time=0.279, grad_norm=36.341, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.865e-05, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:06:07,257 (trainer:732) INFO: 4epoch:train:460-510batch: iter_time=3.346e-04, forward_time=0.202, loss_att=160.660, acc=0.685, loss=160.660, backward_time=0.279, grad_norm=35.217, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.990e-05, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:06:40,471 (trainer:732) INFO: 4epoch:train:511-561batch: iter_time=3.604e-04, forward_time=0.205, loss_att=159.206, acc=0.694, loss=159.206, backward_time=0.285, grad_norm=36.680, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.120e-05, train_time=2.603
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:07:13,190 (trainer:732) INFO: 4epoch:train:562-612batch: iter_time=3.424e-04, forward_time=0.203, loss_att=162.074, acc=0.692, loss=162.074, backward_time=0.281, grad_norm=34.750, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.250e-05, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:07:45,683 (trainer:732) INFO: 4epoch:train:613-663batch: iter_time=3.551e-04, forward_time=0.202, loss_att=154.398, acc=0.694, loss=154.398, backward_time=0.278, grad_norm=34.997, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.375e-05, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:08:18,351 (trainer:732) INFO: 4epoch:train:664-714batch: iter_time=3.549e-04, forward_time=0.203, loss_att=156.198, acc=0.695, loss=156.198, backward_time=0.280, grad_norm=35.168, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.500e-05, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:08:50,603 (trainer:732) INFO: 4epoch:train:715-765batch: iter_time=3.032e-04, forward_time=0.201, loss_att=158.587, acc=0.691, loss=158.587, backward_time=0.276, grad_norm=33.266, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.630e-05, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:09:23,280 (trainer:732) INFO: 4epoch:train:766-816batch: iter_time=3.328e-04, forward_time=0.202, loss_att=156.137, acc=0.699, loss=156.137, backward_time=0.280, grad_norm=35.098, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.760e-05, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:09:56,019 (trainer:732) INFO: 4epoch:train:817-867batch: iter_time=3.450e-04, forward_time=0.205, loss_att=152.075, acc=0.699, loss=152.075, backward_time=0.283, grad_norm=34.797, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.885e-05, train_time=2.590
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:10:28,552 (trainer:732) INFO: 4epoch:train:868-918batch: iter_time=3.077e-04, forward_time=0.201, loss_att=152.547, acc=0.702, loss=152.547, backward_time=0.277, grad_norm=34.604, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=1.001e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:11:00,963 (trainer:732) INFO: 4epoch:train:919-969batch: iter_time=3.142e-04, forward_time=0.201, loss_att=153.893, acc=0.699, loss=153.893, backward_time=0.278, grad_norm=34.008, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.014e-04, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:11:33,586 (trainer:732) INFO: 4epoch:train:970-1020batch: iter_time=2.883e-04, forward_time=0.202, loss_att=150.182, acc=0.706, loss=150.182, backward_time=0.279, grad_norm=33.650, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.027e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:21:59,951 (trainer:338) INFO: 4epoch results: [train] iter_time=8.385e-04, forward_time=0.202, loss_att=159.536, acc=0.688, loss=159.536, backward_time=0.279, grad_norm=34.694, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.080e-05, train_time=3.133, time=13 minutes and 33.41 seconds, total_count=4148, gpu_max_cached_mem_GB=30.428, [valid] loss_att=146.720, acc=0.722, cer=0.331, wer=0.687, loss=146.720, time=4 minutes and 49.31 seconds, total_count=352, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 25.12 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:22:03,818 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:22:03,820 (trainer:272) INFO: 5/60epoch started. Estimated time to finish: 21 hours, 50 minutes and 18.6 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:25:07,749 (trainer:732) INFO: 5epoch:train:1-51batch: iter_time=0.011, forward_time=0.205, loss_att=151.620, acc=0.712, loss=151.620, backward_time=0.283, grad_norm=35.837, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.043e-04, train_time=15.178
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:25:39,936 (trainer:732) INFO: 5epoch:train:52-102batch: iter_time=3.556e-04, forward_time=0.200, loss_att=144.402, acc=0.708, loss=144.402, backward_time=0.275, grad_norm=34.260, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.056e-04, train_time=2.521
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:26:12,078 (trainer:732) INFO: 5epoch:train:103-153batch: iter_time=3.717e-04, forward_time=0.200, loss_att=141.835, acc=0.712, loss=141.835, backward_time=0.275, grad_norm=34.324, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.069e-04, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:26:44,373 (trainer:732) INFO: 5epoch:train:154-204batch: iter_time=3.591e-04, forward_time=0.200, loss_att=145.727, acc=0.713, loss=145.727, backward_time=0.275, grad_norm=34.615, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.082e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:27:16,865 (trainer:732) INFO: 5epoch:train:205-255batch: iter_time=3.488e-04, forward_time=0.203, loss_att=145.028, acc=0.715, loss=145.028, backward_time=0.281, grad_norm=35.993, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.094e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:27:49,036 (trainer:732) INFO: 5epoch:train:256-306batch: iter_time=3.675e-04, forward_time=0.200, loss_att=143.133, acc=0.717, loss=143.133, backward_time=0.276, grad_norm=32.781, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.107e-04, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:28:21,573 (trainer:732) INFO: 5epoch:train:307-357batch: iter_time=3.772e-04, forward_time=0.201, loss_att=143.996, acc=0.720, loss=143.996, backward_time=0.279, grad_norm=33.891, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.120e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:28:54,467 (trainer:732) INFO: 5epoch:train:358-408batch: iter_time=3.539e-04, forward_time=0.204, loss_att=149.220, acc=0.724, loss=149.220, backward_time=0.283, grad_norm=33.957, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.133e-04, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:29:26,678 (trainer:732) INFO: 5epoch:train:409-459batch: iter_time=3.643e-04, forward_time=0.202, loss_att=143.364, acc=0.722, loss=143.364, backward_time=0.278, grad_norm=32.475, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.145e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:29:58,821 (trainer:732) INFO: 5epoch:train:460-510batch: iter_time=3.849e-04, forward_time=0.201, loss_att=141.758, acc=0.720, loss=141.758, backward_time=0.275, grad_norm=32.823, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.158e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:30:31,553 (trainer:732) INFO: 5epoch:train:511-561batch: iter_time=3.758e-04, forward_time=0.203, loss_att=140.223, acc=0.730, loss=140.223, backward_time=0.282, grad_norm=34.643, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.171e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:31:04,113 (trainer:732) INFO: 5epoch:train:562-612batch: iter_time=3.857e-04, forward_time=0.202, loss_att=140.999, acc=0.728, loss=140.999, backward_time=0.278, grad_norm=35.176, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.184e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:31:36,294 (trainer:732) INFO: 5epoch:train:613-663batch: iter_time=3.846e-04, forward_time=0.201, loss_att=138.952, acc=0.730, loss=138.952, backward_time=0.278, grad_norm=35.888, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.196e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:32:09,034 (trainer:732) INFO: 5epoch:train:664-714batch: iter_time=3.841e-04, forward_time=0.202, loss_att=136.235, acc=0.737, loss=136.235, backward_time=0.280, grad_norm=35.932, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.209e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:32:41,845 (trainer:732) INFO: 5epoch:train:715-765batch: iter_time=3.342e-04, forward_time=0.203, loss_att=142.385, acc=0.736, loss=142.385, backward_time=0.281, grad_norm=36.267, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.222e-04, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:33:14,972 (trainer:732) INFO: 5epoch:train:766-816batch: iter_time=3.888e-04, forward_time=0.205, loss_att=137.060, acc=0.738, loss=137.060, backward_time=0.285, grad_norm=35.468, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.235e-04, train_time=2.587
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:33:47,188 (trainer:732) INFO: 5epoch:train:817-867batch: iter_time=3.513e-04, forward_time=0.201, loss_att=134.123, acc=0.735, loss=134.123, backward_time=0.277, grad_norm=34.427, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.247e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:34:19,947 (trainer:732) INFO: 5epoch:train:868-918batch: iter_time=3.776e-04, forward_time=0.203, loss_att=140.881, acc=0.735, loss=140.881, backward_time=0.279, grad_norm=36.062, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.260e-04, train_time=2.564
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:34:52,263 (trainer:732) INFO: 5epoch:train:919-969batch: iter_time=3.280e-04, forward_time=0.202, loss_att=132.549, acc=0.739, loss=132.549, backward_time=0.278, grad_norm=35.441, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.273e-04, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:35:24,781 (trainer:732) INFO: 5epoch:train:970-1020batch: iter_time=3.243e-04, forward_time=0.201, loss_att=130.853, acc=0.741, loss=130.853, backward_time=0.277, grad_norm=35.483, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.286e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:45:48,413 (trainer:338) INFO: 5epoch results: [train] iter_time=9.053e-04, forward_time=0.202, loss_att=141.029, acc=0.726, loss=141.029, backward_time=0.279, grad_norm=34.812, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.167e-04, train_time=3.132, time=13 minutes and 33.05 seconds, total_count=5185, gpu_max_cached_mem_GB=30.428, [valid] loss_att=131.469, acc=0.757, cer=0.298, wer=0.639, loss=131.469, time=4 minutes and 47.46 seconds, total_count=440, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 24.08 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:45:52,439 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:45:52,442 (trainer:272) INFO: 6/60epoch started. Estimated time to finish: 21 hours, 31 minutes and 26.6 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:48:58,974 (trainer:732) INFO: 6epoch:train:1-51batch: iter_time=0.015, forward_time=0.205, loss_att=131.548, acc=0.746, loss=131.548, backward_time=0.280, grad_norm=34.097, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=1.302e-04, train_time=15.399
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:49:31,750 (trainer:732) INFO: 6epoch:train:52-102batch: iter_time=3.378e-04, forward_time=0.204, loss_att=133.370, acc=0.752, loss=133.370, backward_time=0.282, grad_norm=34.443, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.315e-04, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:50:04,709 (trainer:732) INFO: 6epoch:train:103-153batch: iter_time=3.401e-04, forward_time=0.205, loss_att=130.757, acc=0.749, loss=130.757, backward_time=0.285, grad_norm=34.956, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.328e-04, train_time=2.582
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:50:36,893 (trainer:732) INFO: 6epoch:train:154-204batch: iter_time=3.445e-04, forward_time=0.199, loss_att=129.334, acc=0.747, loss=129.334, backward_time=0.274, grad_norm=34.661, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.341e-04, train_time=2.515
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:51:08,949 (trainer:732) INFO: 6epoch:train:205-255batch: iter_time=3.505e-04, forward_time=0.200, loss_att=129.200, acc=0.750, loss=129.200, backward_time=0.277, grad_norm=33.994, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.353e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:51:41,596 (trainer:732) INFO: 6epoch:train:256-306batch: iter_time=3.486e-04, forward_time=0.202, loss_att=129.838, acc=0.754, loss=129.838, backward_time=0.280, grad_norm=36.428, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.366e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:52:14,083 (trainer:732) INFO: 6epoch:train:307-357batch: iter_time=3.381e-04, forward_time=0.202, loss_att=130.169, acc=0.753, loss=130.169, backward_time=0.279, grad_norm=35.254, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.379e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:52:46,267 (trainer:732) INFO: 6epoch:train:358-408batch: iter_time=3.359e-04, forward_time=0.200, loss_att=121.919, acc=0.756, loss=121.919, backward_time=0.275, grad_norm=32.269, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.392e-04, train_time=2.516
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:53:18,451 (trainer:732) INFO: 6epoch:train:409-459batch: iter_time=4.122e-04, forward_time=0.201, loss_att=126.747, acc=0.755, loss=126.747, backward_time=0.276, grad_norm=31.425, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=1.404e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:53:50,888 (trainer:732) INFO: 6epoch:train:460-510batch: iter_time=3.562e-04, forward_time=0.201, loss_att=125.782, acc=0.761, loss=125.782, backward_time=0.277, grad_norm=34.699, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.417e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:54:23,548 (trainer:732) INFO: 6epoch:train:511-561batch: iter_time=3.721e-04, forward_time=0.203, loss_att=124.648, acc=0.763, loss=124.648, backward_time=0.280, grad_norm=35.245, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.430e-04, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:54:56,367 (trainer:732) INFO: 6epoch:train:562-612batch: iter_time=3.669e-04, forward_time=0.203, loss_att=127.928, acc=0.764, loss=127.928, backward_time=0.280, grad_norm=34.730, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=1.443e-04, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:55:28,702 (trainer:732) INFO: 6epoch:train:613-663batch: iter_time=3.613e-04, forward_time=0.203, loss_att=124.561, acc=0.760, loss=124.561, backward_time=0.280, grad_norm=35.444, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.455e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:56:01,364 (trainer:732) INFO: 6epoch:train:664-714batch: iter_time=3.514e-04, forward_time=0.202, loss_att=125.237, acc=0.761, loss=125.237, backward_time=0.278, grad_norm=34.227, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.468e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:56:34,149 (trainer:732) INFO: 6epoch:train:715-765batch: iter_time=3.116e-04, forward_time=0.202, loss_att=120.298, acc=0.769, loss=120.298, backward_time=0.282, grad_norm=85.536, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.481e-04, train_time=2.576
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:57:06,475 (trainer:732) INFO: 6epoch:train:766-816batch: iter_time=3.757e-04, forward_time=0.201, loss_att=120.139, acc=0.765, loss=120.139, backward_time=0.275, grad_norm=32.392, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.494e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:57:38,615 (trainer:732) INFO: 6epoch:train:817-867batch: iter_time=3.731e-04, forward_time=0.201, loss_att=120.365, acc=0.767, loss=120.365, backward_time=0.276, grad_norm=33.480, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.506e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:58:10,823 (trainer:732) INFO: 6epoch:train:868-918batch: iter_time=3.408e-04, forward_time=0.200, loss_att=117.444, acc=0.768, loss=117.444, backward_time=0.275, grad_norm=32.998, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.519e-04, train_time=2.521
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:58:43,522 (trainer:732) INFO: 6epoch:train:919-969batch: iter_time=3.749e-04, forward_time=0.203, loss_att=118.165, acc=0.771, loss=118.165, backward_time=0.282, grad_norm=35.077, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=1.532e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 17:59:16,029 (trainer:732) INFO: 6epoch:train:970-1020batch: iter_time=3.036e-04, forward_time=0.201, loss_att=120.741, acc=0.773, loss=120.741, backward_time=0.278, grad_norm=33.434, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.545e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:09:34,109 (trainer:338) INFO: 6epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=125.272, acc=0.759, loss=125.272, backward_time=0.279, grad_norm=36.785, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.426e-04, train_time=3.142, time=13 minutes and 35.65 seconds, total_count=6222, gpu_max_cached_mem_GB=30.428, [valid] loss_att=118.915, acc=0.785, cer=0.262, wer=0.586, loss=118.915, time=4 minutes and 44 seconds, total_count=528, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 22.01 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:09:38,300 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:09:38,303 (trainer:272) INFO: 7/60epoch started. Estimated time to finish: 21 hours, 10 minutes and 30.88 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:12:45,533 (trainer:732) INFO: 7epoch:train:1-51batch: iter_time=0.014, forward_time=0.203, loss_att=117.535, acc=0.777, loss=117.535, backward_time=0.279, grad_norm=33.732, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.562e-04, train_time=15.456
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:13:18,013 (trainer:732) INFO: 7epoch:train:52-102batch: iter_time=3.225e-04, forward_time=0.202, loss_att=116.196, acc=0.777, loss=116.196, backward_time=0.279, grad_norm=33.854, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.574e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:13:50,209 (trainer:732) INFO: 7epoch:train:103-153batch: iter_time=3.322e-04, forward_time=0.201, loss_att=114.206, acc=0.774, loss=114.206, backward_time=0.277, grad_norm=33.204, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.587e-04, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:14:22,426 (trainer:732) INFO: 7epoch:train:154-204batch: iter_time=3.362e-04, forward_time=0.200, loss_att=106.409, acc=0.784, loss=106.409, backward_time=0.275, grad_norm=33.173, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.600e-04, train_time=2.517
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:14:54,924 (trainer:732) INFO: 7epoch:train:205-255batch: iter_time=3.255e-04, forward_time=0.203, loss_att=121.702, acc=0.780, loss=121.702, backward_time=0.280, grad_norm=35.635, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.612e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:15:27,377 (trainer:732) INFO: 7epoch:train:256-306batch: iter_time=3.125e-04, forward_time=0.201, loss_att=114.536, acc=0.783, loss=114.536, backward_time=0.278, grad_norm=34.491, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.625e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:16:00,115 (trainer:732) INFO: 7epoch:train:307-357batch: iter_time=3.731e-04, forward_time=0.202, loss_att=112.925, acc=0.782, loss=112.925, backward_time=0.277, grad_norm=34.447, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=1.638e-04, train_time=2.572
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:16:32,719 (trainer:732) INFO: 7epoch:train:358-408batch: iter_time=3.543e-04, forward_time=0.203, loss_att=114.698, acc=0.786, loss=114.698, backward_time=0.278, grad_norm=34.210, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.651e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:17:05,207 (trainer:732) INFO: 7epoch:train:409-459batch: iter_time=3.326e-04, forward_time=0.204, loss_att=111.574, acc=0.788, loss=111.574, backward_time=0.281, grad_norm=35.747, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.664e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:17:38,315 (trainer:732) INFO: 7epoch:train:460-510batch: iter_time=3.628e-04, forward_time=0.205, loss_att=115.584, acc=0.787, loss=115.584, backward_time=0.284, grad_norm=36.732, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.676e-04, train_time=2.589
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:18:10,625 (trainer:732) INFO: 7epoch:train:511-561batch: iter_time=3.387e-04, forward_time=0.202, loss_att=106.687, acc=0.794, loss=106.687, backward_time=0.277, grad_norm=36.678, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.689e-04, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:18:43,268 (trainer:732) INFO: 7epoch:train:562-612batch: iter_time=3.321e-04, forward_time=0.202, loss_att=111.507, acc=0.795, loss=111.507, backward_time=0.280, grad_norm=37.854, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.702e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:19:15,983 (trainer:732) INFO: 7epoch:train:613-663batch: iter_time=3.453e-04, forward_time=0.204, loss_att=113.519, acc=0.795, loss=113.519, backward_time=0.282, grad_norm=37.041, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.714e-04, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:19:48,652 (trainer:732) INFO: 7epoch:train:664-714batch: iter_time=3.402e-04, forward_time=0.203, loss_att=114.148, acc=0.792, loss=114.148, backward_time=0.280, grad_norm=38.244, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.727e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:20:21,064 (trainer:732) INFO: 7epoch:train:715-765batch: iter_time=2.929e-04, forward_time=0.201, loss_att=107.278, acc=0.794, loss=107.278, backward_time=0.277, grad_norm=32.472, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.740e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:20:53,532 (trainer:732) INFO: 7epoch:train:766-816batch: iter_time=3.528e-04, forward_time=0.201, loss_att=108.084, acc=0.796, loss=108.084, backward_time=0.278, grad_norm=35.338, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=1.753e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:21:25,895 (trainer:732) INFO: 7epoch:train:817-867batch: iter_time=3.308e-04, forward_time=0.203, loss_att=103.748, acc=0.800, loss=103.748, backward_time=0.279, grad_norm=38.356, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.766e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:21:58,428 (trainer:732) INFO: 7epoch:train:868-918batch: iter_time=3.430e-04, forward_time=0.202, loss_att=108.298, acc=0.796, loss=108.298, backward_time=0.279, grad_norm=34.597, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.778e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:22:30,755 (trainer:732) INFO: 7epoch:train:919-969batch: iter_time=3.340e-04, forward_time=0.200, loss_att=103.127, acc=0.802, loss=103.127, backward_time=0.277, grad_norm=34.144, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.791e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:23:03,348 (trainer:732) INFO: 7epoch:train:970-1020batch: iter_time=3.185e-04, forward_time=0.203, loss_att=107.777, acc=0.799, loss=107.777, backward_time=0.280, grad_norm=34.923, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.804e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:33:27,980 (trainer:338) INFO: 7epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=111.328, acc=0.789, loss=111.328, backward_time=0.279, grad_norm=35.265, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.685e-04, train_time=3.147, time=13 minutes and 36.92 seconds, total_count=7259, gpu_max_cached_mem_GB=30.428, [valid] loss_att=109.012, acc=0.806, cer=0.236, wer=0.528, loss=109.012, time=4 minutes and 47.02 seconds, total_count=616, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 25.73 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:33:31,783 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:33:31,786 (trainer:272) INFO: 8/60epoch started. Estimated time to finish: 20 hours, 49 minutes and 44.26 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:36:33,884 (trainer:732) INFO: 8epoch:train:1-51batch: iter_time=0.015, forward_time=0.204, loss_att=106.407, acc=0.802, loss=106.407, backward_time=0.279, grad_norm=37.928, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.821e-04, train_time=15.027

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<21395>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 132)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<21449>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 132)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<60674>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 132)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 132)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 132)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 132)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 132)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<63833>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<37430>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<37440>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 139)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 139)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 139)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 139)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 139)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 139)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<41327>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 139)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 133)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 135)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<41437>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 135)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 135)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 135)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 135)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 135)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:37:06,536 (trainer:732) INFO: 8epoch:train:52-102batch: iter_time=3.442e-04, forward_time=0.201, loss_att=105.222, acc=0.803, loss=105.222, backward_time=0.279, grad_norm=36.152, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.833e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:37:39,663 (trainer:732) INFO: 8epoch:train:103-153batch: iter_time=3.443e-04, forward_time=0.203, loss_att=104.281, acc=0.806, loss=104.281, backward_time=0.281, grad_norm=35.606, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.846e-04, train_time=2.582
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:38:12,050 (trainer:732) INFO: 8epoch:train:154-204batch: iter_time=3.597e-04, forward_time=0.201, loss_att=101.754, acc=0.809, loss=101.754, backward_time=0.276, grad_norm=33.112, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.859e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:38:44,371 (trainer:732) INFO: 8epoch:train:205-255batch: iter_time=3.623e-04, forward_time=0.201, loss_att=101.625, acc=0.809, loss=101.625, backward_time=0.278, grad_norm=37.145, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.871e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:39:16,510 (trainer:732) INFO: 8epoch:train:256-306batch: iter_time=3.347e-04, forward_time=0.201, loss_att=98.354, acc=0.813, loss=98.354, backward_time=0.275, grad_norm=38.055, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.884e-04, train_time=2.519
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:39:49,037 (trainer:732) INFO: 8epoch:train:307-357batch: iter_time=3.528e-04, forward_time=0.202, loss_att=106.751, acc=0.806, loss=106.751, backward_time=0.279, grad_norm=37.233, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.897e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:40:21,974 (trainer:732) INFO: 8epoch:train:358-408batch: iter_time=3.757e-04, forward_time=0.204, loss_att=96.324, acc=0.815, loss=96.324, backward_time=0.281, grad_norm=37.655, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.910e-04, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:40:54,463 (trainer:732) INFO: 8epoch:train:409-459batch: iter_time=3.828e-04, forward_time=0.203, loss_att=102.513, acc=0.810, loss=102.513, backward_time=0.280, grad_norm=35.084, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.923e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:41:27,022 (trainer:732) INFO: 8epoch:train:460-510batch: iter_time=3.797e-04, forward_time=0.203, loss_att=101.533, acc=0.812, loss=101.533, backward_time=0.279, grad_norm=37.154, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=1.935e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:41:59,038 (trainer:732) INFO: 8epoch:train:511-561batch: iter_time=3.810e-04, forward_time=0.200, loss_att=90.654, acc=0.817, loss=90.654, backward_time=0.274, grad_norm=36.333, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.948e-04, train_time=2.512
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:42:31,933 (trainer:732) INFO: 8epoch:train:562-612batch: iter_time=3.383e-04, forward_time=0.204, loss_att=94.988, acc=0.818, loss=94.988, backward_time=0.281, grad_norm=34.922, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.961e-04, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:43:04,401 (trainer:732) INFO: 8epoch:train:613-663batch: iter_time=3.965e-04, forward_time=0.203, loss_att=98.049, acc=0.819, loss=98.049, backward_time=0.279, grad_norm=34.034, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=1.973e-04, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:43:36,503 (trainer:732) INFO: 8epoch:train:664-714batch: iter_time=3.530e-04, forward_time=0.200, loss_att=94.438, acc=0.819, loss=94.438, backward_time=0.275, grad_norm=33.763, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.986e-04, train_time=2.521
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:44:09,340 (trainer:732) INFO: 8epoch:train:715-765batch: iter_time=3.231e-04, forward_time=0.203, loss_att=101.825, acc=0.815, loss=101.825, backward_time=0.281, grad_norm=35.724, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.999e-04, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:44:41,985 (trainer:732) INFO: 8epoch:train:766-816batch: iter_time=3.687e-04, forward_time=0.202, loss_att=102.354, acc=0.815, loss=102.354, backward_time=0.280, grad_norm=37.322, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.012e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:45:14,156 (trainer:732) INFO: 8epoch:train:817-867batch: iter_time=3.548e-04, forward_time=0.202, loss_att=94.474, acc=0.824, loss=94.474, backward_time=0.277, grad_norm=39.850, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.024e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:45:46,446 (trainer:732) INFO: 8epoch:train:868-918batch: iter_time=3.634e-04, forward_time=0.201, loss_att=97.279, acc=0.816, loss=97.279, backward_time=0.277, grad_norm=34.066, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.037e-04, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:46:19,270 (trainer:732) INFO: 8epoch:train:919-969batch: iter_time=3.495e-04, forward_time=0.204, loss_att=97.937, acc=0.823, loss=97.937, backward_time=0.282, grad_norm=38.854, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.050e-04, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:46:52,222 (trainer:732) INFO: 8epoch:train:970-1020batch: iter_time=3.338e-04, forward_time=0.204, loss_att=94.091, acc=0.824, loss=94.091, backward_time=0.281, grad_norm=34.570, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.063e-04, train_time=2.572

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<40219>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 98)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<40283>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 98)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<22155>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 98)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 98)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 98)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 98)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 98)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<27496>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<62876>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<62866>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 109)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 109)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<5644>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 109)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 109)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 103)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 109)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.11.213<5768>
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:445 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO include/socket.h:457 -> 2
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:229 -> 2

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 109)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 109)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 111)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 111)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 111)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 111)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:57:08,899 (trainer:338) INFO: 8epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=99.368, acc=0.814, loss=99.368, backward_time=0.279, grad_norm=36.226, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=1.944e-04, train_time=3.130, time=13 minutes and 32.57 seconds, total_count=8296, gpu_max_cached_mem_GB=30.428, [valid] loss_att=101.243, acc=0.821, cer=0.216, wer=0.476, loss=101.243, time=4 minutes and 44.65 seconds, total_count=704, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 19.88 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:57:13,077 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 18:57:13,081 (trainer:272) INFO: 9/60epoch started. Estimated time to finish: 20 hours, 26 minutes and 51.69 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:00:17,243 (trainer:732) INFO: 9epoch:train:1-51batch: iter_time=0.017, forward_time=0.206, loss_att=94.048, acc=0.827, loss=94.048, backward_time=0.282, grad_norm=37.220, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.080e-04, train_time=15.200
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:00:49,517 (trainer:732) INFO: 9epoch:train:52-102batch: iter_time=3.365e-04, forward_time=0.201, loss_att=88.168, acc=0.824, loss=88.168, backward_time=0.275, grad_norm=36.248, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.092e-04, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:01:22,077 (trainer:732) INFO: 9epoch:train:103-153batch: iter_time=3.436e-04, forward_time=0.202, loss_att=92.501, acc=0.829, loss=92.501, backward_time=0.280, grad_norm=37.052, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.105e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:01:54,593 (trainer:732) INFO: 9epoch:train:154-204batch: iter_time=3.360e-04, forward_time=0.201, loss_att=90.982, acc=0.830, loss=90.982, backward_time=0.278, grad_norm=34.840, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.118e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:02:26,779 (trainer:732) INFO: 9epoch:train:205-255batch: iter_time=3.173e-04, forward_time=0.200, loss_att=92.738, acc=0.829, loss=92.738, backward_time=0.277, grad_norm=35.743, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.131e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:02:59,897 (trainer:732) INFO: 9epoch:train:256-306batch: iter_time=3.540e-04, forward_time=0.204, loss_att=95.023, acc=0.826, loss=95.023, backward_time=0.283, grad_norm=37.493, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.143e-04, train_time=2.596
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:03:32,732 (trainer:732) INFO: 9epoch:train:307-357batch: iter_time=3.353e-04, forward_time=0.204, loss_att=90.075, acc=0.832, loss=90.075, backward_time=0.281, grad_norm=39.778, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.156e-04, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:04:05,083 (trainer:732) INFO: 9epoch:train:358-408batch: iter_time=3.465e-04, forward_time=0.201, loss_att=89.841, acc=0.832, loss=89.841, backward_time=0.276, grad_norm=38.434, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.169e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:04:37,622 (trainer:732) INFO: 9epoch:train:409-459batch: iter_time=3.339e-04, forward_time=0.202, loss_att=92.883, acc=0.831, loss=92.883, backward_time=0.279, grad_norm=37.082, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.181e-04, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:05:10,036 (trainer:732) INFO: 9epoch:train:460-510batch: iter_time=3.319e-04, forward_time=0.201, loss_att=90.671, acc=0.831, loss=90.671, backward_time=0.278, grad_norm=39.882, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.194e-04, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:05:42,572 (trainer:732) INFO: 9epoch:train:511-561batch: iter_time=3.657e-04, forward_time=0.203, loss_att=91.032, acc=0.833, loss=91.032, backward_time=0.280, grad_norm=38.991, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.207e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:06:15,416 (trainer:732) INFO: 9epoch:train:562-612batch: iter_time=3.715e-04, forward_time=0.204, loss_att=92.934, acc=0.831, loss=92.934, backward_time=0.281, grad_norm=34.497, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.220e-04, train_time=2.564
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:06:47,719 (trainer:732) INFO: 9epoch:train:613-663batch: iter_time=4.013e-04, forward_time=0.202, loss_att=87.204, acc=0.837, loss=87.204, backward_time=0.278, grad_norm=35.306, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.233e-04, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:07:19,763 (trainer:732) INFO: 9epoch:train:664-714batch: iter_time=3.680e-04, forward_time=0.200, loss_att=82.896, acc=0.841, loss=82.896, backward_time=0.274, grad_norm=32.911, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=2.245e-04, train_time=2.506
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:07:52,301 (trainer:732) INFO: 9epoch:train:715-765batch: iter_time=3.465e-04, forward_time=0.203, loss_att=90.089, acc=0.833, loss=90.089, backward_time=0.279, grad_norm=35.871, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=2.258e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:08:25,067 (trainer:732) INFO: 9epoch:train:766-816batch: iter_time=3.379e-04, forward_time=0.203, loss_att=92.378, acc=0.832, loss=92.378, backward_time=0.280, grad_norm=38.801, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.271e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:08:57,431 (trainer:732) INFO: 9epoch:train:817-867batch: iter_time=3.556e-04, forward_time=0.203, loss_att=86.160, acc=0.838, loss=86.160, backward_time=0.277, grad_norm=33.847, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.283e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:09:29,614 (trainer:732) INFO: 9epoch:train:868-918batch: iter_time=3.590e-04, forward_time=0.200, loss_att=83.115, acc=0.843, loss=83.115, backward_time=0.275, grad_norm=37.178, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.296e-04, train_time=2.514
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:10:02,273 (trainer:732) INFO: 9epoch:train:919-969batch: iter_time=3.505e-04, forward_time=0.203, loss_att=85.828, acc=0.841, loss=85.828, backward_time=0.281, grad_norm=38.425, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.309e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:10:34,900 (trainer:732) INFO: 9epoch:train:970-1020batch: iter_time=3.023e-04, forward_time=0.202, loss_att=87.299, acc=0.839, loss=87.299, backward_time=0.280, grad_norm=41.459, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.322e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:20:53,120 (trainer:338) INFO: 9epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=89.654, acc=0.833, loss=89.654, backward_time=0.279, grad_norm=37.094, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.203e-04, train_time=3.135, time=13 minutes and 33.92 seconds, total_count=9333, gpu_max_cached_mem_GB=30.428, [valid] loss_att=95.602, acc=0.830, cer=0.205, wer=0.453, loss=95.602, time=4 minutes and 44.62 seconds, total_count=792, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 21.5 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:20:57,660 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:20:57,664 (trainer:272) INFO: 10/60epoch started. Estimated time to finish: 20 hours, 4 minutes and 6.93 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:24:02,801 (trainer:732) INFO: 10epoch:train:1-51batch: iter_time=0.012, forward_time=0.204, loss_att=82.842, acc=0.844, loss=82.842, backward_time=0.280, grad_norm=38.260, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.338e-04, train_time=15.279
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:24:34,968 (trainer:732) INFO: 10epoch:train:52-102batch: iter_time=3.408e-04, forward_time=0.200, loss_att=82.201, acc=0.840, loss=82.201, backward_time=0.275, grad_norm=38.915, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.351e-04, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:25:08,001 (trainer:732) INFO: 10epoch:train:103-153batch: iter_time=3.505e-04, forward_time=0.204, loss_att=89.350, acc=0.843, loss=89.350, backward_time=0.284, grad_norm=37.983, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.364e-04, train_time=2.589
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:25:40,402 (trainer:732) INFO: 10epoch:train:154-204batch: iter_time=3.289e-04, forward_time=0.200, loss_att=83.524, acc=0.844, loss=83.524, backward_time=0.277, grad_norm=39.974, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.377e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:26:12,672 (trainer:732) INFO: 10epoch:train:205-255batch: iter_time=3.204e-04, forward_time=0.201, loss_att=85.475, acc=0.842, loss=85.475, backward_time=0.279, grad_norm=37.586, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.390e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:26:45,150 (trainer:732) INFO: 10epoch:train:256-306batch: iter_time=3.338e-04, forward_time=0.202, loss_att=85.094, acc=0.842, loss=85.094, backward_time=0.279, grad_norm=38.697, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.402e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:27:17,601 (trainer:732) INFO: 10epoch:train:307-357batch: iter_time=3.211e-04, forward_time=0.201, loss_att=82.205, acc=0.847, loss=82.205, backward_time=0.278, grad_norm=39.290, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.415e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:27:50,339 (trainer:732) INFO: 10epoch:train:358-408batch: iter_time=3.084e-04, forward_time=0.202, loss_att=82.348, acc=0.849, loss=82.348, backward_time=0.279, grad_norm=39.463, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.428e-04, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:28:22,793 (trainer:732) INFO: 10epoch:train:409-459batch: iter_time=3.542e-04, forward_time=0.202, loss_att=83.338, acc=0.848, loss=83.338, backward_time=0.279, grad_norm=38.116, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.440e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:28:55,225 (trainer:732) INFO: 10epoch:train:460-510batch: iter_time=3.612e-04, forward_time=0.202, loss_att=77.215, acc=0.852, loss=77.215, backward_time=0.278, grad_norm=34.034, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.453e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:29:27,534 (trainer:732) INFO: 10epoch:train:511-561batch: iter_time=3.687e-04, forward_time=0.202, loss_att=82.741, acc=0.844, loss=82.741, backward_time=0.276, grad_norm=34.263, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.466e-04, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:30:00,191 (trainer:732) INFO: 10epoch:train:562-612batch: iter_time=3.462e-04, forward_time=0.203, loss_att=79.765, acc=0.850, loss=79.765, backward_time=0.281, grad_norm=36.988, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.479e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:30:32,767 (trainer:732) INFO: 10epoch:train:613-663batch: iter_time=3.625e-04, forward_time=0.204, loss_att=82.863, acc=0.849, loss=82.863, backward_time=0.281, grad_norm=37.946, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.492e-04, train_time=2.568
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:31:05,367 (trainer:732) INFO: 10epoch:train:664-714batch: iter_time=3.547e-04, forward_time=0.203, loss_att=76.089, acc=0.854, loss=76.089, backward_time=0.278, grad_norm=35.745, clip=100.000, loss_scale=1.000, optim_step_time=0.067, optim0_lr0=2.504e-04, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:31:37,999 (trainer:732) INFO: 10epoch:train:715-765batch: iter_time=3.288e-04, forward_time=0.203, loss_att=82.709, acc=0.847, loss=82.709, backward_time=0.280, grad_norm=34.779, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=2.517e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:32:10,292 (trainer:732) INFO: 10epoch:train:766-816batch: iter_time=3.255e-04, forward_time=0.200, loss_att=76.160, acc=0.856, loss=76.160, backward_time=0.276, grad_norm=33.539, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.530e-04, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:32:42,551 (trainer:732) INFO: 10epoch:train:817-867batch: iter_time=2.798e-04, forward_time=0.201, loss_att=82.340, acc=0.848, loss=82.340, backward_time=0.278, grad_norm=34.351, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.542e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:33:15,264 (trainer:732) INFO: 10epoch:train:868-918batch: iter_time=3.186e-04, forward_time=0.202, loss_att=84.982, acc=0.849, loss=84.982, backward_time=0.281, grad_norm=39.712, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.555e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:33:47,603 (trainer:732) INFO: 10epoch:train:919-969batch: iter_time=3.018e-04, forward_time=0.200, loss_att=77.222, acc=0.852, loss=77.222, backward_time=0.276, grad_norm=41.964, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.568e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:34:20,074 (trainer:732) INFO: 10epoch:train:970-1020batch: iter_time=3.300e-04, forward_time=0.203, loss_att=79.666, acc=0.849, loss=79.666, backward_time=0.279, grad_norm=35.304, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.581e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:44:38,038 (trainer:338) INFO: 10epoch results: [train] iter_time=8.957e-04, forward_time=0.202, loss_att=81.780, acc=0.848, loss=81.780, backward_time=0.279, grad_norm=37.349, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.462e-04, train_time=3.137, time=13 minutes and 34.51 seconds, total_count=10370, gpu_max_cached_mem_GB=30.428, [valid] loss_att=92.191, acc=0.837, cer=0.196, wer=0.442, loss=92.191, time=4 minutes and 38.25 seconds, total_count=880, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 27.61 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:44:42,311 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:44:42,316 (trainer:272) INFO: 11/60epoch started. Estimated time to finish: 19 hours, 41 minutes and 10.55 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:47:47,921 (trainer:732) INFO: 11epoch:train:1-51batch: iter_time=0.016, forward_time=0.203, loss_att=80.020, acc=0.853, loss=80.020, backward_time=0.279, grad_norm=34.579, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=2.597e-04, train_time=15.318
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:48:20,586 (trainer:732) INFO: 11epoch:train:52-102batch: iter_time=3.373e-04, forward_time=0.203, loss_att=79.298, acc=0.857, loss=79.298, backward_time=0.280, grad_norm=39.889, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.610e-04, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:48:53,050 (trainer:732) INFO: 11epoch:train:103-153batch: iter_time=3.194e-04, forward_time=0.201, loss_att=76.625, acc=0.857, loss=76.625, backward_time=0.280, grad_norm=39.988, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.623e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:49:25,849 (trainer:732) INFO: 11epoch:train:154-204batch: iter_time=5.076e-04, forward_time=0.204, loss_att=75.055, acc=0.853, loss=75.055, backward_time=0.280, grad_norm=37.979, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.636e-04, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:49:58,155 (trainer:732) INFO: 11epoch:train:205-255batch: iter_time=3.549e-04, forward_time=0.202, loss_att=76.698, acc=0.858, loss=76.698, backward_time=0.278, grad_norm=35.106, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.649e-04, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:50:30,603 (trainer:732) INFO: 11epoch:train:256-306batch: iter_time=3.410e-04, forward_time=0.202, loss_att=75.421, acc=0.858, loss=75.421, backward_time=0.278, grad_norm=35.662, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.661e-04, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:51:03,336 (trainer:732) INFO: 11epoch:train:307-357batch: iter_time=3.512e-04, forward_time=0.202, loss_att=78.587, acc=0.854, loss=78.587, backward_time=0.278, grad_norm=33.967, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=2.674e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:51:36,085 (trainer:732) INFO: 11epoch:train:358-408batch: iter_time=3.736e-04, forward_time=0.203, loss_att=81.013, acc=0.854, loss=81.013, backward_time=0.280, grad_norm=34.653, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.687e-04, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:52:08,340 (trainer:732) INFO: 11epoch:train:409-459batch: iter_time=3.716e-04, forward_time=0.201, loss_att=74.000, acc=0.862, loss=74.000, backward_time=0.278, grad_norm=37.664, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.699e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:52:40,652 (trainer:732) INFO: 11epoch:train:460-510batch: iter_time=4.649e-04, forward_time=0.201, loss_att=72.707, acc=0.864, loss=72.707, backward_time=0.277, grad_norm=35.660, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.712e-04, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:53:13,151 (trainer:732) INFO: 11epoch:train:511-561batch: iter_time=3.362e-04, forward_time=0.201, loss_att=77.222, acc=0.856, loss=77.222, backward_time=0.277, grad_norm=35.146, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.725e-04, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:53:45,850 (trainer:732) INFO: 11epoch:train:562-612batch: iter_time=3.473e-04, forward_time=0.202, loss_att=73.830, acc=0.862, loss=73.830, backward_time=0.280, grad_norm=34.705, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.738e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:54:18,369 (trainer:732) INFO: 11epoch:train:613-663batch: iter_time=3.506e-04, forward_time=0.203, loss_att=81.758, acc=0.855, loss=81.758, backward_time=0.280, grad_norm=38.178, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.750e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:54:50,869 (trainer:732) INFO: 11epoch:train:664-714batch: iter_time=3.275e-04, forward_time=0.202, loss_att=75.756, acc=0.860, loss=75.756, backward_time=0.279, grad_norm=38.059, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.763e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:55:22,883 (trainer:732) INFO: 11epoch:train:715-765batch: iter_time=3.716e-04, forward_time=0.200, loss_att=71.124, acc=0.862, loss=71.124, backward_time=0.275, grad_norm=37.221, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.776e-04, train_time=2.507
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:55:55,654 (trainer:732) INFO: 11epoch:train:766-816batch: iter_time=4.655e-04, forward_time=0.202, loss_att=74.145, acc=0.863, loss=74.145, backward_time=0.279, grad_norm=36.786, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.789e-04, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:56:27,717 (trainer:732) INFO: 11epoch:train:817-867batch: iter_time=3.282e-04, forward_time=0.201, loss_att=68.100, acc=0.866, loss=68.100, backward_time=0.277, grad_norm=37.364, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.802e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:57:00,254 (trainer:732) INFO: 11epoch:train:868-918batch: iter_time=3.745e-04, forward_time=0.201, loss_att=69.975, acc=0.865, loss=69.975, backward_time=0.277, grad_norm=34.568, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.814e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:57:32,648 (trainer:732) INFO: 11epoch:train:919-969batch: iter_time=3.508e-04, forward_time=0.201, loss_att=70.267, acc=0.866, loss=70.267, backward_time=0.277, grad_norm=42.106, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.827e-04, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 19:58:05,914 (trainer:732) INFO: 11epoch:train:970-1020batch: iter_time=3.168e-04, forward_time=0.205, loss_att=74.549, acc=0.864, loss=74.549, backward_time=0.285, grad_norm=38.250, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.840e-04, train_time=2.599
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:08:32,468 (trainer:338) INFO: 11epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=75.213, acc=0.859, loss=75.213, backward_time=0.279, grad_norm=36.833, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.721e-04, train_time=3.141, time=13 minutes and 35.55 seconds, total_count=11407, gpu_max_cached_mem_GB=30.428, [valid] loss_att=88.934, acc=0.842, cer=0.191, wer=0.431, loss=88.934, time=4 minutes and 50.85 seconds, total_count=968, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 23.75 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:08:36,662 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:08:36,670 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/1epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:08:36,670 (trainer:272) INFO: 12/60epoch started. Estimated time to finish: 19 hours, 18 minutes and 48.62 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:11:37,794 (trainer:732) INFO: 12epoch:train:1-51batch: iter_time=0.012, forward_time=0.203, loss_att=69.734, acc=0.868, loss=69.734, backward_time=0.279, grad_norm=36.115, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.856e-04, train_time=14.948
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:12:10,401 (trainer:732) INFO: 12epoch:train:52-102batch: iter_time=3.389e-04, forward_time=0.202, loss_att=72.007, acc=0.867, loss=72.007, backward_time=0.279, grad_norm=34.721, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.869e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:12:42,918 (trainer:732) INFO: 12epoch:train:103-153batch: iter_time=3.425e-04, forward_time=0.201, loss_att=72.912, acc=0.866, loss=72.912, backward_time=0.279, grad_norm=41.254, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.882e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:13:15,424 (trainer:732) INFO: 12epoch:train:154-204batch: iter_time=3.561e-04, forward_time=0.201, loss_att=72.225, acc=0.868, loss=72.225, backward_time=0.278, grad_norm=35.145, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.895e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:13:47,813 (trainer:732) INFO: 12epoch:train:205-255batch: iter_time=3.303e-04, forward_time=0.202, loss_att=69.597, acc=0.869, loss=69.597, backward_time=0.278, grad_norm=32.437, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.907e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:14:20,672 (trainer:732) INFO: 12epoch:train:256-306batch: iter_time=3.404e-04, forward_time=0.203, loss_att=74.526, acc=0.864, loss=74.526, backward_time=0.282, grad_norm=36.280, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=2.920e-04, train_time=2.580
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:14:53,288 (trainer:732) INFO: 12epoch:train:307-357batch: iter_time=3.470e-04, forward_time=0.202, loss_att=73.804, acc=0.866, loss=73.804, backward_time=0.279, grad_norm=34.641, clip=100.000, loss_scale=1.000, optim_step_time=0.067, optim0_lr0=2.933e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:15:25,747 (trainer:732) INFO: 12epoch:train:358-408batch: iter_time=3.091e-04, forward_time=0.201, loss_att=68.535, acc=0.869, loss=68.535, backward_time=0.276, grad_norm=35.081, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.946e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:15:58,309 (trainer:732) INFO: 12epoch:train:409-459batch: iter_time=3.711e-04, forward_time=0.203, loss_att=68.601, acc=0.871, loss=68.601, backward_time=0.281, grad_norm=36.079, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.959e-04, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:16:31,141 (trainer:732) INFO: 12epoch:train:460-510batch: iter_time=3.259e-04, forward_time=0.203, loss_att=69.803, acc=0.871, loss=69.803, backward_time=0.282, grad_norm=34.945, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.971e-04, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:17:03,285 (trainer:732) INFO: 12epoch:train:511-561batch: iter_time=3.268e-04, forward_time=0.201, loss_att=66.475, acc=0.873, loss=66.475, backward_time=0.275, grad_norm=39.511, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.984e-04, train_time=2.517
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:17:35,970 (trainer:732) INFO: 12epoch:train:562-612batch: iter_time=3.456e-04, forward_time=0.203, loss_att=69.004, acc=0.872, loss=69.004, backward_time=0.279, grad_norm=33.712, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.997e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:18:08,477 (trainer:732) INFO: 12epoch:train:613-663batch: iter_time=3.638e-04, forward_time=0.203, loss_att=70.573, acc=0.870, loss=70.573, backward_time=0.281, grad_norm=36.687, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.009e-04, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:18:40,964 (trainer:732) INFO: 12epoch:train:664-714batch: iter_time=3.353e-04, forward_time=0.202, loss_att=67.762, acc=0.871, loss=67.762, backward_time=0.278, grad_norm=31.767, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.022e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:19:13,567 (trainer:732) INFO: 12epoch:train:715-765batch: iter_time=3.221e-04, forward_time=0.202, loss_att=70.311, acc=0.868, loss=70.311, backward_time=0.280, grad_norm=31.416, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.035e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:19:45,855 (trainer:732) INFO: 12epoch:train:766-816batch: iter_time=3.383e-04, forward_time=0.201, loss_att=67.369, acc=0.871, loss=67.369, backward_time=0.276, grad_norm=32.332, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.048e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:20:18,098 (trainer:732) INFO: 12epoch:train:817-867batch: iter_time=3.826e-04, forward_time=0.203, loss_att=68.788, acc=0.869, loss=68.788, backward_time=0.277, grad_norm=33.916, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=3.060e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:20:50,415 (trainer:732) INFO: 12epoch:train:868-918batch: iter_time=3.522e-04, forward_time=0.202, loss_att=65.672, acc=0.873, loss=65.672, backward_time=0.276, grad_norm=32.552, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.073e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:21:23,419 (trainer:732) INFO: 12epoch:train:919-969batch: iter_time=3.729e-04, forward_time=0.206, loss_att=69.584, acc=0.867, loss=69.584, backward_time=0.282, grad_norm=36.576, clip=100.000, loss_scale=1.000, optim_step_time=0.068, optim0_lr0=3.086e-04, train_time=2.588
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:21:55,967 (trainer:732) INFO: 12epoch:train:970-1020batch: iter_time=3.233e-04, forward_time=0.202, loss_att=66.552, acc=0.873, loss=66.552, backward_time=0.276, grad_norm=33.318, clip=100.000, loss_scale=1.000, optim_step_time=0.067, optim0_lr0=3.099e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:32:22,879 (trainer:338) INFO: 12epoch results: [train] iter_time=9.132e-04, forward_time=0.202, loss_att=69.731, acc=0.869, loss=69.731, backward_time=0.279, grad_norm=34.936, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=2.980e-04, train_time=3.126, time=13 minutes and 31.65 seconds, total_count=12444, gpu_max_cached_mem_GB=30.428, [valid] loss_att=86.618, acc=0.847, cer=0.186, wer=0.421, loss=86.618, time=4 minutes and 48.66 seconds, total_count=1056, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 25.9 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:32:27,127 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:32:27,133 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/2epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:32:27,133 (trainer:272) INFO: 13/60epoch started. Estimated time to finish: 18 hours, 55 minutes and 55.71 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:35:30,245 (trainer:732) INFO: 13epoch:train:1-51batch: iter_time=0.015, forward_time=0.202, loss_att=67.600, acc=0.874, loss=67.600, backward_time=0.274, grad_norm=33.707, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.116e-04, train_time=15.117
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:36:02,728 (trainer:732) INFO: 13epoch:train:52-102batch: iter_time=3.276e-04, forward_time=0.202, loss_att=67.506, acc=0.877, loss=67.506, backward_time=0.280, grad_norm=34.216, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.128e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:36:35,332 (trainer:732) INFO: 13epoch:train:103-153batch: iter_time=3.690e-04, forward_time=0.202, loss_att=66.500, acc=0.876, loss=66.500, backward_time=0.278, grad_norm=37.012, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.141e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:37:08,219 (trainer:732) INFO: 13epoch:train:154-204batch: iter_time=3.512e-04, forward_time=0.204, loss_att=66.096, acc=0.877, loss=66.096, backward_time=0.281, grad_norm=36.349, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.154e-04, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:37:40,992 (trainer:732) INFO: 13epoch:train:205-255batch: iter_time=3.436e-04, forward_time=0.204, loss_att=70.022, acc=0.874, loss=70.022, backward_time=0.282, grad_norm=36.870, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.166e-04, train_time=2.588
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:38:13,272 (trainer:732) INFO: 13epoch:train:256-306batch: iter_time=3.330e-04, forward_time=0.201, loss_att=60.381, acc=0.883, loss=60.381, backward_time=0.277, grad_norm=32.467, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.179e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:38:45,653 (trainer:732) INFO: 13epoch:train:307-357batch: iter_time=3.327e-04, forward_time=0.201, loss_att=64.144, acc=0.874, loss=64.144, backward_time=0.277, grad_norm=33.980, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.192e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:39:18,756 (trainer:732) INFO: 13epoch:train:358-408batch: iter_time=3.826e-04, forward_time=0.205, loss_att=67.080, acc=0.875, loss=67.080, backward_time=0.283, grad_norm=33.620, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.205e-04, train_time=2.585
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:39:50,993 (trainer:732) INFO: 13epoch:train:409-459batch: iter_time=3.776e-04, forward_time=0.203, loss_att=64.428, acc=0.878, loss=64.428, backward_time=0.278, grad_norm=33.167, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.217e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:40:24,091 (trainer:732) INFO: 13epoch:train:460-510batch: iter_time=3.685e-04, forward_time=0.206, loss_att=66.313, acc=0.880, loss=66.313, backward_time=0.284, grad_norm=36.302, clip=100.000, loss_scale=1.000, optim_step_time=0.068, optim0_lr0=3.230e-04, train_time=2.599
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:40:56,270 (trainer:732) INFO: 13epoch:train:511-561batch: iter_time=3.643e-04, forward_time=0.200, loss_att=59.360, acc=0.884, loss=59.360, backward_time=0.276, grad_norm=33.332, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=3.243e-04, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:41:28,770 (trainer:732) INFO: 13epoch:train:562-612batch: iter_time=3.532e-04, forward_time=0.202, loss_att=65.189, acc=0.878, loss=65.189, backward_time=0.278, grad_norm=38.995, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.256e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:42:00,924 (trainer:732) INFO: 13epoch:train:613-663batch: iter_time=3.529e-04, forward_time=0.201, loss_att=60.382, acc=0.882, loss=60.382, backward_time=0.275, grad_norm=35.689, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=3.269e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:42:33,476 (trainer:732) INFO: 13epoch:train:664-714batch: iter_time=3.495e-04, forward_time=0.203, loss_att=64.115, acc=0.880, loss=64.115, backward_time=0.280, grad_norm=33.264, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.281e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:43:06,173 (trainer:732) INFO: 13epoch:train:715-765batch: iter_time=3.251e-04, forward_time=0.203, loss_att=65.480, acc=0.876, loss=65.480, backward_time=0.280, grad_norm=33.927, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.294e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:43:38,849 (trainer:732) INFO: 13epoch:train:766-816batch: iter_time=3.531e-04, forward_time=0.203, loss_att=66.070, acc=0.877, loss=66.070, backward_time=0.280, grad_norm=35.331, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.307e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:44:11,094 (trainer:732) INFO: 13epoch:train:817-867batch: iter_time=3.471e-04, forward_time=0.202, loss_att=63.016, acc=0.880, loss=63.016, backward_time=0.277, grad_norm=33.152, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.319e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:44:43,418 (trainer:732) INFO: 13epoch:train:868-918batch: iter_time=3.604e-04, forward_time=0.201, loss_att=62.302, acc=0.879, loss=62.302, backward_time=0.277, grad_norm=31.722, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.332e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:45:15,711 (trainer:732) INFO: 13epoch:train:919-969batch: iter_time=3.024e-04, forward_time=0.201, loss_att=63.164, acc=0.881, loss=63.164, backward_time=0.279, grad_norm=33.230, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.345e-04, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:45:48,898 (trainer:732) INFO: 13epoch:train:970-1020batch: iter_time=3.013e-04, forward_time=0.205, loss_att=68.738, acc=0.874, loss=68.738, backward_time=0.283, grad_norm=38.915, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=3.358e-04, train_time=2.591
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:56:18,459 (trainer:338) INFO: 13epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=64.767, acc=0.878, loss=64.767, backward_time=0.279, grad_norm=34.853, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.239e-04, train_time=3.134, time=13 minutes and 33.88 seconds, total_count=13481, gpu_max_cached_mem_GB=30.428, [valid] loss_att=86.927, acc=0.849, cer=0.190, wer=0.416, loss=86.927, time=4 minutes and 50.74 seconds, total_count=1144, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 26.71 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:56:22,786 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:56:22,792 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/3epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:56:22,793 (trainer:272) INFO: 14/60epoch started. Estimated time to finish: 18 hours, 33 minutes and 12.74 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:59:26,955 (trainer:732) INFO: 14epoch:train:1-51batch: iter_time=0.012, forward_time=0.204, loss_att=63.508, acc=0.883, loss=63.508, backward_time=0.279, grad_norm=37.397, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.374e-04, train_time=15.208
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 20:59:59,497 (trainer:732) INFO: 14epoch:train:52-102batch: iter_time=3.315e-04, forward_time=0.202, loss_att=63.142, acc=0.883, loss=63.142, backward_time=0.279, grad_norm=34.788, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.387e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:00:31,841 (trainer:732) INFO: 14epoch:train:103-153batch: iter_time=3.397e-04, forward_time=0.200, loss_att=60.733, acc=0.884, loss=60.733, backward_time=0.277, grad_norm=35.163, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.400e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:01:04,180 (trainer:732) INFO: 14epoch:train:154-204batch: iter_time=3.268e-04, forward_time=0.201, loss_att=59.671, acc=0.885, loss=59.671, backward_time=0.277, grad_norm=34.425, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.413e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:01:36,618 (trainer:732) INFO: 14epoch:train:205-255batch: iter_time=4.811e-04, forward_time=0.202, loss_att=58.801, acc=0.887, loss=58.801, backward_time=0.280, grad_norm=33.488, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.426e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:02:09,341 (trainer:732) INFO: 14epoch:train:256-306batch: iter_time=3.541e-04, forward_time=0.204, loss_att=62.726, acc=0.885, loss=62.726, backward_time=0.281, grad_norm=33.552, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.438e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:02:41,597 (trainer:732) INFO: 14epoch:train:307-357batch: iter_time=3.384e-04, forward_time=0.201, loss_att=60.154, acc=0.886, loss=60.154, backward_time=0.276, grad_norm=32.799, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.451e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:03:14,468 (trainer:732) INFO: 14epoch:train:358-408batch: iter_time=3.280e-04, forward_time=0.204, loss_att=61.240, acc=0.881, loss=61.240, backward_time=0.281, grad_norm=34.452, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.464e-04, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:03:46,588 (trainer:732) INFO: 14epoch:train:409-459batch: iter_time=2.904e-04, forward_time=0.199, loss_att=59.420, acc=0.888, loss=59.420, backward_time=0.276, grad_norm=33.535, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.476e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:04:19,249 (trainer:732) INFO: 14epoch:train:460-510batch: iter_time=3.185e-04, forward_time=0.203, loss_att=59.544, acc=0.887, loss=59.544, backward_time=0.279, grad_norm=33.211, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.489e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:04:52,099 (trainer:732) INFO: 14epoch:train:511-561batch: iter_time=3.579e-04, forward_time=0.204, loss_att=61.016, acc=0.884, loss=61.016, backward_time=0.281, grad_norm=34.378, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.502e-04, train_time=2.577
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:05:24,477 (trainer:732) INFO: 14epoch:train:562-612batch: iter_time=3.277e-04, forward_time=0.201, loss_att=59.833, acc=0.885, loss=59.833, backward_time=0.277, grad_norm=32.243, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.515e-04, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:05:56,769 (trainer:732) INFO: 14epoch:train:613-663batch: iter_time=3.222e-04, forward_time=0.201, loss_att=61.543, acc=0.886, loss=61.543, backward_time=0.278, grad_norm=30.499, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.527e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:06:29,283 (trainer:732) INFO: 14epoch:train:664-714batch: iter_time=2.944e-04, forward_time=0.201, loss_att=62.880, acc=0.884, loss=62.880, backward_time=0.279, grad_norm=34.527, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.540e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:07:01,792 (trainer:732) INFO: 14epoch:train:715-765batch: iter_time=2.963e-04, forward_time=0.202, loss_att=59.181, acc=0.887, loss=59.181, backward_time=0.279, grad_norm=36.962, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.553e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:07:34,447 (trainer:732) INFO: 14epoch:train:766-816batch: iter_time=3.091e-04, forward_time=0.202, loss_att=61.340, acc=0.886, loss=61.340, backward_time=0.280, grad_norm=37.306, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.566e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:08:06,550 (trainer:732) INFO: 14epoch:train:817-867batch: iter_time=3.399e-04, forward_time=0.200, loss_att=57.958, acc=0.889, loss=57.958, backward_time=0.277, grad_norm=36.187, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.578e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:08:39,194 (trainer:732) INFO: 14epoch:train:868-918batch: iter_time=3.154e-04, forward_time=0.203, loss_att=56.799, acc=0.891, loss=56.799, backward_time=0.281, grad_norm=32.235, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.591e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:09:11,618 (trainer:732) INFO: 14epoch:train:919-969batch: iter_time=3.424e-04, forward_time=0.201, loss_att=58.579, acc=0.888, loss=58.579, backward_time=0.277, grad_norm=34.902, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.604e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:09:44,284 (trainer:732) INFO: 14epoch:train:970-1020batch: iter_time=2.777e-04, forward_time=0.202, loss_att=61.263, acc=0.885, loss=61.263, backward_time=0.281, grad_norm=36.438, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.617e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:20:20,233 (trainer:338) INFO: 14epoch results: [train] iter_time=8.946e-04, forward_time=0.202, loss_att=60.520, acc=0.886, loss=60.520, backward_time=0.279, grad_norm=34.459, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.498e-04, train_time=3.134, time=13 minutes and 33.5 seconds, total_count=14518, gpu_max_cached_mem_GB=30.428, [valid] loss_att=83.871, acc=0.854, cer=0.182, wer=0.409, loss=83.871, time=4 minutes and 51.58 seconds, total_count=1232, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.36 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:20:24,317 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:20:24,323 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/4epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:20:24,323 (trainer:272) INFO: 15/60epoch started. Estimated time to finish: 18 hours, 10 minutes and 38.67 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:23:25,860 (trainer:732) INFO: 15epoch:train:1-51batch: iter_time=0.015, forward_time=0.202, loss_att=57.582, acc=0.891, loss=57.582, backward_time=0.276, grad_norm=32.630, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.633e-04, train_time=14.980
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:23:58,813 (trainer:732) INFO: 15epoch:train:52-102batch: iter_time=3.283e-04, forward_time=0.203, loss_att=54.847, acc=0.895, loss=54.847, backward_time=0.281, grad_norm=31.900, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.646e-04, train_time=2.581
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:24:31,123 (trainer:732) INFO: 15epoch:train:103-153batch: iter_time=3.273e-04, forward_time=0.200, loss_att=56.853, acc=0.892, loss=56.853, backward_time=0.278, grad_norm=31.834, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=3.659e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:25:03,401 (trainer:732) INFO: 15epoch:train:154-204batch: iter_time=3.125e-04, forward_time=0.200, loss_att=58.223, acc=0.890, loss=58.223, backward_time=0.277, grad_norm=31.934, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.672e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:25:35,695 (trainer:732) INFO: 15epoch:train:205-255batch: iter_time=3.306e-04, forward_time=0.201, loss_att=56.087, acc=0.894, loss=56.087, backward_time=0.279, grad_norm=32.066, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.684e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:26:07,772 (trainer:732) INFO: 15epoch:train:256-306batch: iter_time=2.941e-04, forward_time=0.200, loss_att=55.855, acc=0.889, loss=55.855, backward_time=0.275, grad_norm=31.700, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.697e-04, train_time=2.511
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:26:39,939 (trainer:732) INFO: 15epoch:train:307-357batch: iter_time=3.276e-04, forward_time=0.200, loss_att=54.918, acc=0.895, loss=54.918, backward_time=0.276, grad_norm=35.028, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.710e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:27:12,129 (trainer:732) INFO: 15epoch:train:358-408batch: iter_time=3.033e-04, forward_time=0.199, loss_att=55.796, acc=0.891, loss=55.796, backward_time=0.275, grad_norm=35.801, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.723e-04, train_time=2.513
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:27:44,277 (trainer:732) INFO: 15epoch:train:409-459batch: iter_time=3.231e-04, forward_time=0.201, loss_att=57.215, acc=0.892, loss=57.215, backward_time=0.278, grad_norm=34.833, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.735e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:28:17,119 (trainer:732) INFO: 15epoch:train:460-510batch: iter_time=3.409e-04, forward_time=0.204, loss_att=57.927, acc=0.893, loss=57.927, backward_time=0.282, grad_norm=33.754, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=3.748e-04, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:28:49,997 (trainer:732) INFO: 15epoch:train:511-561batch: iter_time=3.262e-04, forward_time=0.204, loss_att=58.062, acc=0.892, loss=58.062, backward_time=0.282, grad_norm=33.749, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.761e-04, train_time=2.583
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:29:23,056 (trainer:732) INFO: 15epoch:train:562-612batch: iter_time=3.232e-04, forward_time=0.204, loss_att=58.255, acc=0.893, loss=58.255, backward_time=0.283, grad_norm=34.437, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.774e-04, train_time=2.581
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:29:55,484 (trainer:732) INFO: 15epoch:train:613-663batch: iter_time=3.379e-04, forward_time=0.203, loss_att=58.766, acc=0.890, loss=58.766, backward_time=0.280, grad_norm=34.272, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.786e-04, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:30:27,975 (trainer:732) INFO: 15epoch:train:664-714batch: iter_time=2.923e-04, forward_time=0.202, loss_att=56.746, acc=0.892, loss=56.746, backward_time=0.278, grad_norm=32.845, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.799e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:31:00,663 (trainer:732) INFO: 15epoch:train:715-765batch: iter_time=2.640e-04, forward_time=0.203, loss_att=60.600, acc=0.890, loss=60.600, backward_time=0.280, grad_norm=32.893, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.812e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:31:33,141 (trainer:732) INFO: 15epoch:train:766-816batch: iter_time=3.463e-04, forward_time=0.201, loss_att=55.392, acc=0.893, loss=55.392, backward_time=0.278, grad_norm=33.204, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.825e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:32:05,414 (trainer:732) INFO: 15epoch:train:817-867batch: iter_time=3.136e-04, forward_time=0.201, loss_att=55.835, acc=0.895, loss=55.835, backward_time=0.278, grad_norm=37.716, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.837e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:32:37,838 (trainer:732) INFO: 15epoch:train:868-918batch: iter_time=3.472e-04, forward_time=0.201, loss_att=57.624, acc=0.893, loss=57.624, backward_time=0.277, grad_norm=37.002, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.850e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:33:10,195 (trainer:732) INFO: 15epoch:train:919-969batch: iter_time=3.367e-04, forward_time=0.201, loss_att=54.635, acc=0.894, loss=54.635, backward_time=0.277, grad_norm=33.021, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.863e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:33:43,226 (trainer:732) INFO: 15epoch:train:970-1020batch: iter_time=2.995e-04, forward_time=0.204, loss_att=53.622, acc=0.898, loss=53.622, backward_time=0.283, grad_norm=33.961, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.876e-04, train_time=2.580
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:44:12,415 (trainer:338) INFO: 15epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=56.734, acc=0.893, loss=56.734, backward_time=0.279, grad_norm=33.698, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.757e-04, train_time=3.124, time=13 minutes and 31.05 seconds, total_count=15555, gpu_max_cached_mem_GB=30.428, [valid] loss_att=83.043, acc=0.856, cer=0.181, wer=0.403, loss=83.043, time=4 minutes and 49.71 seconds, total_count=1320, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 27.33 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:44:16,728 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:44:16,735 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/5epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:44:16,735 (trainer:272) INFO: 16/60epoch started. Estimated time to finish: 17 hours, 47 minutes and 25.59 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:47:19,822 (trainer:732) INFO: 16epoch:train:1-51batch: iter_time=0.012, forward_time=0.206, loss_att=54.121, acc=0.899, loss=54.121, backward_time=0.281, grad_norm=32.606, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=3.893e-04, train_time=15.106
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:47:52,457 (trainer:732) INFO: 16epoch:train:52-102batch: iter_time=3.221e-04, forward_time=0.203, loss_att=55.153, acc=0.896, loss=55.153, backward_time=0.280, grad_norm=33.800, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.905e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:48:24,828 (trainer:732) INFO: 16epoch:train:103-153batch: iter_time=3.587e-04, forward_time=0.201, loss_att=53.162, acc=0.898, loss=53.162, backward_time=0.277, grad_norm=31.801, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=3.918e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:48:57,072 (trainer:732) INFO: 16epoch:train:154-204batch: iter_time=3.289e-04, forward_time=0.200, loss_att=52.192, acc=0.899, loss=52.192, backward_time=0.275, grad_norm=30.180, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.931e-04, train_time=2.519
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:49:29,368 (trainer:732) INFO: 16epoch:train:205-255batch: iter_time=3.359e-04, forward_time=0.201, loss_att=54.472, acc=0.898, loss=54.472, backward_time=0.278, grad_norm=32.052, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.943e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:50:02,333 (trainer:732) INFO: 16epoch:train:256-306batch: iter_time=3.513e-04, forward_time=0.205, loss_att=51.913, acc=0.900, loss=51.913, backward_time=0.283, grad_norm=32.470, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.956e-04, train_time=2.593
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:50:35,070 (trainer:732) INFO: 16epoch:train:307-357batch: iter_time=3.284e-04, forward_time=0.204, loss_att=55.469, acc=0.896, loss=55.469, backward_time=0.280, grad_norm=34.193, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=3.969e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:51:07,357 (trainer:732) INFO: 16epoch:train:358-408batch: iter_time=3.356e-04, forward_time=0.200, loss_att=53.766, acc=0.897, loss=53.766, backward_time=0.275, grad_norm=31.634, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.982e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:51:39,436 (trainer:732) INFO: 16epoch:train:409-459batch: iter_time=3.621e-04, forward_time=0.201, loss_att=51.944, acc=0.900, loss=51.944, backward_time=0.277, grad_norm=30.887, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=3.994e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:52:12,056 (trainer:732) INFO: 16epoch:train:460-510batch: iter_time=3.228e-04, forward_time=0.201, loss_att=54.596, acc=0.899, loss=54.596, backward_time=0.280, grad_norm=32.831, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.007e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:52:44,711 (trainer:732) INFO: 16epoch:train:511-561batch: iter_time=3.436e-04, forward_time=0.203, loss_att=52.133, acc=0.901, loss=52.133, backward_time=0.279, grad_norm=32.438, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.020e-04, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:53:17,182 (trainer:732) INFO: 16epoch:train:562-612batch: iter_time=3.107e-04, forward_time=0.202, loss_att=52.030, acc=0.898, loss=52.030, backward_time=0.278, grad_norm=31.864, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.033e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:53:49,594 (trainer:732) INFO: 16epoch:train:613-663batch: iter_time=3.391e-04, forward_time=0.202, loss_att=55.419, acc=0.896, loss=55.419, backward_time=0.279, grad_norm=33.594, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.045e-04, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:54:21,810 (trainer:732) INFO: 16epoch:train:664-714batch: iter_time=3.284e-04, forward_time=0.201, loss_att=50.592, acc=0.901, loss=50.592, backward_time=0.276, grad_norm=31.349, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.058e-04, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:54:54,508 (trainer:732) INFO: 16epoch:train:715-765batch: iter_time=2.982e-04, forward_time=0.202, loss_att=52.568, acc=0.903, loss=52.568, backward_time=0.281, grad_norm=31.340, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.071e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:55:27,237 (trainer:732) INFO: 16epoch:train:766-816batch: iter_time=3.356e-04, forward_time=0.202, loss_att=53.183, acc=0.900, loss=53.183, backward_time=0.280, grad_norm=35.396, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.084e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:55:59,672 (trainer:732) INFO: 16epoch:train:817-867batch: iter_time=3.277e-04, forward_time=0.203, loss_att=53.578, acc=0.896, loss=53.578, backward_time=0.280, grad_norm=39.374, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=4.096e-04, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:56:32,370 (trainer:732) INFO: 16epoch:train:868-918batch: iter_time=3.532e-04, forward_time=0.204, loss_att=53.921, acc=0.900, loss=53.921, backward_time=0.280, grad_norm=33.809, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.109e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:57:05,126 (trainer:732) INFO: 16epoch:train:919-969batch: iter_time=3.463e-04, forward_time=0.203, loss_att=55.074, acc=0.898, loss=55.074, backward_time=0.279, grad_norm=33.730, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.122e-04, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 21:57:37,638 (trainer:732) INFO: 16epoch:train:970-1020batch: iter_time=3.144e-04, forward_time=0.201, loss_att=50.229, acc=0.903, loss=50.229, backward_time=0.277, grad_norm=32.204, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.135e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:08:10,915 (trainer:338) INFO: 16epoch results: [train] iter_time=8.866e-04, forward_time=0.202, loss_att=53.285, acc=0.899, loss=53.285, backward_time=0.279, grad_norm=32.853, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.016e-04, train_time=3.131, time=13 minutes and 32.93 seconds, total_count=16592, gpu_max_cached_mem_GB=30.428, [valid] loss_att=80.718, acc=0.859, cer=0.170, wer=0.403, loss=80.718, time=4 minutes and 50.8 seconds, total_count=1408, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 30.45 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:08:15,495 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:08:15,515 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/6epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:08:15,516 (trainer:272) INFO: 17/60epoch started. Estimated time to finish: 17 hours, 24 minutes and 25.1 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:11:18,294 (trainer:732) INFO: 17epoch:train:1-51batch: iter_time=0.015, forward_time=0.204, loss_att=50.091, acc=0.904, loss=50.091, backward_time=0.278, grad_norm=30.556, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.151e-04, train_time=15.088
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:11:50,875 (trainer:732) INFO: 17epoch:train:52-102batch: iter_time=3.471e-04, forward_time=0.203, loss_att=52.163, acc=0.903, loss=52.163, backward_time=0.279, grad_norm=30.775, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=4.164e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:12:23,124 (trainer:732) INFO: 17epoch:train:103-153batch: iter_time=3.154e-04, forward_time=0.201, loss_att=49.338, acc=0.904, loss=49.338, backward_time=0.276, grad_norm=32.952, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.177e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:12:55,100 (trainer:732) INFO: 17epoch:train:154-204batch: iter_time=3.717e-04, forward_time=0.198, loss_att=48.447, acc=0.905, loss=48.447, backward_time=0.273, grad_norm=28.737, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.190e-04, train_time=2.498
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:13:27,150 (trainer:732) INFO: 17epoch:train:205-255batch: iter_time=3.456e-04, forward_time=0.201, loss_att=51.006, acc=0.902, loss=51.006, backward_time=0.278, grad_norm=32.017, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.202e-04, train_time=2.520
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:13:59,710 (trainer:732) INFO: 17epoch:train:256-306batch: iter_time=3.029e-04, forward_time=0.203, loss_att=51.063, acc=0.903, loss=51.063, backward_time=0.279, grad_norm=35.004, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=4.215e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:14:32,174 (trainer:732) INFO: 17epoch:train:307-357batch: iter_time=3.258e-04, forward_time=0.202, loss_att=50.604, acc=0.901, loss=50.604, backward_time=0.278, grad_norm=33.472, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.228e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:15:05,254 (trainer:732) INFO: 17epoch:train:358-408batch: iter_time=3.132e-04, forward_time=0.204, loss_att=53.676, acc=0.902, loss=53.676, backward_time=0.284, grad_norm=32.918, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.241e-04, train_time=2.586
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:15:37,891 (trainer:732) INFO: 17epoch:train:409-459batch: iter_time=3.308e-04, forward_time=0.204, loss_att=52.963, acc=0.904, loss=52.963, backward_time=0.283, grad_norm=32.809, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.253e-04, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:16:10,219 (trainer:732) INFO: 17epoch:train:460-510batch: iter_time=3.254e-04, forward_time=0.201, loss_att=49.322, acc=0.905, loss=49.322, backward_time=0.276, grad_norm=29.753, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.266e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:16:42,881 (trainer:732) INFO: 17epoch:train:511-561batch: iter_time=3.605e-04, forward_time=0.204, loss_att=51.635, acc=0.904, loss=51.635, backward_time=0.280, grad_norm=34.157, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.279e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:17:15,502 (trainer:732) INFO: 17epoch:train:562-612batch: iter_time=3.433e-04, forward_time=0.202, loss_att=48.812, acc=0.904, loss=48.812, backward_time=0.279, grad_norm=31.933, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.292e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:17:47,991 (trainer:732) INFO: 17epoch:train:613-663batch: iter_time=3.411e-04, forward_time=0.204, loss_att=54.969, acc=0.899, loss=54.969, backward_time=0.281, grad_norm=33.820, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.304e-04, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:18:20,260 (trainer:732) INFO: 17epoch:train:664-714batch: iter_time=3.498e-04, forward_time=0.200, loss_att=49.149, acc=0.906, loss=49.149, backward_time=0.275, grad_norm=30.580, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.317e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:18:52,506 (trainer:732) INFO: 17epoch:train:715-765batch: iter_time=3.227e-04, forward_time=0.200, loss_att=48.299, acc=0.908, loss=48.299, backward_time=0.278, grad_norm=29.956, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.330e-04, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:19:24,771 (trainer:732) INFO: 17epoch:train:766-816batch: iter_time=3.448e-04, forward_time=0.200, loss_att=47.928, acc=0.907, loss=47.928, backward_time=0.276, grad_norm=31.215, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.343e-04, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:19:57,367 (trainer:732) INFO: 17epoch:train:817-867batch: iter_time=3.386e-04, forward_time=0.204, loss_att=51.147, acc=0.904, loss=51.147, backward_time=0.282, grad_norm=33.277, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.355e-04, train_time=2.568
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:20:29,855 (trainer:732) INFO: 17epoch:train:868-918batch: iter_time=3.433e-04, forward_time=0.202, loss_att=49.075, acc=0.907, loss=49.075, backward_time=0.278, grad_norm=31.733, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.368e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:21:02,368 (trainer:732) INFO: 17epoch:train:919-969batch: iter_time=3.476e-04, forward_time=0.203, loss_att=48.726, acc=0.909, loss=48.726, backward_time=0.280, grad_norm=32.976, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.381e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:21:35,176 (trainer:732) INFO: 17epoch:train:970-1020batch: iter_time=3.138e-04, forward_time=0.203, loss_att=47.483, acc=0.907, loss=47.483, backward_time=0.281, grad_norm=35.510, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.394e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:32:12,649 (trainer:338) INFO: 17epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=50.249, acc=0.904, loss=50.249, backward_time=0.279, grad_norm=32.235, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.275e-04, train_time=3.127, time=13 minutes and 31.95 seconds, total_count=17629, gpu_max_cached_mem_GB=30.428, [valid] loss_att=82.015, acc=0.861, cer=0.172, wer=0.396, loss=82.015, time=4 minutes and 51.46 seconds, total_count=1496, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 33.73 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:32:17,161 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:32:17,169 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/7epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:32:17,169 (trainer:272) INFO: 18/60epoch started. Estimated time to finish: 17 hours, 1 minute and 25.03 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:35:20,593 (trainer:732) INFO: 18epoch:train:1-51batch: iter_time=0.011, forward_time=0.204, loss_att=45.031, acc=0.912, loss=45.031, backward_time=0.276, grad_norm=32.627, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.410e-04, train_time=15.144
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:35:53,025 (trainer:732) INFO: 18epoch:train:52-102batch: iter_time=3.806e-04, forward_time=0.202, loss_att=47.623, acc=0.910, loss=47.623, backward_time=0.278, grad_norm=31.979, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.423e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:36:26,390 (trainer:732) INFO: 18epoch:train:103-153batch: iter_time=3.665e-04, forward_time=0.206, loss_att=49.687, acc=0.907, loss=49.687, backward_time=0.286, grad_norm=36.442, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=4.436e-04, train_time=2.616
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:36:58,821 (trainer:732) INFO: 18epoch:train:154-204batch: iter_time=3.399e-04, forward_time=0.200, loss_att=46.237, acc=0.910, loss=46.237, backward_time=0.275, grad_norm=28.404, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.449e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:37:31,380 (trainer:732) INFO: 18epoch:train:205-255batch: iter_time=3.358e-04, forward_time=0.203, loss_att=48.449, acc=0.908, loss=48.449, backward_time=0.280, grad_norm=31.394, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.461e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:38:04,082 (trainer:732) INFO: 18epoch:train:256-306batch: iter_time=3.281e-04, forward_time=0.202, loss_att=49.757, acc=0.909, loss=49.757, backward_time=0.281, grad_norm=34.275, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=4.474e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:38:36,478 (trainer:732) INFO: 18epoch:train:307-357batch: iter_time=3.289e-04, forward_time=0.201, loss_att=47.823, acc=0.909, loss=47.823, backward_time=0.278, grad_norm=33.341, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.487e-04, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:39:09,277 (trainer:732) INFO: 18epoch:train:358-408batch: iter_time=3.506e-04, forward_time=0.204, loss_att=46.927, acc=0.911, loss=46.927, backward_time=0.281, grad_norm=31.498, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.500e-04, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:39:41,436 (trainer:732) INFO: 18epoch:train:409-459batch: iter_time=3.676e-04, forward_time=0.201, loss_att=49.095, acc=0.908, loss=49.095, backward_time=0.278, grad_norm=30.351, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.512e-04, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:40:13,854 (trainer:732) INFO: 18epoch:train:460-510batch: iter_time=3.303e-04, forward_time=0.201, loss_att=46.187, acc=0.913, loss=46.187, backward_time=0.279, grad_norm=32.284, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.525e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:40:46,297 (trainer:732) INFO: 18epoch:train:511-561batch: iter_time=3.263e-04, forward_time=0.201, loss_att=48.632, acc=0.909, loss=48.632, backward_time=0.279, grad_norm=32.009, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.538e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:41:18,973 (trainer:732) INFO: 18epoch:train:562-612batch: iter_time=3.092e-04, forward_time=0.202, loss_att=49.610, acc=0.909, loss=49.610, backward_time=0.280, grad_norm=31.637, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.551e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:41:50,963 (trainer:732) INFO: 18epoch:train:613-663batch: iter_time=3.390e-04, forward_time=0.200, loss_att=45.146, acc=0.912, loss=45.146, backward_time=0.276, grad_norm=29.376, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.563e-04, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:42:23,661 (trainer:732) INFO: 18epoch:train:664-714batch: iter_time=3.178e-04, forward_time=0.203, loss_att=45.951, acc=0.914, loss=45.951, backward_time=0.281, grad_norm=33.370, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.576e-04, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:42:56,198 (trainer:732) INFO: 18epoch:train:715-765batch: iter_time=3.168e-04, forward_time=0.202, loss_att=50.355, acc=0.907, loss=50.355, backward_time=0.280, grad_norm=32.146, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.589e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:43:28,324 (trainer:732) INFO: 18epoch:train:766-816batch: iter_time=3.582e-04, forward_time=0.199, loss_att=46.358, acc=0.910, loss=46.358, backward_time=0.274, grad_norm=32.307, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=4.602e-04, train_time=2.512
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:44:00,858 (trainer:732) INFO: 18epoch:train:817-867batch: iter_time=3.130e-04, forward_time=0.203, loss_att=47.896, acc=0.910, loss=47.896, backward_time=0.281, grad_norm=33.022, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.614e-04, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:44:33,419 (trainer:732) INFO: 18epoch:train:868-918batch: iter_time=3.583e-04, forward_time=0.202, loss_att=45.936, acc=0.909, loss=45.936, backward_time=0.279, grad_norm=37.003, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.627e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:45:05,967 (trainer:732) INFO: 18epoch:train:919-969batch: iter_time=2.883e-04, forward_time=0.201, loss_att=46.279, acc=0.911, loss=46.279, backward_time=0.278, grad_norm=35.753, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.640e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:45:37,947 (trainer:732) INFO: 18epoch:train:970-1020batch: iter_time=3.010e-04, forward_time=0.199, loss_att=45.739, acc=0.910, loss=45.739, backward_time=0.274, grad_norm=31.543, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.653e-04, train_time=2.499
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:56:11,272 (trainer:338) INFO: 18epoch results: [train] iter_time=8.679e-04, forward_time=0.202, loss_att=47.366, acc=0.910, loss=47.366, backward_time=0.279, grad_norm=32.608, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.534e-04, train_time=3.130, time=13 minutes and 32.76 seconds, total_count=18666, gpu_max_cached_mem_GB=30.428, [valid] loss_att=78.023, acc=0.865, cer=0.167, wer=0.393, loss=78.023, time=4 minutes and 50.26 seconds, total_count=1584, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 31.08 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:56:15,799 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:56:15,806 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/8epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:56:15,806 (trainer:272) INFO: 19/60epoch started. Estimated time to finish: 16 hours, 38 minutes and 11.07 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:59:16,079 (trainer:732) INFO: 19epoch:train:1-51batch: iter_time=0.010, forward_time=0.203, loss_att=42.603, acc=0.917, loss=42.603, backward_time=0.278, grad_norm=31.260, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.669e-04, train_time=14.879
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 22:59:48,156 (trainer:732) INFO: 19epoch:train:52-102batch: iter_time=3.311e-04, forward_time=0.200, loss_att=42.401, acc=0.917, loss=42.401, backward_time=0.275, grad_norm=29.594, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.682e-04, train_time=2.513
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:00:20,441 (trainer:732) INFO: 19epoch:train:103-153batch: iter_time=3.178e-04, forward_time=0.201, loss_att=44.699, acc=0.914, loss=44.699, backward_time=0.277, grad_norm=32.320, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.695e-04, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:00:52,910 (trainer:732) INFO: 19epoch:train:154-204batch: iter_time=3.189e-04, forward_time=0.201, loss_att=45.212, acc=0.915, loss=45.212, backward_time=0.279, grad_norm=33.952, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.708e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:01:24,899 (trainer:732) INFO: 19epoch:train:205-255batch: iter_time=3.149e-04, forward_time=0.199, loss_att=43.771, acc=0.916, loss=43.771, backward_time=0.277, grad_norm=33.328, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.720e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:01:56,899 (trainer:732) INFO: 19epoch:train:256-306batch: iter_time=2.980e-04, forward_time=0.199, loss_att=43.591, acc=0.913, loss=43.591, backward_time=0.276, grad_norm=32.228, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.733e-04, train_time=2.500
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:02:29,559 (trainer:732) INFO: 19epoch:train:307-357batch: iter_time=3.282e-04, forward_time=0.203, loss_att=43.594, acc=0.915, loss=43.594, backward_time=0.281, grad_norm=30.933, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=4.746e-04, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:03:01,929 (trainer:732) INFO: 19epoch:train:358-408batch: iter_time=4.037e-04, forward_time=0.201, loss_att=44.164, acc=0.915, loss=44.164, backward_time=0.278, grad_norm=31.211, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.759e-04, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:03:34,652 (trainer:732) INFO: 19epoch:train:409-459batch: iter_time=3.098e-04, forward_time=0.204, loss_att=49.603, acc=0.911, loss=49.603, backward_time=0.284, grad_norm=34.807, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.771e-04, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:04:07,523 (trainer:732) INFO: 19epoch:train:460-510batch: iter_time=3.361e-04, forward_time=0.203, loss_att=46.562, acc=0.915, loss=46.562, backward_time=0.282, grad_norm=33.248, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.784e-04, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:04:40,029 (trainer:732) INFO: 19epoch:train:511-561batch: iter_time=3.226e-04, forward_time=0.202, loss_att=45.667, acc=0.915, loss=45.667, backward_time=0.280, grad_norm=32.620, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.797e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:05:12,839 (trainer:732) INFO: 19epoch:train:562-612batch: iter_time=3.651e-04, forward_time=0.203, loss_att=45.015, acc=0.915, loss=45.015, backward_time=0.281, grad_norm=32.878, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.810e-04, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:05:45,821 (trainer:732) INFO: 19epoch:train:613-663batch: iter_time=3.588e-04, forward_time=0.205, loss_att=47.825, acc=0.914, loss=47.825, backward_time=0.286, grad_norm=33.120, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.822e-04, train_time=2.597
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:06:18,606 (trainer:732) INFO: 19epoch:train:664-714batch: iter_time=3.485e-04, forward_time=0.203, loss_att=46.205, acc=0.913, loss=46.205, backward_time=0.282, grad_norm=33.837, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.835e-04, train_time=2.572
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:06:50,734 (trainer:732) INFO: 19epoch:train:715-765batch: iter_time=3.517e-04, forward_time=0.200, loss_att=42.704, acc=0.917, loss=42.704, backward_time=0.277, grad_norm=30.370, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.848e-04, train_time=2.520
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:07:23,357 (trainer:732) INFO: 19epoch:train:766-816batch: iter_time=3.489e-04, forward_time=0.202, loss_att=45.244, acc=0.915, loss=45.244, backward_time=0.280, grad_norm=33.106, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.861e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:07:55,296 (trainer:732) INFO: 19epoch:train:817-867batch: iter_time=3.424e-04, forward_time=0.199, loss_att=42.573, acc=0.913, loss=42.573, backward_time=0.277, grad_norm=33.646, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.873e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:08:28,165 (trainer:732) INFO: 19epoch:train:868-918batch: iter_time=3.273e-04, forward_time=0.203, loss_att=45.198, acc=0.916, loss=45.198, backward_time=0.280, grad_norm=37.781, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.886e-04, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:09:00,394 (trainer:732) INFO: 19epoch:train:919-969batch: iter_time=3.629e-04, forward_time=0.202, loss_att=44.479, acc=0.916, loss=44.479, backward_time=0.277, grad_norm=29.626, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.899e-04, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:09:32,979 (trainer:732) INFO: 19epoch:train:970-1020batch: iter_time=3.107e-04, forward_time=0.202, loss_att=42.854, acc=0.919, loss=42.854, backward_time=0.278, grad_norm=31.181, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.912e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:20:10,985 (trainer:338) INFO: 19epoch results: [train] iter_time=7.944e-04, forward_time=0.202, loss_att=44.642, acc=0.915, loss=44.642, backward_time=0.279, grad_norm=32.541, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.793e-04, train_time=3.117, time=13 minutes and 29.21 seconds, total_count=19703, gpu_max_cached_mem_GB=30.428, [valid] loss_att=78.736, acc=0.866, cer=0.166, wer=0.391, loss=78.736, time=4 minutes and 53.45 seconds, total_count=1672, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.52 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:20:15,632 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:20:15,640 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/9epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:20:15,640 (trainer:272) INFO: 20/60epoch started. Estimated time to finish: 16 hours, 14 minutes and 54.99 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:23:18,728 (trainer:732) INFO: 20epoch:train:1-51batch: iter_time=0.010, forward_time=0.203, loss_att=42.570, acc=0.919, loss=42.570, backward_time=0.278, grad_norm=31.248, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.928e-04, train_time=15.105
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:23:51,149 (trainer:732) INFO: 20epoch:train:52-102batch: iter_time=3.412e-04, forward_time=0.201, loss_att=42.262, acc=0.920, loss=42.262, backward_time=0.277, grad_norm=31.851, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.941e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:24:23,741 (trainer:732) INFO: 20epoch:train:103-153batch: iter_time=3.707e-04, forward_time=0.202, loss_att=42.594, acc=0.919, loss=42.594, backward_time=0.281, grad_norm=31.911, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.954e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:24:56,259 (trainer:732) INFO: 20epoch:train:154-204batch: iter_time=3.462e-04, forward_time=0.201, loss_att=42.399, acc=0.920, loss=42.399, backward_time=0.278, grad_norm=30.765, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.967e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:25:28,555 (trainer:732) INFO: 20epoch:train:205-255batch: iter_time=3.511e-04, forward_time=0.201, loss_att=43.769, acc=0.919, loss=43.769, backward_time=0.279, grad_norm=32.940, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=4.979e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:26:00,734 (trainer:732) INFO: 20epoch:train:256-306batch: iter_time=3.220e-04, forward_time=0.200, loss_att=39.554, acc=0.923, loss=39.554, backward_time=0.277, grad_norm=29.917, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.992e-04, train_time=2.521
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:26:33,549 (trainer:732) INFO: 20epoch:train:307-357batch: iter_time=3.623e-04, forward_time=0.203, loss_att=46.007, acc=0.916, loss=46.007, backward_time=0.283, grad_norm=32.880, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.005e-04, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:27:05,983 (trainer:732) INFO: 20epoch:train:358-408batch: iter_time=3.447e-04, forward_time=0.201, loss_att=43.430, acc=0.918, loss=43.430, backward_time=0.278, grad_norm=31.327, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.018e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:27:38,428 (trainer:732) INFO: 20epoch:train:409-459batch: iter_time=3.558e-04, forward_time=0.202, loss_att=40.701, acc=0.922, loss=40.701, backward_time=0.279, grad_norm=29.741, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=5.031e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:28:10,908 (trainer:732) INFO: 20epoch:train:460-510batch: iter_time=3.651e-04, forward_time=0.202, loss_att=42.208, acc=0.917, loss=42.208, backward_time=0.279, grad_norm=32.102, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.043e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:28:43,716 (trainer:732) INFO: 20epoch:train:511-561batch: iter_time=3.532e-04, forward_time=0.203, loss_att=43.517, acc=0.918, loss=43.517, backward_time=0.281, grad_norm=35.862, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.056e-04, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:29:16,202 (trainer:732) INFO: 20epoch:train:562-612batch: iter_time=3.421e-04, forward_time=0.201, loss_att=42.861, acc=0.919, loss=42.861, backward_time=0.277, grad_norm=31.765, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.069e-04, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:29:48,215 (trainer:732) INFO: 20epoch:train:613-663batch: iter_time=3.458e-04, forward_time=0.200, loss_att=40.573, acc=0.920, loss=40.573, backward_time=0.276, grad_norm=31.684, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.082e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:30:20,776 (trainer:732) INFO: 20epoch:train:664-714batch: iter_time=3.392e-04, forward_time=0.202, loss_att=41.876, acc=0.919, loss=41.876, backward_time=0.278, grad_norm=31.114, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.094e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:30:53,523 (trainer:732) INFO: 20epoch:train:715-765batch: iter_time=3.179e-04, forward_time=0.203, loss_att=43.254, acc=0.920, loss=43.254, backward_time=0.282, grad_norm=31.657, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.107e-04, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:31:26,451 (trainer:732) INFO: 20epoch:train:766-816batch: iter_time=3.871e-04, forward_time=0.204, loss_att=43.303, acc=0.919, loss=43.303, backward_time=0.282, grad_norm=33.887, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.120e-04, train_time=2.576
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:31:59,064 (trainer:732) INFO: 20epoch:train:817-867batch: iter_time=3.368e-04, forward_time=0.203, loss_att=42.731, acc=0.922, loss=42.731, backward_time=0.282, grad_norm=31.071, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.132e-04, train_time=2.568
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:32:31,707 (trainer:732) INFO: 20epoch:train:868-918batch: iter_time=3.846e-04, forward_time=0.203, loss_att=43.755, acc=0.918, loss=43.755, backward_time=0.280, grad_norm=35.786, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=5.145e-04, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:33:04,012 (trainer:732) INFO: 20epoch:train:919-969batch: iter_time=3.453e-04, forward_time=0.201, loss_att=42.769, acc=0.918, loss=42.769, backward_time=0.277, grad_norm=35.435, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.158e-04, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:33:36,117 (trainer:732) INFO: 20epoch:train:970-1020batch: iter_time=2.963e-04, forward_time=0.199, loss_att=38.580, acc=0.923, loss=38.580, backward_time=0.274, grad_norm=30.067, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.171e-04, train_time=2.508
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:44:05,700 (trainer:338) INFO: 20epoch results: [train] iter_time=8.054e-04, forward_time=0.202, loss_att=42.275, acc=0.920, loss=42.275, backward_time=0.279, grad_norm=32.099, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.052e-04, train_time=3.129, time=13 minutes and 32.46 seconds, total_count=20740, gpu_max_cached_mem_GB=30.428, [valid] loss_att=77.316, acc=0.867, cer=0.167, wer=0.392, loss=77.316, time=4 minutes and 51.62 seconds, total_count=1760, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 25.98 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:44:09,107 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:44:09,113 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/10epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:44:09,114 (trainer:272) INFO: 21/60epoch started. Estimated time to finish: 15 hours, 51 minutes and 21.82 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:46:46,054 (trainer:732) INFO: 21epoch:train:1-51batch: iter_time=0.007, forward_time=0.197, loss_att=39.861, acc=0.924, loss=39.861, backward_time=0.273, grad_norm=29.718, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=5.188e-04, train_time=12.929
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:47:18,160 (trainer:732) INFO: 21epoch:train:52-102batch: iter_time=2.969e-04, forward_time=0.198, loss_att=42.287, acc=0.922, loss=42.287, backward_time=0.277, grad_norm=31.373, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=5.200e-04, train_time=2.518
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:47:50,269 (trainer:732) INFO: 21epoch:train:103-153batch: iter_time=3.062e-04, forward_time=0.199, loss_att=41.197, acc=0.923, loss=41.197, backward_time=0.278, grad_norm=30.630, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=5.213e-04, train_time=2.517
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:48:22,651 (trainer:732) INFO: 21epoch:train:154-204batch: iter_time=3.414e-04, forward_time=0.201, loss_att=39.346, acc=0.923, loss=39.346, backward_time=0.277, grad_norm=31.922, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.226e-04, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:48:55,083 (trainer:732) INFO: 21epoch:train:205-255batch: iter_time=3.553e-04, forward_time=0.202, loss_att=41.515, acc=0.922, loss=41.515, backward_time=0.281, grad_norm=32.454, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.239e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:49:27,500 (trainer:732) INFO: 21epoch:train:256-306batch: iter_time=3.385e-04, forward_time=0.201, loss_att=39.529, acc=0.923, loss=39.529, backward_time=0.279, grad_norm=32.369, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.251e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:50:00,196 (trainer:732) INFO: 21epoch:train:307-357batch: iter_time=3.627e-04, forward_time=0.202, loss_att=41.365, acc=0.923, loss=41.365, backward_time=0.281, grad_norm=30.988, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=5.264e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:50:33,232 (trainer:732) INFO: 21epoch:train:358-408batch: iter_time=3.642e-04, forward_time=0.203, loss_att=39.832, acc=0.924, loss=39.832, backward_time=0.283, grad_norm=32.568, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.277e-04, train_time=2.582
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:51:05,815 (trainer:732) INFO: 21epoch:train:409-459batch: iter_time=3.981e-04, forward_time=0.202, loss_att=43.848, acc=0.920, loss=43.848, backward_time=0.281, grad_norm=32.510, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.289e-04, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:51:37,819 (trainer:732) INFO: 21epoch:train:460-510batch: iter_time=3.641e-04, forward_time=0.199, loss_att=37.708, acc=0.925, loss=37.708, backward_time=0.273, grad_norm=31.228, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.302e-04, train_time=2.512
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:52:10,374 (trainer:732) INFO: 21epoch:train:511-561batch: iter_time=3.555e-04, forward_time=0.202, loss_att=41.902, acc=0.922, loss=41.902, backward_time=0.279, grad_norm=33.382, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.315e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:52:42,787 (trainer:732) INFO: 21epoch:train:562-612batch: iter_time=3.513e-04, forward_time=0.200, loss_att=38.366, acc=0.924, loss=38.366, backward_time=0.276, grad_norm=31.412, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.328e-04, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:53:15,196 (trainer:732) INFO: 21epoch:train:613-663batch: iter_time=3.964e-04, forward_time=0.202, loss_att=39.146, acc=0.924, loss=39.146, backward_time=0.278, grad_norm=29.868, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.340e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:53:47,495 (trainer:732) INFO: 21epoch:train:664-714batch: iter_time=3.775e-04, forward_time=0.200, loss_att=38.570, acc=0.925, loss=38.570, backward_time=0.275, grad_norm=32.248, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.353e-04, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:54:20,212 (trainer:732) INFO: 21epoch:train:715-765batch: iter_time=3.743e-04, forward_time=0.203, loss_att=39.634, acc=0.926, loss=39.634, backward_time=0.281, grad_norm=31.489, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.366e-04, train_time=2.568
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:54:52,931 (trainer:732) INFO: 21epoch:train:766-816batch: iter_time=3.972e-04, forward_time=0.203, loss_att=42.662, acc=0.921, loss=42.662, backward_time=0.280, grad_norm=34.360, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.379e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:55:25,022 (trainer:732) INFO: 21epoch:train:817-867batch: iter_time=3.859e-04, forward_time=0.200, loss_att=35.818, acc=0.928, loss=35.818, backward_time=0.274, grad_norm=29.576, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.391e-04, train_time=2.521
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:55:57,641 (trainer:732) INFO: 21epoch:train:868-918batch: iter_time=3.743e-04, forward_time=0.203, loss_att=40.368, acc=0.925, loss=40.368, backward_time=0.280, grad_norm=34.445, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.404e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:56:30,746 (trainer:732) INFO: 21epoch:train:919-969batch: iter_time=3.585e-04, forward_time=0.205, loss_att=41.101, acc=0.924, loss=41.101, backward_time=0.285, grad_norm=34.493, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.417e-04, train_time=2.593
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-20 23:57:03,186 (trainer:732) INFO: 21epoch:train:970-1020batch: iter_time=3.473e-04, forward_time=0.201, loss_att=39.506, acc=0.924, loss=39.506, backward_time=0.276, grad_norm=32.074, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.430e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:07:39,469 (trainer:338) INFO: 21epoch results: [train] iter_time=6.707e-04, forward_time=0.201, loss_att=40.070, acc=0.924, loss=40.070, backward_time=0.278, grad_norm=31.971, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.311e-04, train_time=3.027, time=13 minutes and 5.97 seconds, total_count=21777, gpu_max_cached_mem_GB=30.428, [valid] loss_att=77.732, acc=0.868, cer=0.162, wer=0.389, loss=77.732, time=4 minutes and 51.83 seconds, total_count=1848, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.55 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:07:43,660 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:07:43,668 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/11epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:07:43,669 (trainer:272) INFO: 22/60epoch started. Estimated time to finish: 15 hours, 27 minutes and 11.57 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:10:44,358 (trainer:732) INFO: 22epoch:train:1-51batch: iter_time=0.009, forward_time=0.203, loss_att=39.472, acc=0.926, loss=39.472, backward_time=0.279, grad_norm=31.584, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.446e-04, train_time=14.914
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:11:17,196 (trainer:732) INFO: 22epoch:train:52-102batch: iter_time=3.453e-04, forward_time=0.204, loss_att=38.036, acc=0.928, loss=38.036, backward_time=0.283, grad_norm=32.902, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.459e-04, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:11:49,672 (trainer:732) INFO: 22epoch:train:103-153batch: iter_time=3.496e-04, forward_time=0.201, loss_att=38.253, acc=0.929, loss=38.253, backward_time=0.279, grad_norm=30.444, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.472e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:12:21,816 (trainer:732) INFO: 22epoch:train:154-204batch: iter_time=3.459e-04, forward_time=0.198, loss_att=36.048, acc=0.928, loss=36.048, backward_time=0.273, grad_norm=29.444, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.485e-04, train_time=2.512
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:12:54,350 (trainer:732) INFO: 22epoch:train:205-255batch: iter_time=3.606e-04, forward_time=0.202, loss_att=36.759, acc=0.928, loss=36.759, backward_time=0.281, grad_norm=31.916, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.497e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:13:26,630 (trainer:732) INFO: 22epoch:train:256-306batch: iter_time=3.766e-04, forward_time=0.200, loss_att=39.291, acc=0.925, loss=39.291, backward_time=0.277, grad_norm=32.800, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.510e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:13:59,082 (trainer:732) INFO: 22epoch:train:307-357batch: iter_time=3.623e-04, forward_time=0.202, loss_att=38.350, acc=0.928, loss=38.350, backward_time=0.279, grad_norm=31.350, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.523e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:14:31,390 (trainer:732) INFO: 22epoch:train:358-408batch: iter_time=3.403e-04, forward_time=0.200, loss_att=37.331, acc=0.928, loss=37.331, backward_time=0.277, grad_norm=29.738, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.536e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:15:04,496 (trainer:732) INFO: 22epoch:train:409-459batch: iter_time=3.456e-04, forward_time=0.205, loss_att=40.424, acc=0.927, loss=40.424, backward_time=0.286, grad_norm=33.427, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.548e-04, train_time=2.621
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:15:37,175 (trainer:732) INFO: 22epoch:train:460-510batch: iter_time=3.119e-04, forward_time=0.203, loss_att=38.022, acc=0.925, loss=38.022, backward_time=0.282, grad_norm=31.084, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.561e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:16:09,806 (trainer:732) INFO: 22epoch:train:511-561batch: iter_time=3.517e-04, forward_time=0.202, loss_att=38.851, acc=0.927, loss=38.851, backward_time=0.280, grad_norm=32.886, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.574e-04, train_time=2.564
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:16:42,523 (trainer:732) INFO: 22epoch:train:562-612batch: iter_time=3.078e-04, forward_time=0.202, loss_att=37.049, acc=0.930, loss=37.049, backward_time=0.280, grad_norm=31.582, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=5.587e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:17:14,813 (trainer:732) INFO: 22epoch:train:613-663batch: iter_time=3.838e-04, forward_time=0.202, loss_att=38.526, acc=0.927, loss=38.526, backward_time=0.279, grad_norm=37.217, clip=100.000, loss_scale=1.000, optim_step_time=0.067, optim0_lr0=5.599e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:17:47,721 (trainer:732) INFO: 22epoch:train:664-714batch: iter_time=3.339e-04, forward_time=0.202, loss_att=37.773, acc=0.929, loss=37.773, backward_time=0.279, grad_norm=32.944, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.612e-04, train_time=2.578
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:18:20,087 (trainer:732) INFO: 22epoch:train:715-765batch: iter_time=3.176e-04, forward_time=0.201, loss_att=39.010, acc=0.927, loss=39.010, backward_time=0.276, grad_norm=31.565, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=5.625e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:18:52,581 (trainer:732) INFO: 22epoch:train:766-816batch: iter_time=3.780e-04, forward_time=0.202, loss_att=38.703, acc=0.928, loss=38.703, backward_time=0.279, grad_norm=31.068, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.638e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:19:24,861 (trainer:732) INFO: 22epoch:train:817-867batch: iter_time=3.581e-04, forward_time=0.201, loss_att=36.496, acc=0.931, loss=36.496, backward_time=0.278, grad_norm=30.188, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.650e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:19:56,880 (trainer:732) INFO: 22epoch:train:868-918batch: iter_time=3.750e-04, forward_time=0.199, loss_att=35.835, acc=0.929, loss=35.835, backward_time=0.273, grad_norm=27.984, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.663e-04, train_time=2.513
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:20:29,337 (trainer:732) INFO: 22epoch:train:919-969batch: iter_time=3.545e-04, forward_time=0.202, loss_att=39.205, acc=0.926, loss=39.205, backward_time=0.278, grad_norm=32.959, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.676e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:21:01,798 (trainer:732) INFO: 22epoch:train:970-1020batch: iter_time=3.177e-04, forward_time=0.201, loss_att=37.712, acc=0.928, loss=37.712, backward_time=0.278, grad_norm=31.607, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.689e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:31:37,532 (trainer:338) INFO: 22epoch results: [train] iter_time=7.837e-04, forward_time=0.202, loss_att=37.977, acc=0.928, loss=37.977, backward_time=0.279, grad_norm=31.695, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.570e-04, train_time=3.120, time=13 minutes and 30.11 seconds, total_count=22814, gpu_max_cached_mem_GB=30.428, [valid] loss_att=76.443, acc=0.870, cer=0.162, wer=0.385, loss=76.443, time=4 minutes and 52.73 seconds, total_count=1936, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 31.01 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:31:41,801 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:31:41,810 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/12epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:31:41,810 (trainer:272) INFO: 23/60epoch started. Estimated time to finish: 15 hours, 3 minutes and 45.32 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:34:44,732 (trainer:732) INFO: 23epoch:train:1-51batch: iter_time=0.012, forward_time=0.203, loss_att=35.467, acc=0.933, loss=35.467, backward_time=0.279, grad_norm=29.262, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.705e-04, train_time=15.094
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:35:17,124 (trainer:732) INFO: 23epoch:train:52-102batch: iter_time=3.534e-04, forward_time=0.202, loss_att=37.999, acc=0.929, loss=37.999, backward_time=0.279, grad_norm=32.602, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.718e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:35:49,723 (trainer:732) INFO: 23epoch:train:103-153batch: iter_time=3.486e-04, forward_time=0.202, loss_att=36.190, acc=0.931, loss=36.190, backward_time=0.281, grad_norm=31.911, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.731e-04, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:36:22,344 (trainer:732) INFO: 23epoch:train:154-204batch: iter_time=3.461e-04, forward_time=0.201, loss_att=36.481, acc=0.933, loss=36.481, backward_time=0.279, grad_norm=30.698, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.744e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:36:54,558 (trainer:732) INFO: 23epoch:train:205-255batch: iter_time=3.833e-04, forward_time=0.201, loss_att=36.312, acc=0.932, loss=36.312, backward_time=0.279, grad_norm=33.110, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.756e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:37:27,864 (trainer:732) INFO: 23epoch:train:256-306batch: iter_time=3.246e-04, forward_time=0.203, loss_att=33.848, acc=0.934, loss=33.848, backward_time=0.281, grad_norm=31.170, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.769e-04, train_time=2.613
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:38:00,340 (trainer:732) INFO: 23epoch:train:307-357batch: iter_time=3.663e-04, forward_time=0.203, loss_att=36.516, acc=0.930, loss=36.516, backward_time=0.279, grad_norm=29.583, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.782e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:38:33,361 (trainer:732) INFO: 23epoch:train:358-408batch: iter_time=3.456e-04, forward_time=0.204, loss_att=38.590, acc=0.929, loss=38.590, backward_time=0.283, grad_norm=32.673, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.795e-04, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:39:05,316 (trainer:732) INFO: 23epoch:train:409-459batch: iter_time=3.954e-04, forward_time=0.200, loss_att=34.040, acc=0.932, loss=34.040, backward_time=0.275, grad_norm=30.958, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.807e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:39:37,487 (trainer:732) INFO: 23epoch:train:460-510batch: iter_time=3.589e-04, forward_time=0.200, loss_att=33.543, acc=0.934, loss=33.543, backward_time=0.275, grad_norm=27.259, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.820e-04, train_time=2.510
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:40:09,893 (trainer:732) INFO: 23epoch:train:511-561batch: iter_time=4.024e-04, forward_time=0.202, loss_att=36.006, acc=0.931, loss=36.006, backward_time=0.278, grad_norm=29.982, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.833e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:40:42,783 (trainer:732) INFO: 23epoch:train:562-612batch: iter_time=3.709e-04, forward_time=0.204, loss_att=36.869, acc=0.932, loss=36.869, backward_time=0.281, grad_norm=32.012, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.846e-04, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:41:15,556 (trainer:732) INFO: 23epoch:train:613-663batch: iter_time=3.914e-04, forward_time=0.204, loss_att=36.805, acc=0.931, loss=36.805, backward_time=0.283, grad_norm=30.928, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.858e-04, train_time=2.577
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:41:48,228 (trainer:732) INFO: 23epoch:train:664-714batch: iter_time=3.834e-04, forward_time=0.204, loss_att=36.828, acc=0.931, loss=36.828, backward_time=0.281, grad_norm=32.286, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.871e-04, train_time=2.568
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:42:20,524 (trainer:732) INFO: 23epoch:train:715-765batch: iter_time=3.411e-04, forward_time=0.201, loss_att=35.616, acc=0.930, loss=35.616, backward_time=0.277, grad_norm=30.466, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.884e-04, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:42:53,172 (trainer:732) INFO: 23epoch:train:766-816batch: iter_time=3.873e-04, forward_time=0.203, loss_att=37.309, acc=0.930, loss=37.309, backward_time=0.279, grad_norm=31.726, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.897e-04, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:43:25,023 (trainer:732) INFO: 23epoch:train:817-867batch: iter_time=3.779e-04, forward_time=0.200, loss_att=33.248, acc=0.933, loss=33.248, backward_time=0.273, grad_norm=30.251, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.909e-04, train_time=2.506
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:43:57,432 (trainer:732) INFO: 23epoch:train:868-918batch: iter_time=3.473e-04, forward_time=0.201, loss_att=35.694, acc=0.932, loss=35.694, backward_time=0.278, grad_norm=34.938, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.922e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:44:29,827 (trainer:732) INFO: 23epoch:train:919-969batch: iter_time=3.592e-04, forward_time=0.201, loss_att=34.508, acc=0.933, loss=34.508, backward_time=0.278, grad_norm=29.725, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.935e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:45:02,402 (trainer:732) INFO: 23epoch:train:970-1020batch: iter_time=3.272e-04, forward_time=0.202, loss_att=36.826, acc=0.931, loss=36.826, backward_time=0.278, grad_norm=36.459, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.948e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:55:39,146 (trainer:338) INFO: 23epoch results: [train] iter_time=9.316e-04, forward_time=0.202, loss_att=35.916, acc=0.932, loss=35.916, backward_time=0.279, grad_norm=31.409, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.829e-04, train_time=3.130, time=13 minutes and 32.79 seconds, total_count=23851, gpu_max_cached_mem_GB=30.428, [valid] loss_att=79.139, acc=0.869, cer=0.162, wer=0.387, loss=79.139, time=4 minutes and 53.99 seconds, total_count=2024, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 30.55 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:55:43,541 (trainer:384) INFO: There are no improvements in this epoch
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:55:43,548 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/13epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:55:43,549 (trainer:272) INFO: 24/60epoch started. Estimated time to finish: 14 hours, 40 minutes and 22.07 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:58:48,332 (trainer:732) INFO: 24epoch:train:1-51batch: iter_time=0.012, forward_time=0.203, loss_att=34.657, acc=0.936, loss=34.657, backward_time=0.279, grad_norm=33.723, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=5.964e-04, train_time=15.234
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:59:20,917 (trainer:732) INFO: 24epoch:train:52-102batch: iter_time=3.592e-04, forward_time=0.203, loss_att=34.356, acc=0.935, loss=34.356, backward_time=0.280, grad_norm=30.305, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.977e-04, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 00:59:53,511 (trainer:732) INFO: 24epoch:train:103-153batch: iter_time=3.586e-04, forward_time=0.202, loss_att=32.986, acc=0.937, loss=32.986, backward_time=0.280, grad_norm=29.624, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=5.990e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:00:25,890 (trainer:732) INFO: 24epoch:train:154-204batch: iter_time=3.695e-04, forward_time=0.201, loss_att=33.892, acc=0.935, loss=33.892, backward_time=0.277, grad_norm=30.169, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.003e-04, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:00:57,854 (trainer:732) INFO: 24epoch:train:205-255batch: iter_time=3.697e-04, forward_time=0.199, loss_att=33.179, acc=0.934, loss=33.179, backward_time=0.274, grad_norm=31.467, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.016e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:01:30,346 (trainer:732) INFO: 24epoch:train:256-306batch: iter_time=3.650e-04, forward_time=0.202, loss_att=34.757, acc=0.932, loss=34.757, backward_time=0.279, grad_norm=31.058, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.028e-04, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:02:02,602 (trainer:732) INFO: 24epoch:train:307-357batch: iter_time=3.858e-04, forward_time=0.202, loss_att=33.948, acc=0.934, loss=33.948, backward_time=0.276, grad_norm=30.653, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.041e-04, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:02:35,348 (trainer:732) INFO: 24epoch:train:358-408batch: iter_time=4.857e-04, forward_time=0.203, loss_att=34.113, acc=0.936, loss=34.113, backward_time=0.280, grad_norm=31.393, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.054e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:03:07,340 (trainer:732) INFO: 24epoch:train:409-459batch: iter_time=3.803e-04, forward_time=0.199, loss_att=32.358, acc=0.938, loss=32.358, backward_time=0.275, grad_norm=29.940, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=6.066e-04, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:03:39,675 (trainer:732) INFO: 24epoch:train:460-510batch: iter_time=3.752e-04, forward_time=0.201, loss_att=33.353, acc=0.935, loss=33.353, backward_time=0.277, grad_norm=29.958, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.079e-04, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:04:12,277 (trainer:732) INFO: 24epoch:train:511-561batch: iter_time=3.717e-04, forward_time=0.203, loss_att=36.360, acc=0.933, loss=36.360, backward_time=0.279, grad_norm=33.126, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.092e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:04:45,200 (trainer:732) INFO: 24epoch:train:562-612batch: iter_time=3.635e-04, forward_time=0.205, loss_att=35.635, acc=0.935, loss=35.635, backward_time=0.283, grad_norm=33.687, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.105e-04, train_time=2.572
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:05:17,539 (trainer:732) INFO: 24epoch:train:613-663batch: iter_time=3.483e-04, forward_time=0.202, loss_att=31.467, acc=0.937, loss=31.467, backward_time=0.278, grad_norm=31.001, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.117e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:05:49,856 (trainer:732) INFO: 24epoch:train:664-714batch: iter_time=3.528e-04, forward_time=0.201, loss_att=33.128, acc=0.937, loss=33.128, backward_time=0.278, grad_norm=30.351, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.130e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:06:22,370 (trainer:732) INFO: 24epoch:train:715-765batch: iter_time=3.178e-04, forward_time=0.201, loss_att=31.930, acc=0.937, loss=31.930, backward_time=0.278, grad_norm=30.516, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.143e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:06:54,933 (trainer:732) INFO: 24epoch:train:766-816batch: iter_time=3.435e-04, forward_time=0.201, loss_att=33.269, acc=0.938, loss=33.269, backward_time=0.277, grad_norm=29.641, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=6.156e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:07:27,656 (trainer:732) INFO: 24epoch:train:817-867batch: iter_time=3.704e-04, forward_time=0.204, loss_att=34.708, acc=0.935, loss=34.708, backward_time=0.282, grad_norm=33.437, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.168e-04, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:08:00,191 (trainer:732) INFO: 24epoch:train:868-918batch: iter_time=3.734e-04, forward_time=0.202, loss_att=34.617, acc=0.935, loss=34.617, backward_time=0.278, grad_norm=31.915, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.181e-04, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:08:33,056 (trainer:732) INFO: 24epoch:train:919-969batch: iter_time=3.543e-04, forward_time=0.204, loss_att=37.208, acc=0.933, loss=37.208, backward_time=0.283, grad_norm=31.712, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.194e-04, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:09:05,645 (trainer:732) INFO: 24epoch:train:970-1020batch: iter_time=3.122e-04, forward_time=0.202, loss_att=32.541, acc=0.937, loss=32.541, backward_time=0.279, grad_norm=30.961, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.207e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:19:43,907 (trainer:338) INFO: 24epoch results: [train] iter_time=9.450e-04, forward_time=0.202, loss_att=33.910, acc=0.935, loss=33.910, backward_time=0.279, grad_norm=31.240, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.088e-04, train_time=3.136, time=13 minutes and 34.2 seconds, total_count=24888, gpu_max_cached_mem_GB=30.428, [valid] loss_att=77.515, acc=0.872, cer=0.162, wer=0.385, loss=77.515, time=4 minutes and 53.36 seconds, total_count=2112, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.79 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:19:48,418 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:19:48,427 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/14epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:19:48,427 (trainer:272) INFO: 25/60epoch started. Estimated time to finish: 14 hours, 17 minutes and 0.33 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:22:49,900 (trainer:732) INFO: 25epoch:train:1-51batch: iter_time=0.015, forward_time=0.204, loss_att=31.794, acc=0.940, loss=31.794, backward_time=0.278, grad_norm=29.409, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=6.223e-04, train_time=14.976
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:23:22,038 (trainer:732) INFO: 25epoch:train:52-102batch: iter_time=3.491e-04, forward_time=0.199, loss_att=33.813, acc=0.936, loss=33.813, backward_time=0.275, grad_norm=31.391, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.236e-04, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:23:55,335 (trainer:732) INFO: 25epoch:train:103-153batch: iter_time=3.447e-04, forward_time=0.206, loss_att=33.225, acc=0.939, loss=33.225, backward_time=0.287, grad_norm=31.249, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.249e-04, train_time=2.605
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:24:27,569 (trainer:732) INFO: 25epoch:train:154-204batch: iter_time=3.361e-04, forward_time=0.200, loss_att=30.786, acc=0.939, loss=30.786, backward_time=0.275, grad_norm=29.535, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.262e-04, train_time=2.519
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:24:59,914 (trainer:732) INFO: 25epoch:train:205-255batch: iter_time=3.415e-04, forward_time=0.201, loss_att=33.900, acc=0.937, loss=33.900, backward_time=0.279, grad_norm=33.002, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.274e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:25:31,979 (trainer:732) INFO: 25epoch:train:256-306batch: iter_time=3.257e-04, forward_time=0.199, loss_att=30.517, acc=0.939, loss=30.517, backward_time=0.274, grad_norm=29.908, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.287e-04, train_time=2.516
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:26:04,711 (trainer:732) INFO: 25epoch:train:307-357batch: iter_time=3.634e-04, forward_time=0.203, loss_att=33.178, acc=0.938, loss=33.178, backward_time=0.282, grad_norm=30.413, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.300e-04, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:26:37,542 (trainer:732) INFO: 25epoch:train:358-408batch: iter_time=3.355e-04, forward_time=0.203, loss_att=35.221, acc=0.937, loss=35.221, backward_time=0.281, grad_norm=35.896, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.313e-04, train_time=2.564
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:27:10,019 (trainer:732) INFO: 25epoch:train:409-459batch: iter_time=3.430e-04, forward_time=0.202, loss_att=32.191, acc=0.939, loss=32.191, backward_time=0.280, grad_norm=30.829, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.325e-04, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:27:42,661 (trainer:732) INFO: 25epoch:train:460-510batch: iter_time=3.353e-04, forward_time=0.203, loss_att=34.658, acc=0.936, loss=34.658, backward_time=0.281, grad_norm=32.133, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.338e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:28:14,846 (trainer:732) INFO: 25epoch:train:511-561batch: iter_time=3.592e-04, forward_time=0.201, loss_att=33.182, acc=0.936, loss=33.182, backward_time=0.277, grad_norm=29.413, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.351e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:28:47,782 (trainer:732) INFO: 25epoch:train:562-612batch: iter_time=3.273e-04, forward_time=0.204, loss_att=31.127, acc=0.941, loss=31.127, backward_time=0.283, grad_norm=31.741, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.364e-04, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:29:19,897 (trainer:732) INFO: 25epoch:train:613-663batch: iter_time=3.731e-04, forward_time=0.200, loss_att=30.947, acc=0.940, loss=30.947, backward_time=0.276, grad_norm=28.999, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.376e-04, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:29:52,163 (trainer:732) INFO: 25epoch:train:664-714batch: iter_time=3.156e-04, forward_time=0.201, loss_att=30.805, acc=0.940, loss=30.805, backward_time=0.277, grad_norm=31.427, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.389e-04, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:30:24,709 (trainer:732) INFO: 25epoch:train:715-765batch: iter_time=3.315e-04, forward_time=0.202, loss_att=31.384, acc=0.938, loss=31.384, backward_time=0.280, grad_norm=31.838, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.402e-04, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:30:56,806 (trainer:732) INFO: 25epoch:train:766-816batch: iter_time=3.399e-04, forward_time=0.198, loss_att=30.017, acc=0.941, loss=30.017, backward_time=0.274, grad_norm=28.504, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.415e-04, train_time=2.511
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:31:29,109 (trainer:732) INFO: 25epoch:train:817-867batch: iter_time=3.460e-04, forward_time=0.201, loss_att=31.202, acc=0.941, loss=31.202, backward_time=0.278, grad_norm=30.152, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.427e-04, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:32:01,764 (trainer:732) INFO: 25epoch:train:868-918batch: iter_time=3.363e-04, forward_time=0.203, loss_att=32.173, acc=0.938, loss=32.173, backward_time=0.281, grad_norm=31.528, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.440e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:32:34,177 (trainer:732) INFO: 25epoch:train:919-969batch: iter_time=3.507e-04, forward_time=0.201, loss_att=31.913, acc=0.939, loss=31.913, backward_time=0.279, grad_norm=29.667, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=6.453e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:33:06,730 (trainer:732) INFO: 25epoch:train:970-1020batch: iter_time=2.777e-04, forward_time=0.202, loss_att=33.083, acc=0.938, loss=33.083, backward_time=0.279, grad_norm=32.103, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.466e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:43:46,266 (trainer:338) INFO: 25epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=32.176, acc=0.939, loss=32.176, backward_time=0.279, grad_norm=30.909, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.347e-04, train_time=3.121, time=13 minutes and 30.19 seconds, total_count=25925, gpu_max_cached_mem_GB=30.428, [valid] loss_att=76.242, acc=0.874, cer=0.156, wer=0.376, loss=76.242, time=4 minutes and 54.85 seconds, total_count=2200, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.79 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:43:50,693 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:43:50,702 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/15epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:43:50,703 (trainer:272) INFO: 26/60epoch started. Estimated time to finish: 13 hours, 53 minutes and 31.5 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:46:54,238 (trainer:732) INFO: 26epoch:train:1-51batch: iter_time=0.011, forward_time=0.203, loss_att=29.600, acc=0.944, loss=29.600, backward_time=0.278, grad_norm=28.196, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.482e-04, train_time=15.155
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:47:26,863 (trainer:732) INFO: 26epoch:train:52-102batch: iter_time=3.281e-04, forward_time=0.202, loss_att=31.125, acc=0.942, loss=31.125, backward_time=0.280, grad_norm=29.863, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.495e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:47:59,465 (trainer:732) INFO: 26epoch:train:103-153batch: iter_time=3.492e-04, forward_time=0.202, loss_att=32.194, acc=0.941, loss=32.194, backward_time=0.281, grad_norm=32.570, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.508e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:48:32,182 (trainer:732) INFO: 26epoch:train:154-204batch: iter_time=3.139e-04, forward_time=0.202, loss_att=30.828, acc=0.941, loss=30.828, backward_time=0.281, grad_norm=31.413, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.521e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:49:04,475 (trainer:732) INFO: 26epoch:train:205-255batch: iter_time=3.283e-04, forward_time=0.201, loss_att=28.635, acc=0.945, loss=28.635, backward_time=0.278, grad_norm=33.515, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.533e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:49:37,036 (trainer:732) INFO: 26epoch:train:256-306batch: iter_time=3.322e-04, forward_time=0.202, loss_att=28.849, acc=0.943, loss=28.849, backward_time=0.279, grad_norm=28.013, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=6.546e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:50:09,439 (trainer:732) INFO: 26epoch:train:307-357batch: iter_time=3.654e-04, forward_time=0.202, loss_att=31.637, acc=0.941, loss=31.637, backward_time=0.280, grad_norm=31.763, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.559e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:50:42,019 (trainer:732) INFO: 26epoch:train:358-408batch: iter_time=3.217e-04, forward_time=0.201, loss_att=31.591, acc=0.940, loss=31.591, backward_time=0.277, grad_norm=30.422, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.572e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:51:14,117 (trainer:732) INFO: 26epoch:train:409-459batch: iter_time=3.722e-04, forward_time=0.200, loss_att=28.889, acc=0.944, loss=28.889, backward_time=0.276, grad_norm=27.882, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.584e-04, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:51:46,556 (trainer:732) INFO: 26epoch:train:460-510batch: iter_time=3.246e-04, forward_time=0.202, loss_att=30.010, acc=0.943, loss=30.010, backward_time=0.278, grad_norm=28.912, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.597e-04, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:52:19,379 (trainer:732) INFO: 26epoch:train:511-561batch: iter_time=3.215e-04, forward_time=0.203, loss_att=32.241, acc=0.940, loss=32.241, backward_time=0.280, grad_norm=31.606, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.610e-04, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:52:51,974 (trainer:732) INFO: 26epoch:train:562-612batch: iter_time=3.559e-04, forward_time=0.202, loss_att=31.102, acc=0.940, loss=31.102, backward_time=0.277, grad_norm=32.863, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=6.623e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:53:24,039 (trainer:732) INFO: 26epoch:train:613-663batch: iter_time=3.401e-04, forward_time=0.201, loss_att=31.142, acc=0.941, loss=31.142, backward_time=0.276, grad_norm=29.603, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.636e-04, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:53:56,780 (trainer:732) INFO: 26epoch:train:664-714batch: iter_time=3.255e-04, forward_time=0.204, loss_att=31.343, acc=0.941, loss=31.343, backward_time=0.282, grad_norm=31.209, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.648e-04, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:54:29,480 (trainer:732) INFO: 26epoch:train:715-765batch: iter_time=2.822e-04, forward_time=0.203, loss_att=30.969, acc=0.943, loss=30.969, backward_time=0.281, grad_norm=33.815, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.661e-04, train_time=2.568
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:55:02,441 (trainer:732) INFO: 26epoch:train:766-816batch: iter_time=3.548e-04, forward_time=0.203, loss_att=33.656, acc=0.940, loss=33.656, backward_time=0.282, grad_norm=34.604, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.674e-04, train_time=2.572
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:55:34,563 (trainer:732) INFO: 26epoch:train:817-867batch: iter_time=3.187e-04, forward_time=0.201, loss_att=30.094, acc=0.941, loss=30.094, backward_time=0.277, grad_norm=35.289, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.687e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:56:07,263 (trainer:732) INFO: 26epoch:train:868-918batch: iter_time=3.364e-04, forward_time=0.203, loss_att=29.875, acc=0.943, loss=29.875, backward_time=0.281, grad_norm=31.123, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.699e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:56:39,499 (trainer:732) INFO: 26epoch:train:919-969batch: iter_time=3.441e-04, forward_time=0.201, loss_att=27.946, acc=0.943, loss=27.946, backward_time=0.276, grad_norm=27.781, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.712e-04, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 01:57:11,609 (trainer:732) INFO: 26epoch:train:970-1020batch: iter_time=3.447e-04, forward_time=0.199, loss_att=28.201, acc=0.944, loss=28.201, backward_time=0.275, grad_norm=29.572, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.725e-04, train_time=2.512
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:07:46,338 (trainer:338) INFO: 26epoch results: [train] iter_time=8.495e-04, forward_time=0.202, loss_att=30.433, acc=0.942, loss=30.433, backward_time=0.279, grad_norm=31.009, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.606e-04, train_time=3.131, time=13 minutes and 32.94 seconds, total_count=26962, gpu_max_cached_mem_GB=30.428, [valid] loss_att=75.656, acc=0.875, cer=0.159, wer=0.374, loss=75.656, time=4 minutes and 52.99 seconds, total_count=2288, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 29.7 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:07:50,832 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:07:50,841 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/16epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:07:50,841 (trainer:272) INFO: 27/60epoch started. Estimated time to finish: 13 hours, 29 minutes and 57.29 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:10:55,719 (trainer:732) INFO: 27epoch:train:1-51batch: iter_time=0.015, forward_time=0.203, loss_att=30.503, acc=0.945, loss=30.503, backward_time=0.280, grad_norm=32.795, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.741e-04, train_time=15.260
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:11:28,735 (trainer:732) INFO: 27epoch:train:52-102batch: iter_time=3.425e-04, forward_time=0.205, loss_att=29.925, acc=0.945, loss=29.925, backward_time=0.284, grad_norm=32.091, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.754e-04, train_time=2.588
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:12:01,196 (trainer:732) INFO: 27epoch:train:103-153batch: iter_time=3.703e-04, forward_time=0.200, loss_att=28.459, acc=0.946, loss=28.459, backward_time=0.278, grad_norm=29.124, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.767e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:12:33,400 (trainer:732) INFO: 27epoch:train:154-204batch: iter_time=3.513e-04, forward_time=0.200, loss_att=28.622, acc=0.942, loss=28.622, backward_time=0.275, grad_norm=27.762, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.780e-04, train_time=2.515
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:13:05,672 (trainer:732) INFO: 27epoch:train:205-255batch: iter_time=3.465e-04, forward_time=0.201, loss_att=29.545, acc=0.945, loss=29.545, backward_time=0.278, grad_norm=30.610, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.792e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:13:37,983 (trainer:732) INFO: 27epoch:train:256-306batch: iter_time=3.482e-04, forward_time=0.201, loss_att=29.614, acc=0.944, loss=29.614, backward_time=0.277, grad_norm=28.480, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.805e-04, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:14:10,632 (trainer:732) INFO: 27epoch:train:307-357batch: iter_time=3.998e-04, forward_time=0.203, loss_att=29.707, acc=0.944, loss=29.707, backward_time=0.279, grad_norm=30.712, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=6.818e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:14:43,013 (trainer:732) INFO: 27epoch:train:358-408batch: iter_time=3.794e-04, forward_time=0.201, loss_att=28.366, acc=0.944, loss=28.366, backward_time=0.276, grad_norm=31.160, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.831e-04, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:15:15,006 (trainer:732) INFO: 27epoch:train:409-459batch: iter_time=3.896e-04, forward_time=0.201, loss_att=28.004, acc=0.946, loss=28.004, backward_time=0.276, grad_norm=29.123, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.843e-04, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:15:47,907 (trainer:732) INFO: 27epoch:train:460-510batch: iter_time=3.751e-04, forward_time=0.204, loss_att=29.084, acc=0.945, loss=29.084, backward_time=0.281, grad_norm=30.405, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.856e-04, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:16:20,221 (trainer:732) INFO: 27epoch:train:511-561batch: iter_time=3.674e-04, forward_time=0.201, loss_att=28.918, acc=0.944, loss=28.918, backward_time=0.277, grad_norm=33.293, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=6.869e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:16:52,663 (trainer:732) INFO: 27epoch:train:562-612batch: iter_time=3.643e-04, forward_time=0.201, loss_att=29.756, acc=0.942, loss=29.756, backward_time=0.278, grad_norm=31.238, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.882e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:17:24,570 (trainer:732) INFO: 27epoch:train:613-663batch: iter_time=3.657e-04, forward_time=0.200, loss_att=25.993, acc=0.948, loss=25.993, backward_time=0.274, grad_norm=29.550, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=6.894e-04, train_time=2.518
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:17:56,877 (trainer:732) INFO: 27epoch:train:664-714batch: iter_time=3.161e-04, forward_time=0.200, loss_att=29.119, acc=0.945, loss=29.119, backward_time=0.279, grad_norm=30.358, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=6.907e-04, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:18:29,371 (trainer:732) INFO: 27epoch:train:715-765batch: iter_time=3.159e-04, forward_time=0.203, loss_att=27.711, acc=0.947, loss=27.711, backward_time=0.278, grad_norm=29.018, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.920e-04, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:19:02,149 (trainer:732) INFO: 27epoch:train:766-816batch: iter_time=3.717e-04, forward_time=0.202, loss_att=29.772, acc=0.945, loss=29.772, backward_time=0.281, grad_norm=31.820, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=6.933e-04, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:19:34,372 (trainer:732) INFO: 27epoch:train:817-867batch: iter_time=3.841e-04, forward_time=0.201, loss_att=28.579, acc=0.946, loss=28.579, backward_time=0.278, grad_norm=30.808, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.945e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:20:07,358 (trainer:732) INFO: 27epoch:train:868-918batch: iter_time=3.444e-04, forward_time=0.205, loss_att=30.157, acc=0.946, loss=30.157, backward_time=0.283, grad_norm=32.109, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.958e-04, train_time=2.582
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:20:40,439 (trainer:732) INFO: 27epoch:train:919-969batch: iter_time=3.688e-04, forward_time=0.206, loss_att=29.142, acc=0.945, loss=29.142, backward_time=0.285, grad_norm=32.572, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.971e-04, train_time=2.592
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:21:12,992 (trainer:732) INFO: 27epoch:train:970-1020batch: iter_time=3.034e-04, forward_time=0.202, loss_att=27.443, acc=0.948, loss=27.443, backward_time=0.279, grad_norm=30.180, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.984e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:31:43,478 (trainer:338) INFO: 27epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=28.819, acc=0.945, loss=28.819, backward_time=0.279, grad_norm=30.619, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.865e-04, train_time=3.135, time=13 minutes and 33.93 seconds, total_count=27999, gpu_max_cached_mem_GB=30.428, [valid] loss_att=77.346, acc=0.875, cer=0.162, wer=0.374, loss=77.346, time=4 minutes and 51.75 seconds, total_count=2376, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 26.95 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:31:47,934 (trainer:384) INFO: There are no improvements in this epoch
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:31:47,943 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/17epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:31:47,944 (trainer:272) INFO: 28/60epoch started. Estimated time to finish: 13 hours, 6 minutes and 17.46 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:34:51,672 (trainer:732) INFO: 28epoch:train:1-51batch: iter_time=0.013, forward_time=0.204, loss_att=27.364, acc=0.948, loss=27.364, backward_time=0.280, grad_norm=30.348, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=7.000e-04, train_time=15.161
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:35:23,950 (trainer:732) INFO: 28epoch:train:52-102batch: iter_time=3.676e-04, forward_time=0.200, loss_att=27.119, acc=0.949, loss=27.119, backward_time=0.277, grad_norm=28.342, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.013e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:35:56,954 (trainer:732) INFO: 28epoch:train:103-153batch: iter_time=3.244e-04, forward_time=0.204, loss_att=28.128, acc=0.947, loss=28.128, backward_time=0.284, grad_norm=32.662, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.026e-04, train_time=2.585
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:36:29,633 (trainer:732) INFO: 28epoch:train:154-204batch: iter_time=3.446e-04, forward_time=0.202, loss_att=27.829, acc=0.948, loss=27.829, backward_time=0.280, grad_norm=30.358, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.039e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:37:02,159 (trainer:732) INFO: 28epoch:train:205-255batch: iter_time=3.181e-04, forward_time=0.201, loss_att=27.491, acc=0.948, loss=27.491, backward_time=0.279, grad_norm=31.238, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.051e-04, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:37:34,657 (trainer:732) INFO: 28epoch:train:256-306batch: iter_time=3.298e-04, forward_time=0.203, loss_att=27.927, acc=0.946, loss=27.927, backward_time=0.279, grad_norm=30.923, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.064e-04, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:38:07,315 (trainer:732) INFO: 28epoch:train:307-357batch: iter_time=3.419e-04, forward_time=0.202, loss_att=27.216, acc=0.949, loss=27.216, backward_time=0.279, grad_norm=31.423, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.077e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:38:39,867 (trainer:732) INFO: 28epoch:train:358-408batch: iter_time=3.011e-04, forward_time=0.201, loss_att=27.383, acc=0.947, loss=27.383, backward_time=0.278, grad_norm=29.285, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=7.090e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:39:12,665 (trainer:732) INFO: 28epoch:train:409-459batch: iter_time=3.428e-04, forward_time=0.203, loss_att=26.841, acc=0.951, loss=26.841, backward_time=0.283, grad_norm=30.999, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.102e-04, train_time=2.593
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:39:45,084 (trainer:732) INFO: 28epoch:train:460-510batch: iter_time=3.540e-04, forward_time=0.202, loss_att=26.936, acc=0.949, loss=26.936, backward_time=0.278, grad_norm=30.291, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.115e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:40:17,438 (trainer:732) INFO: 28epoch:train:511-561batch: iter_time=3.092e-04, forward_time=0.201, loss_att=27.786, acc=0.948, loss=27.786, backward_time=0.277, grad_norm=29.364, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.128e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:40:49,725 (trainer:732) INFO: 28epoch:train:562-612batch: iter_time=3.333e-04, forward_time=0.200, loss_att=27.422, acc=0.946, loss=27.422, backward_time=0.276, grad_norm=28.294, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.141e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:41:22,379 (trainer:732) INFO: 28epoch:train:613-663batch: iter_time=4.758e-04, forward_time=0.204, loss_att=26.399, acc=0.949, loss=26.399, backward_time=0.280, grad_norm=29.500, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.154e-04, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:41:54,686 (trainer:732) INFO: 28epoch:train:664-714batch: iter_time=3.268e-04, forward_time=0.201, loss_att=26.973, acc=0.948, loss=26.973, backward_time=0.277, grad_norm=31.047, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.166e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:42:27,056 (trainer:732) INFO: 28epoch:train:715-765batch: iter_time=2.884e-04, forward_time=0.201, loss_att=28.799, acc=0.946, loss=28.799, backward_time=0.277, grad_norm=32.242, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.179e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:42:59,419 (trainer:732) INFO: 28epoch:train:766-816batch: iter_time=3.808e-04, forward_time=0.201, loss_att=28.478, acc=0.946, loss=28.478, backward_time=0.278, grad_norm=29.825, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.192e-04, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:43:31,673 (trainer:732) INFO: 28epoch:train:817-867batch: iter_time=3.448e-04, forward_time=0.202, loss_att=26.038, acc=0.950, loss=26.038, backward_time=0.278, grad_norm=29.943, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.204e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:44:04,360 (trainer:732) INFO: 28epoch:train:868-918batch: iter_time=3.568e-04, forward_time=0.203, loss_att=28.269, acc=0.947, loss=28.269, backward_time=0.281, grad_norm=30.876, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.217e-04, train_time=2.568
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:44:36,610 (trainer:732) INFO: 28epoch:train:919-969batch: iter_time=3.421e-04, forward_time=0.200, loss_att=27.079, acc=0.948, loss=27.079, backward_time=0.278, grad_norm=29.957, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.230e-04, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:45:08,800 (trainer:732) INFO: 28epoch:train:970-1020batch: iter_time=3.676e-04, forward_time=0.199, loss_att=26.611, acc=0.948, loss=26.611, backward_time=0.275, grad_norm=29.992, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.243e-04, train_time=2.516
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:55:44,907 (trainer:338) INFO: 28epoch results: [train] iter_time=9.489e-04, forward_time=0.202, loss_att=27.449, acc=0.948, loss=27.449, backward_time=0.279, grad_norm=30.373, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.124e-04, train_time=3.131, time=13 minutes and 32.99 seconds, total_count=29036, gpu_max_cached_mem_GB=30.428, [valid] loss_att=74.913, acc=0.878, cer=0.151, wer=0.370, loss=74.913, time=4 minutes and 53.82 seconds, total_count=2464, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 30.14 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:55:48,670 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:55:48,684 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/18epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:55:48,685 (trainer:272) INFO: 29/60epoch started. Estimated time to finish: 12 hours, 42 minutes and 40.55 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:58:51,035 (trainer:732) INFO: 29epoch:train:1-51batch: iter_time=0.011, forward_time=0.203, loss_att=25.362, acc=0.952, loss=25.362, backward_time=0.278, grad_norm=31.761, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.259e-04, train_time=15.051
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:59:23,756 (trainer:732) INFO: 29epoch:train:52-102batch: iter_time=3.688e-04, forward_time=0.203, loss_att=27.357, acc=0.950, loss=27.357, backward_time=0.281, grad_norm=33.302, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.272e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 02:59:56,586 (trainer:732) INFO: 29epoch:train:103-153batch: iter_time=3.570e-04, forward_time=0.205, loss_att=26.085, acc=0.950, loss=26.085, backward_time=0.283, grad_norm=30.664, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.285e-04, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:00:28,925 (trainer:732) INFO: 29epoch:train:154-204batch: iter_time=3.187e-04, forward_time=0.202, loss_att=25.031, acc=0.952, loss=25.031, backward_time=0.276, grad_norm=31.322, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.298e-04, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:01:01,500 (trainer:732) INFO: 29epoch:train:205-255batch: iter_time=3.744e-04, forward_time=0.203, loss_att=25.726, acc=0.953, loss=25.726, backward_time=0.281, grad_norm=32.356, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.310e-04, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:01:33,831 (trainer:732) INFO: 29epoch:train:256-306batch: iter_time=3.238e-04, forward_time=0.202, loss_att=27.305, acc=0.948, loss=27.305, backward_time=0.278, grad_norm=32.307, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.323e-04, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:02:05,814 (trainer:732) INFO: 29epoch:train:307-357batch: iter_time=3.939e-04, forward_time=0.200, loss_att=24.300, acc=0.949, loss=24.300, backward_time=0.273, grad_norm=27.652, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.336e-04, train_time=2.506
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:02:38,365 (trainer:732) INFO: 29epoch:train:358-408batch: iter_time=3.714e-04, forward_time=0.202, loss_att=25.200, acc=0.951, loss=25.200, backward_time=0.278, grad_norm=32.614, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.349e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:03:10,589 (trainer:732) INFO: 29epoch:train:409-459batch: iter_time=4.045e-04, forward_time=0.201, loss_att=23.305, acc=0.954, loss=23.305, backward_time=0.276, grad_norm=27.381, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.361e-04, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:03:43,446 (trainer:732) INFO: 29epoch:train:460-510batch: iter_time=3.515e-04, forward_time=0.204, loss_att=25.322, acc=0.951, loss=25.322, backward_time=0.281, grad_norm=28.815, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.374e-04, train_time=2.582
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:04:16,174 (trainer:732) INFO: 29epoch:train:511-561batch: iter_time=3.769e-04, forward_time=0.203, loss_att=25.613, acc=0.951, loss=25.613, backward_time=0.280, grad_norm=28.903, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.387e-04, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:04:48,946 (trainer:732) INFO: 29epoch:train:562-612batch: iter_time=3.788e-04, forward_time=0.203, loss_att=28.815, acc=0.948, loss=28.815, backward_time=0.281, grad_norm=32.189, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.400e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:05:21,390 (trainer:732) INFO: 29epoch:train:613-663batch: iter_time=3.752e-04, forward_time=0.203, loss_att=28.075, acc=0.947, loss=28.075, backward_time=0.280, grad_norm=30.834, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.413e-04, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:05:53,847 (trainer:732) INFO: 29epoch:train:664-714batch: iter_time=3.451e-04, forward_time=0.201, loss_att=26.814, acc=0.950, loss=26.814, backward_time=0.279, grad_norm=31.109, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.425e-04, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:06:26,221 (trainer:732) INFO: 29epoch:train:715-765batch: iter_time=3.095e-04, forward_time=0.202, loss_att=25.405, acc=0.952, loss=25.405, backward_time=0.277, grad_norm=31.138, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.438e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:06:58,886 (trainer:732) INFO: 29epoch:train:766-816batch: iter_time=3.436e-04, forward_time=0.202, loss_att=25.487, acc=0.952, loss=25.487, backward_time=0.279, grad_norm=28.897, clip=100.000, loss_scale=1.000, optim_step_time=0.068, optim0_lr0=7.451e-04, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:07:31,305 (trainer:732) INFO: 29epoch:train:817-867batch: iter_time=3.569e-04, forward_time=0.202, loss_att=26.096, acc=0.950, loss=26.096, backward_time=0.280, grad_norm=32.024, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.463e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:08:03,833 (trainer:732) INFO: 29epoch:train:868-918batch: iter_time=3.627e-04, forward_time=0.203, loss_att=27.471, acc=0.949, loss=27.471, backward_time=0.279, grad_norm=32.825, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.476e-04, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:08:36,292 (trainer:732) INFO: 29epoch:train:919-969batch: iter_time=3.929e-04, forward_time=0.203, loss_att=24.719, acc=0.951, loss=24.719, backward_time=0.279, grad_norm=30.870, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.489e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:09:08,902 (trainer:732) INFO: 29epoch:train:970-1020batch: iter_time=3.321e-04, forward_time=0.202, loss_att=25.687, acc=0.951, loss=25.687, backward_time=0.279, grad_norm=34.782, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.502e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:19:45,902 (trainer:338) INFO: 29epoch results: [train] iter_time=9.028e-04, forward_time=0.202, loss_att=25.943, acc=0.951, loss=25.943, backward_time=0.279, grad_norm=31.106, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.383e-04, train_time=3.129, time=13 minutes and 32.69 seconds, total_count=30073, gpu_max_cached_mem_GB=30.428, [valid] loss_att=76.308, acc=0.878, cer=0.154, wer=0.371, loss=76.308, time=4 minutes and 52.45 seconds, total_count=2552, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.07 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:19:50,193 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:19:50,203 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/19epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:19:50,204 (trainer:272) INFO: 30/60epoch started. Estimated time to finish: 12 hours, 19 minutes and 2.83 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:22:53,610 (trainer:732) INFO: 30epoch:train:1-51batch: iter_time=0.014, forward_time=0.205, loss_att=25.271, acc=0.952, loss=25.271, backward_time=0.280, grad_norm=29.850, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.518e-04, train_time=15.138
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:23:25,798 (trainer:732) INFO: 30epoch:train:52-102batch: iter_time=3.647e-04, forward_time=0.199, loss_att=24.933, acc=0.952, loss=24.933, backward_time=0.275, grad_norm=30.091, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.531e-04, train_time=2.518
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:23:57,894 (trainer:732) INFO: 30epoch:train:103-153batch: iter_time=3.842e-04, forward_time=0.199, loss_att=24.252, acc=0.954, loss=24.252, backward_time=0.274, grad_norm=27.563, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=7.544e-04, train_time=2.521
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:24:30,438 (trainer:732) INFO: 30epoch:train:154-204batch: iter_time=3.659e-04, forward_time=0.201, loss_att=23.325, acc=0.955, loss=23.325, backward_time=0.278, grad_norm=29.616, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.557e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:25:02,941 (trainer:732) INFO: 30epoch:train:205-255batch: iter_time=3.640e-04, forward_time=0.203, loss_att=25.438, acc=0.952, loss=25.438, backward_time=0.281, grad_norm=31.489, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.569e-04, train_time=2.568
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:25:35,282 (trainer:732) INFO: 30epoch:train:256-306batch: iter_time=3.451e-04, forward_time=0.201, loss_att=24.220, acc=0.953, loss=24.220, backward_time=0.277, grad_norm=28.088, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.582e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:26:07,938 (trainer:732) INFO: 30epoch:train:307-357batch: iter_time=3.728e-04, forward_time=0.202, loss_att=23.623, acc=0.956, loss=23.623, backward_time=0.281, grad_norm=30.728, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.595e-04, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:26:40,484 (trainer:732) INFO: 30epoch:train:358-408batch: iter_time=3.515e-04, forward_time=0.202, loss_att=23.338, acc=0.954, loss=23.338, backward_time=0.278, grad_norm=29.634, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.608e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:27:13,163 (trainer:732) INFO: 30epoch:train:409-459batch: iter_time=3.742e-04, forward_time=0.204, loss_att=26.935, acc=0.951, loss=26.935, backward_time=0.283, grad_norm=34.689, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=7.620e-04, train_time=2.578
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:27:45,525 (trainer:732) INFO: 30epoch:train:460-510batch: iter_time=3.218e-04, forward_time=0.201, loss_att=24.418, acc=0.955, loss=24.418, backward_time=0.278, grad_norm=30.649, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.633e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:28:18,377 (trainer:732) INFO: 30epoch:train:511-561batch: iter_time=3.790e-04, forward_time=0.203, loss_att=25.845, acc=0.953, loss=25.845, backward_time=0.283, grad_norm=31.396, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.646e-04, train_time=2.577
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:28:50,911 (trainer:732) INFO: 30epoch:train:562-612batch: iter_time=3.386e-04, forward_time=0.202, loss_att=24.401, acc=0.953, loss=24.401, backward_time=0.278, grad_norm=30.425, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.659e-04, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:29:23,147 (trainer:732) INFO: 30epoch:train:613-663batch: iter_time=3.657e-04, forward_time=0.201, loss_att=23.782, acc=0.954, loss=23.782, backward_time=0.276, grad_norm=27.139, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.671e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:29:55,837 (trainer:732) INFO: 30epoch:train:664-714batch: iter_time=3.757e-04, forward_time=0.204, loss_att=25.672, acc=0.950, loss=25.672, backward_time=0.280, grad_norm=29.893, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=7.684e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:30:28,219 (trainer:732) INFO: 30epoch:train:715-765batch: iter_time=3.327e-04, forward_time=0.202, loss_att=22.137, acc=0.955, loss=22.137, backward_time=0.277, grad_norm=27.907, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.697e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:31:01,232 (trainer:732) INFO: 30epoch:train:766-816batch: iter_time=3.453e-04, forward_time=0.204, loss_att=26.767, acc=0.950, loss=26.767, backward_time=0.283, grad_norm=34.732, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.710e-04, train_time=2.576
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:31:33,697 (trainer:732) INFO: 30epoch:train:817-867batch: iter_time=3.765e-04, forward_time=0.203, loss_att=24.077, acc=0.954, loss=24.077, backward_time=0.280, grad_norm=30.098, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.722e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:32:06,087 (trainer:732) INFO: 30epoch:train:868-918batch: iter_time=3.794e-04, forward_time=0.201, loss_att=25.679, acc=0.952, loss=25.679, backward_time=0.277, grad_norm=27.664, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.735e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:32:38,568 (trainer:732) INFO: 30epoch:train:919-969batch: iter_time=3.425e-04, forward_time=0.201, loss_att=25.030, acc=0.953, loss=25.030, backward_time=0.278, grad_norm=28.906, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.748e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:33:10,857 (trainer:732) INFO: 30epoch:train:970-1020batch: iter_time=3.003e-04, forward_time=0.200, loss_att=25.287, acc=0.952, loss=25.287, backward_time=0.277, grad_norm=29.745, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.761e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:43:46,903 (trainer:338) INFO: 30epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=24.667, acc=0.953, loss=24.667, backward_time=0.279, grad_norm=30.013, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.642e-04, train_time=3.130, time=13 minutes and 32.49 seconds, total_count=31110, gpu_max_cached_mem_GB=30.428, [valid] loss_att=76.688, acc=0.879, cer=0.157, wer=0.368, loss=76.688, time=4 minutes and 51.79 seconds, total_count=2640, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.42 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:43:51,486 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:43:51,495 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/20epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:43:51,496 (trainer:272) INFO: 31/60epoch started. Estimated time to finish: 11 hours, 55 minutes and 23.29 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:46:55,069 (trainer:732) INFO: 31epoch:train:1-51batch: iter_time=0.015, forward_time=0.202, loss_att=23.538, acc=0.955, loss=23.538, backward_time=0.276, grad_norm=27.887, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.778e-04, train_time=15.154
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:47:27,293 (trainer:732) INFO: 31epoch:train:52-102batch: iter_time=3.830e-04, forward_time=0.200, loss_att=23.554, acc=0.955, loss=23.554, backward_time=0.276, grad_norm=27.823, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.790e-04, train_time=2.520
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:47:59,947 (trainer:732) INFO: 31epoch:train:103-153batch: iter_time=3.328e-04, forward_time=0.202, loss_att=23.493, acc=0.955, loss=23.493, backward_time=0.281, grad_norm=29.332, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.803e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:48:32,865 (trainer:732) INFO: 31epoch:train:154-204batch: iter_time=3.254e-04, forward_time=0.202, loss_att=23.699, acc=0.956, loss=23.699, backward_time=0.281, grad_norm=32.207, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.816e-04, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:49:05,235 (trainer:732) INFO: 31epoch:train:205-255batch: iter_time=3.189e-04, forward_time=0.201, loss_att=21.792, acc=0.957, loss=21.792, backward_time=0.278, grad_norm=27.392, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.828e-04, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:49:37,918 (trainer:732) INFO: 31epoch:train:256-306batch: iter_time=3.156e-04, forward_time=0.203, loss_att=21.525, acc=0.959, loss=21.525, backward_time=0.280, grad_norm=28.067, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.841e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:50:10,489 (trainer:732) INFO: 31epoch:train:307-357batch: iter_time=3.532e-04, forward_time=0.201, loss_att=24.521, acc=0.954, loss=24.521, backward_time=0.278, grad_norm=29.918, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.854e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:50:43,307 (trainer:732) INFO: 31epoch:train:358-408batch: iter_time=3.443e-04, forward_time=0.203, loss_att=24.839, acc=0.954, loss=24.839, backward_time=0.282, grad_norm=31.798, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.867e-04, train_time=2.564
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:51:15,727 (trainer:732) INFO: 31epoch:train:409-459batch: iter_time=3.602e-04, forward_time=0.201, loss_att=24.026, acc=0.953, loss=24.026, backward_time=0.279, grad_norm=32.043, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.879e-04, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:51:47,976 (trainer:732) INFO: 31epoch:train:460-510batch: iter_time=3.203e-04, forward_time=0.200, loss_att=23.192, acc=0.955, loss=23.192, backward_time=0.276, grad_norm=31.615, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.892e-04, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:52:20,514 (trainer:732) INFO: 31epoch:train:511-561batch: iter_time=3.408e-04, forward_time=0.202, loss_att=23.851, acc=0.955, loss=23.851, backward_time=0.281, grad_norm=32.263, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=7.905e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:52:52,283 (trainer:732) INFO: 31epoch:train:562-612batch: iter_time=3.017e-04, forward_time=0.197, loss_att=21.702, acc=0.957, loss=21.702, backward_time=0.271, grad_norm=26.462, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.918e-04, train_time=2.481
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:53:24,219 (trainer:732) INFO: 31epoch:train:613-663batch: iter_time=3.596e-04, forward_time=0.201, loss_att=23.473, acc=0.955, loss=23.473, backward_time=0.276, grad_norm=28.002, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.931e-04, train_time=2.511
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:53:57,216 (trainer:732) INFO: 31epoch:train:664-714batch: iter_time=3.602e-04, forward_time=0.206, loss_att=24.023, acc=0.954, loss=24.023, backward_time=0.283, grad_norm=32.192, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=7.943e-04, train_time=2.593
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:54:29,978 (trainer:732) INFO: 31epoch:train:715-765batch: iter_time=3.318e-04, forward_time=0.204, loss_att=23.857, acc=0.955, loss=23.857, backward_time=0.282, grad_norm=28.594, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.956e-04, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:55:02,817 (trainer:732) INFO: 31epoch:train:766-816batch: iter_time=3.428e-04, forward_time=0.202, loss_att=24.819, acc=0.954, loss=24.819, backward_time=0.280, grad_norm=31.653, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.969e-04, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:55:35,459 (trainer:732) INFO: 31epoch:train:817-867batch: iter_time=3.302e-04, forward_time=0.203, loss_att=24.013, acc=0.955, loss=24.013, backward_time=0.282, grad_norm=31.453, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=7.982e-04, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:56:07,817 (trainer:732) INFO: 31epoch:train:868-918batch: iter_time=3.669e-04, forward_time=0.201, loss_att=23.611, acc=0.955, loss=23.611, backward_time=0.277, grad_norm=31.865, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.994e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:56:40,264 (trainer:732) INFO: 31epoch:train:919-969batch: iter_time=3.400e-04, forward_time=0.202, loss_att=23.942, acc=0.954, loss=23.942, backward_time=0.279, grad_norm=29.542, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.007e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 03:57:12,640 (trainer:732) INFO: 31epoch:train:970-1020batch: iter_time=3.351e-04, forward_time=0.202, loss_att=22.439, acc=0.956, loss=22.439, backward_time=0.276, grad_norm=28.364, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.020e-04, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:07:55,834 (trainer:338) INFO: 31epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=23.456, acc=0.955, loss=23.456, backward_time=0.279, grad_norm=29.932, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=7.901e-04, train_time=3.132, time=13 minutes and 33.24 seconds, total_count=32147, gpu_max_cached_mem_GB=30.428, [valid] loss_att=76.488, acc=0.880, cer=0.155, wer=0.362, loss=76.488, time=4 minutes and 54.55 seconds, total_count=2728, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 36.54 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:08:00,474 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:08:00,483 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/21epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:08:00,484 (trainer:272) INFO: 32/60epoch started. Estimated time to finish: 11 hours, 31 minutes and 49.55 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:11:05,078 (trainer:732) INFO: 32epoch:train:1-51batch: iter_time=0.016, forward_time=0.203, loss_att=21.893, acc=0.959, loss=21.893, backward_time=0.279, grad_norm=28.210, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.036e-04, train_time=15.239
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:11:37,542 (trainer:732) INFO: 32epoch:train:52-102batch: iter_time=2.825e-04, forward_time=0.201, loss_att=21.718, acc=0.960, loss=21.718, backward_time=0.280, grad_norm=29.420, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=8.049e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:12:10,026 (trainer:732) INFO: 32epoch:train:103-153batch: iter_time=3.158e-04, forward_time=0.201, loss_att=21.048, acc=0.960, loss=21.048, backward_time=0.279, grad_norm=28.191, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.062e-04, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:12:42,774 (trainer:732) INFO: 32epoch:train:154-204batch: iter_time=3.253e-04, forward_time=0.203, loss_att=22.925, acc=0.956, loss=22.925, backward_time=0.280, grad_norm=30.161, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=8.075e-04, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:13:15,353 (trainer:732) INFO: 32epoch:train:205-255batch: iter_time=3.542e-04, forward_time=0.203, loss_att=22.530, acc=0.957, loss=22.530, backward_time=0.282, grad_norm=27.947, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.087e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:13:47,759 (trainer:732) INFO: 32epoch:train:256-306batch: iter_time=3.404e-04, forward_time=0.201, loss_att=22.826, acc=0.957, loss=22.826, backward_time=0.278, grad_norm=29.466, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.100e-04, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:14:19,948 (trainer:732) INFO: 32epoch:train:307-357batch: iter_time=3.584e-04, forward_time=0.200, loss_att=22.510, acc=0.955, loss=22.510, backward_time=0.276, grad_norm=27.031, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.113e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:14:52,392 (trainer:732) INFO: 32epoch:train:358-408batch: iter_time=3.478e-04, forward_time=0.201, loss_att=21.445, acc=0.957, loss=21.445, backward_time=0.278, grad_norm=27.616, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.126e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:15:24,146 (trainer:732) INFO: 32epoch:train:409-459batch: iter_time=3.592e-04, forward_time=0.199, loss_att=21.707, acc=0.957, loss=21.707, backward_time=0.274, grad_norm=29.437, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.138e-04, train_time=2.510
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:15:56,609 (trainer:732) INFO: 32epoch:train:460-510batch: iter_time=3.569e-04, forward_time=0.202, loss_att=21.576, acc=0.958, loss=21.576, backward_time=0.279, grad_norm=27.833, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.151e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:16:29,522 (trainer:732) INFO: 32epoch:train:511-561batch: iter_time=3.317e-04, forward_time=0.204, loss_att=21.792, acc=0.959, loss=21.792, backward_time=0.282, grad_norm=27.473, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.164e-04, train_time=2.580
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:17:02,236 (trainer:732) INFO: 32epoch:train:562-612batch: iter_time=4.614e-04, forward_time=0.202, loss_att=23.903, acc=0.956, loss=23.903, backward_time=0.280, grad_norm=30.046, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.177e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:17:34,342 (trainer:732) INFO: 32epoch:train:613-663batch: iter_time=3.492e-04, forward_time=0.201, loss_att=21.348, acc=0.959, loss=21.348, backward_time=0.277, grad_norm=27.992, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.190e-04, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:18:06,821 (trainer:732) INFO: 32epoch:train:664-714batch: iter_time=3.215e-04, forward_time=0.202, loss_att=22.840, acc=0.957, loss=22.840, backward_time=0.279, grad_norm=32.855, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.202e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:18:39,214 (trainer:732) INFO: 32epoch:train:715-765batch: iter_time=3.050e-04, forward_time=0.201, loss_att=23.247, acc=0.957, loss=23.247, backward_time=0.278, grad_norm=30.198, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.215e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:19:11,589 (trainer:732) INFO: 32epoch:train:766-816batch: iter_time=3.523e-04, forward_time=0.201, loss_att=22.431, acc=0.957, loss=22.431, backward_time=0.277, grad_norm=28.937, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.228e-04, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:19:43,761 (trainer:732) INFO: 32epoch:train:817-867batch: iter_time=3.166e-04, forward_time=0.201, loss_att=22.672, acc=0.958, loss=22.672, backward_time=0.278, grad_norm=29.828, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.240e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:20:16,784 (trainer:732) INFO: 32epoch:train:868-918batch: iter_time=3.260e-04, forward_time=0.204, loss_att=23.533, acc=0.958, loss=23.533, backward_time=0.283, grad_norm=29.621, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.253e-04, train_time=2.583
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:20:49,174 (trainer:732) INFO: 32epoch:train:919-969batch: iter_time=3.431e-04, forward_time=0.201, loss_att=23.698, acc=0.955, loss=23.698, backward_time=0.278, grad_norm=31.753, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.266e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:21:21,764 (trainer:732) INFO: 32epoch:train:970-1020batch: iter_time=3.019e-04, forward_time=0.202, loss_att=22.882, acc=0.958, loss=22.882, backward_time=0.279, grad_norm=39.372, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.279e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:31:53,626 (trainer:338) INFO: 32epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=22.403, acc=0.957, loss=22.403, backward_time=0.279, grad_norm=29.706, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.160e-04, train_time=3.133, time=13 minutes and 33.24 seconds, total_count=33184, gpu_max_cached_mem_GB=30.428, [valid] loss_att=75.520, acc=0.882, cer=0.150, wer=0.365, loss=75.520, time=4 minutes and 51.03 seconds, total_count=2816, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 28.87 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:31:58,128 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:31:58,138 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/23epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:31:58,138 (trainer:272) INFO: 33/60epoch started. Estimated time to finish: 11 hours, 8 minutes and 3.69 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:34:59,999 (trainer:732) INFO: 33epoch:train:1-51batch: iter_time=0.012, forward_time=0.204, loss_att=20.555, acc=0.961, loss=20.555, backward_time=0.278, grad_norm=28.582, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.295e-04, train_time=15.012
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:35:32,390 (trainer:732) INFO: 33epoch:train:52-102batch: iter_time=3.339e-04, forward_time=0.201, loss_att=20.841, acc=0.961, loss=20.841, backward_time=0.278, grad_norm=28.320, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.308e-04, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:36:04,487 (trainer:732) INFO: 33epoch:train:103-153batch: iter_time=3.611e-04, forward_time=0.200, loss_att=20.532, acc=0.960, loss=20.532, backward_time=0.275, grad_norm=26.743, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.321e-04, train_time=2.516
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:36:37,206 (trainer:732) INFO: 33epoch:train:154-204batch: iter_time=3.303e-04, forward_time=0.201, loss_att=21.687, acc=0.959, loss=21.687, backward_time=0.279, grad_norm=29.758, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.334e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:37:09,505 (trainer:732) INFO: 33epoch:train:205-255batch: iter_time=3.717e-04, forward_time=0.202, loss_att=21.048, acc=0.961, loss=21.048, backward_time=0.279, grad_norm=30.504, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.346e-04, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:37:41,591 (trainer:732) INFO: 33epoch:train:256-306batch: iter_time=3.378e-04, forward_time=0.199, loss_att=19.676, acc=0.959, loss=19.676, backward_time=0.275, grad_norm=27.907, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.359e-04, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:38:14,171 (trainer:732) INFO: 33epoch:train:307-357batch: iter_time=3.212e-04, forward_time=0.202, loss_att=21.583, acc=0.959, loss=21.583, backward_time=0.279, grad_norm=31.998, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.372e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:38:46,438 (trainer:732) INFO: 33epoch:train:358-408batch: iter_time=3.512e-04, forward_time=0.200, loss_att=21.209, acc=0.959, loss=21.209, backward_time=0.276, grad_norm=30.846, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.385e-04, train_time=2.520
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:39:18,700 (trainer:732) INFO: 33epoch:train:409-459batch: iter_time=3.575e-04, forward_time=0.201, loss_att=20.088, acc=0.961, loss=20.088, backward_time=0.278, grad_norm=28.320, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.397e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:39:50,550 (trainer:732) INFO: 33epoch:train:460-510batch: iter_time=3.455e-04, forward_time=0.198, loss_att=19.806, acc=0.960, loss=19.806, backward_time=0.273, grad_norm=26.822, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.410e-04, train_time=2.493
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:40:23,331 (trainer:732) INFO: 33epoch:train:511-561batch: iter_time=3.311e-04, forward_time=0.204, loss_att=21.899, acc=0.959, loss=21.899, backward_time=0.282, grad_norm=28.029, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.423e-04, train_time=2.564
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:40:56,043 (trainer:732) INFO: 33epoch:train:562-612batch: iter_time=3.449e-04, forward_time=0.202, loss_att=22.168, acc=0.959, loss=22.168, backward_time=0.280, grad_norm=30.411, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.436e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:41:28,252 (trainer:732) INFO: 33epoch:train:613-663batch: iter_time=3.421e-04, forward_time=0.201, loss_att=20.066, acc=0.961, loss=20.066, backward_time=0.278, grad_norm=28.355, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.448e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:42:00,368 (trainer:732) INFO: 33epoch:train:664-714batch: iter_time=3.461e-04, forward_time=0.200, loss_att=19.882, acc=0.961, loss=19.882, backward_time=0.276, grad_norm=28.236, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.461e-04, train_time=2.515
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:42:32,722 (trainer:732) INFO: 33epoch:train:715-765batch: iter_time=3.498e-04, forward_time=0.201, loss_att=22.648, acc=0.958, loss=22.648, backward_time=0.278, grad_norm=30.488, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.474e-04, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:43:05,405 (trainer:732) INFO: 33epoch:train:766-816batch: iter_time=3.758e-04, forward_time=0.203, loss_att=20.443, acc=0.961, loss=20.443, backward_time=0.279, grad_norm=28.552, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.487e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:43:37,694 (trainer:732) INFO: 33epoch:train:817-867batch: iter_time=3.577e-04, forward_time=0.201, loss_att=22.407, acc=0.959, loss=22.407, backward_time=0.278, grad_norm=29.621, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.500e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:44:11,111 (trainer:732) INFO: 33epoch:train:868-918batch: iter_time=3.653e-04, forward_time=0.207, loss_att=22.686, acc=0.959, loss=22.686, backward_time=0.289, grad_norm=29.717, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.512e-04, train_time=2.613
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:44:44,219 (trainer:732) INFO: 33epoch:train:919-969batch: iter_time=3.438e-04, forward_time=0.205, loss_att=22.643, acc=0.959, loss=22.643, backward_time=0.285, grad_norm=30.844, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.525e-04, train_time=2.598
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:45:16,761 (trainer:732) INFO: 33epoch:train:970-1020batch: iter_time=3.091e-04, forward_time=0.202, loss_att=22.210, acc=0.959, loss=22.210, backward_time=0.279, grad_norm=29.141, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.538e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:55:50,867 (trainer:338) INFO: 33epoch results: [train] iter_time=9.030e-04, forward_time=0.202, loss_att=21.175, acc=0.960, loss=21.175, backward_time=0.279, grad_norm=29.161, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.419e-04, train_time=3.123, time=13 minutes and 30.72 seconds, total_count=34221, gpu_max_cached_mem_GB=30.428, [valid] loss_att=77.639, acc=0.880, cer=0.148, wer=0.366, loss=77.639, time=4 minutes and 50.62 seconds, total_count=2904, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 31.38 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:55:55,376 (trainer:384) INFO: There are no improvements in this epoch
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:55:55,385 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/22epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:55:55,385 (trainer:272) INFO: 34/60epoch started. Estimated time to finish: 10 hours, 44 minutes and 16.78 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:58:59,361 (trainer:732) INFO: 34epoch:train:1-51batch: iter_time=0.017, forward_time=0.204, loss_att=20.837, acc=0.961, loss=20.837, backward_time=0.279, grad_norm=29.280, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.554e-04, train_time=15.181
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 04:59:31,886 (trainer:732) INFO: 34epoch:train:52-102batch: iter_time=3.482e-04, forward_time=0.202, loss_att=20.410, acc=0.962, loss=20.410, backward_time=0.280, grad_norm=30.390, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.567e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:00:04,263 (trainer:732) INFO: 34epoch:train:103-153batch: iter_time=3.674e-04, forward_time=0.201, loss_att=20.689, acc=0.961, loss=20.689, backward_time=0.277, grad_norm=28.335, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.580e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:00:36,691 (trainer:732) INFO: 34epoch:train:154-204batch: iter_time=3.149e-04, forward_time=0.201, loss_att=20.406, acc=0.961, loss=20.406, backward_time=0.278, grad_norm=27.056, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.593e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:01:08,794 (trainer:732) INFO: 34epoch:train:205-255batch: iter_time=3.325e-04, forward_time=0.200, loss_att=20.466, acc=0.961, loss=20.466, backward_time=0.277, grad_norm=26.556, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.605e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:01:41,099 (trainer:732) INFO: 34epoch:train:256-306batch: iter_time=3.269e-04, forward_time=0.201, loss_att=19.953, acc=0.962, loss=19.953, backward_time=0.277, grad_norm=27.869, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.618e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:02:13,701 (trainer:732) INFO: 34epoch:train:307-357batch: iter_time=3.459e-04, forward_time=0.202, loss_att=19.896, acc=0.962, loss=19.896, backward_time=0.279, grad_norm=27.237, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.631e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:02:46,301 (trainer:732) INFO: 34epoch:train:358-408batch: iter_time=3.428e-04, forward_time=0.202, loss_att=19.814, acc=0.962, loss=19.814, backward_time=0.278, grad_norm=28.101, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.644e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:03:18,490 (trainer:732) INFO: 34epoch:train:409-459batch: iter_time=3.640e-04, forward_time=0.201, loss_att=20.506, acc=0.960, loss=20.506, backward_time=0.278, grad_norm=28.929, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.656e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:03:50,558 (trainer:732) INFO: 34epoch:train:460-510batch: iter_time=3.500e-04, forward_time=0.201, loss_att=18.964, acc=0.961, loss=18.964, backward_time=0.274, grad_norm=26.989, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.669e-04, train_time=2.511
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:04:23,436 (trainer:732) INFO: 34epoch:train:511-561batch: iter_time=3.754e-04, forward_time=0.204, loss_att=20.345, acc=0.962, loss=20.345, backward_time=0.283, grad_norm=30.520, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.682e-04, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:04:56,170 (trainer:732) INFO: 34epoch:train:562-612batch: iter_time=3.395e-04, forward_time=0.203, loss_att=19.969, acc=0.962, loss=19.969, backward_time=0.279, grad_norm=27.079, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.695e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:05:28,467 (trainer:732) INFO: 34epoch:train:613-663batch: iter_time=3.315e-04, forward_time=0.202, loss_att=20.245, acc=0.962, loss=20.245, backward_time=0.279, grad_norm=28.007, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.707e-04, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:06:00,889 (trainer:732) INFO: 34epoch:train:664-714batch: iter_time=3.489e-04, forward_time=0.201, loss_att=19.395, acc=0.962, loss=19.395, backward_time=0.276, grad_norm=29.217, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.720e-04, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:06:33,475 (trainer:732) INFO: 34epoch:train:715-765batch: iter_time=3.070e-04, forward_time=0.203, loss_att=19.882, acc=0.962, loss=19.882, backward_time=0.280, grad_norm=30.150, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.733e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:07:06,446 (trainer:732) INFO: 34epoch:train:766-816batch: iter_time=3.476e-04, forward_time=0.204, loss_att=20.982, acc=0.961, loss=20.982, backward_time=0.283, grad_norm=29.715, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.746e-04, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:07:38,400 (trainer:732) INFO: 34epoch:train:817-867batch: iter_time=3.314e-04, forward_time=0.199, loss_att=19.644, acc=0.962, loss=19.644, backward_time=0.275, grad_norm=26.772, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.759e-04, train_time=2.521
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:08:10,785 (trainer:732) INFO: 34epoch:train:868-918batch: iter_time=2.858e-04, forward_time=0.201, loss_att=21.335, acc=0.961, loss=21.335, backward_time=0.280, grad_norm=29.140, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=8.771e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:08:43,706 (trainer:732) INFO: 34epoch:train:919-969batch: iter_time=3.253e-04, forward_time=0.203, loss_att=20.319, acc=0.962, loss=20.319, backward_time=0.283, grad_norm=34.916, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.784e-04, train_time=2.583
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:09:16,390 (trainer:732) INFO: 34epoch:train:970-1020batch: iter_time=3.155e-04, forward_time=0.203, loss_att=22.317, acc=0.959, loss=22.317, backward_time=0.280, grad_norm=28.942, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.797e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:19:49,182 (trainer:338) INFO: 34epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=20.284, acc=0.961, loss=20.284, backward_time=0.279, grad_norm=28.758, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.678e-04, train_time=3.131, time=13 minutes and 32.9 seconds, total_count=35258, gpu_max_cached_mem_GB=30.428, [valid] loss_att=77.521, acc=0.882, cer=0.149, wer=0.361, loss=77.521, time=4 minutes and 51.47 seconds, total_count=2992, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 29.43 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:19:53,569 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:19:53,584 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/24epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:19:53,585 (trainer:272) INFO: 35/60epoch started. Estimated time to finish: 10 hours, 20 minutes and 30 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:22:58,522 (trainer:732) INFO: 35epoch:train:1-51batch: iter_time=0.010, forward_time=0.207, loss_att=20.183, acc=0.963, loss=20.183, backward_time=0.282, grad_norm=28.898, clip=100.000, loss_scale=1.000, optim_step_time=0.068, optim0_lr0=8.813e-04, train_time=15.260
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:23:31,333 (trainer:732) INFO: 35epoch:train:52-102batch: iter_time=3.558e-04, forward_time=0.202, loss_att=20.078, acc=0.963, loss=20.078, backward_time=0.280, grad_norm=28.259, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.826e-04, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:24:03,838 (trainer:732) INFO: 35epoch:train:103-153batch: iter_time=3.879e-04, forward_time=0.202, loss_att=18.127, acc=0.965, loss=18.127, backward_time=0.279, grad_norm=26.038, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.839e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:24:36,375 (trainer:732) INFO: 35epoch:train:154-204batch: iter_time=3.777e-04, forward_time=0.202, loss_att=19.555, acc=0.963, loss=19.555, backward_time=0.279, grad_norm=28.617, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.852e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:25:08,667 (trainer:732) INFO: 35epoch:train:205-255batch: iter_time=3.863e-04, forward_time=0.202, loss_att=20.827, acc=0.960, loss=20.827, backward_time=0.278, grad_norm=29.357, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.864e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:25:40,885 (trainer:732) INFO: 35epoch:train:256-306batch: iter_time=3.698e-04, forward_time=0.201, loss_att=19.139, acc=0.963, loss=19.139, backward_time=0.277, grad_norm=26.691, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.877e-04, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:26:13,164 (trainer:732) INFO: 35epoch:train:307-357batch: iter_time=3.939e-04, forward_time=0.201, loss_att=18.878, acc=0.963, loss=18.878, backward_time=0.277, grad_norm=26.984, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.890e-04, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:26:45,225 (trainer:732) INFO: 35epoch:train:358-408batch: iter_time=3.399e-04, forward_time=0.198, loss_att=19.001, acc=0.963, loss=19.001, backward_time=0.275, grad_norm=25.345, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=8.903e-04, train_time=2.503
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:27:17,773 (trainer:732) INFO: 35epoch:train:409-459batch: iter_time=3.873e-04, forward_time=0.203, loss_att=20.820, acc=0.961, loss=20.820, backward_time=0.281, grad_norm=29.254, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.915e-04, train_time=2.564
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:27:50,357 (trainer:732) INFO: 35epoch:train:460-510batch: iter_time=3.496e-04, forward_time=0.202, loss_att=18.677, acc=0.964, loss=18.677, backward_time=0.279, grad_norm=28.038, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.928e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:28:23,177 (trainer:732) INFO: 35epoch:train:511-561batch: iter_time=3.604e-04, forward_time=0.204, loss_att=20.919, acc=0.962, loss=20.919, backward_time=0.283, grad_norm=30.553, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.941e-04, train_time=2.576
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:28:55,982 (trainer:732) INFO: 35epoch:train:562-612batch: iter_time=3.719e-04, forward_time=0.203, loss_att=20.211, acc=0.963, loss=20.211, backward_time=0.281, grad_norm=28.165, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.954e-04, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:29:28,008 (trainer:732) INFO: 35epoch:train:613-663batch: iter_time=3.641e-04, forward_time=0.201, loss_att=19.302, acc=0.964, loss=19.302, backward_time=0.277, grad_norm=29.081, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=8.967e-04, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:30:00,954 (trainer:732) INFO: 35epoch:train:664-714batch: iter_time=3.386e-04, forward_time=0.204, loss_att=18.365, acc=0.965, loss=18.365, backward_time=0.283, grad_norm=26.514, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.979e-04, train_time=2.584
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:30:32,924 (trainer:732) INFO: 35epoch:train:715-765batch: iter_time=3.307e-04, forward_time=0.198, loss_att=18.140, acc=0.964, loss=18.140, backward_time=0.273, grad_norm=27.367, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.992e-04, train_time=2.506
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:31:05,424 (trainer:732) INFO: 35epoch:train:766-816batch: iter_time=3.974e-04, forward_time=0.202, loss_att=19.827, acc=0.963, loss=19.827, backward_time=0.278, grad_norm=29.758, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.005e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:31:37,636 (trainer:732) INFO: 35epoch:train:817-867batch: iter_time=3.719e-04, forward_time=0.202, loss_att=19.611, acc=0.963, loss=19.611, backward_time=0.278, grad_norm=28.411, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.017e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:32:10,311 (trainer:732) INFO: 35epoch:train:868-918batch: iter_time=3.909e-04, forward_time=0.204, loss_att=20.385, acc=0.963, loss=20.385, backward_time=0.282, grad_norm=30.443, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.030e-04, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:32:42,743 (trainer:732) INFO: 35epoch:train:919-969batch: iter_time=3.844e-04, forward_time=0.202, loss_att=18.535, acc=0.964, loss=18.535, backward_time=0.278, grad_norm=28.574, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.043e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:33:15,254 (trainer:732) INFO: 35epoch:train:970-1020batch: iter_time=3.479e-04, forward_time=0.202, loss_att=18.093, acc=0.964, loss=18.093, backward_time=0.277, grad_norm=25.992, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.056e-04, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:43:54,440 (trainer:338) INFO: 35epoch results: [train] iter_time=8.569e-04, forward_time=0.202, loss_att=19.401, acc=0.963, loss=19.401, backward_time=0.279, grad_norm=28.096, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.937e-04, train_time=3.134, time=13 minutes and 33.57 seconds, total_count=36295, gpu_max_cached_mem_GB=30.428, [valid] loss_att=76.748, acc=0.882, cer=0.152, wer=0.359, loss=76.748, time=4 minutes and 50.54 seconds, total_count=3080, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 36.75 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:43:59,134 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:43:59,148 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/25epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:43:59,149 (trainer:272) INFO: 36/60epoch started. Estimated time to finish: 9 hours, 56 minutes and 47.82 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:47:01,470 (trainer:732) INFO: 36epoch:train:1-51batch: iter_time=0.011, forward_time=0.204, loss_att=18.771, acc=0.965, loss=18.771, backward_time=0.279, grad_norm=28.064, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=9.073e-04, train_time=15.047
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:47:33,805 (trainer:732) INFO: 36epoch:train:52-102batch: iter_time=3.457e-04, forward_time=0.202, loss_att=17.723, acc=0.966, loss=17.723, backward_time=0.278, grad_norm=25.801, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.085e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:48:06,422 (trainer:732) INFO: 36epoch:train:103-153batch: iter_time=3.452e-04, forward_time=0.202, loss_att=18.384, acc=0.965, loss=18.384, backward_time=0.280, grad_norm=28.381, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.098e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:48:38,790 (trainer:732) INFO: 36epoch:train:154-204batch: iter_time=3.544e-04, forward_time=0.201, loss_att=17.904, acc=0.966, loss=17.904, backward_time=0.277, grad_norm=28.518, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.111e-04, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:49:11,046 (trainer:732) INFO: 36epoch:train:205-255batch: iter_time=3.682e-04, forward_time=0.201, loss_att=18.213, acc=0.965, loss=18.213, backward_time=0.279, grad_norm=26.754, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.123e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:49:43,354 (trainer:732) INFO: 36epoch:train:256-306batch: iter_time=3.374e-04, forward_time=0.200, loss_att=18.574, acc=0.964, loss=18.574, backward_time=0.277, grad_norm=29.756, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.136e-04, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:50:15,651 (trainer:732) INFO: 36epoch:train:307-357batch: iter_time=3.590e-04, forward_time=0.200, loss_att=17.976, acc=0.965, loss=17.976, backward_time=0.277, grad_norm=26.711, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.149e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:50:48,331 (trainer:732) INFO: 36epoch:train:358-408batch: iter_time=3.277e-04, forward_time=0.202, loss_att=18.048, acc=0.966, loss=18.048, backward_time=0.280, grad_norm=26.923, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.162e-04, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:51:20,472 (trainer:732) INFO: 36epoch:train:409-459batch: iter_time=3.363e-04, forward_time=0.201, loss_att=18.387, acc=0.965, loss=18.387, backward_time=0.278, grad_norm=29.149, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.174e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:51:52,727 (trainer:732) INFO: 36epoch:train:460-510batch: iter_time=3.409e-04, forward_time=0.200, loss_att=18.198, acc=0.965, loss=18.198, backward_time=0.275, grad_norm=28.695, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.187e-04, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:52:25,405 (trainer:732) INFO: 36epoch:train:511-561batch: iter_time=3.398e-04, forward_time=0.202, loss_att=18.923, acc=0.965, loss=18.923, backward_time=0.281, grad_norm=30.285, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.200e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:52:58,257 (trainer:732) INFO: 36epoch:train:562-612batch: iter_time=3.449e-04, forward_time=0.203, loss_att=18.320, acc=0.964, loss=18.320, backward_time=0.281, grad_norm=29.956, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.213e-04, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:53:30,467 (trainer:732) INFO: 36epoch:train:613-663batch: iter_time=3.476e-04, forward_time=0.201, loss_att=18.195, acc=0.965, loss=18.195, backward_time=0.278, grad_norm=29.966, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.225e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:54:03,624 (trainer:732) INFO: 36epoch:train:664-714batch: iter_time=3.447e-04, forward_time=0.206, loss_att=19.626, acc=0.964, loss=19.626, backward_time=0.286, grad_norm=29.929, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.238e-04, train_time=2.588
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:54:35,970 (trainer:732) INFO: 36epoch:train:715-765batch: iter_time=3.023e-04, forward_time=0.201, loss_att=18.130, acc=0.965, loss=18.130, backward_time=0.277, grad_norm=27.193, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.251e-04, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:55:08,403 (trainer:732) INFO: 36epoch:train:766-816batch: iter_time=3.413e-04, forward_time=0.200, loss_att=18.136, acc=0.964, loss=18.136, backward_time=0.276, grad_norm=28.333, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.264e-04, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:55:41,276 (trainer:732) INFO: 36epoch:train:817-867batch: iter_time=3.251e-04, forward_time=0.204, loss_att=19.312, acc=0.965, loss=19.312, backward_time=0.283, grad_norm=28.363, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.276e-04, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:56:13,744 (trainer:732) INFO: 36epoch:train:868-918batch: iter_time=3.648e-04, forward_time=0.201, loss_att=19.080, acc=0.964, loss=19.080, backward_time=0.279, grad_norm=29.189, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.289e-04, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:56:46,123 (trainer:732) INFO: 36epoch:train:919-969batch: iter_time=3.217e-04, forward_time=0.201, loss_att=19.645, acc=0.963, loss=19.645, backward_time=0.278, grad_norm=28.965, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.302e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 05:57:18,630 (trainer:732) INFO: 36epoch:train:970-1020batch: iter_time=3.131e-04, forward_time=0.201, loss_att=18.463, acc=0.963, loss=18.463, backward_time=0.278, grad_norm=29.829, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=9.315e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:07:49,956 (trainer:338) INFO: 36epoch results: [train] iter_time=8.731e-04, forward_time=0.202, loss_att=18.473, acc=0.965, loss=18.473, backward_time=0.279, grad_norm=28.499, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.196e-04, train_time=3.125, time=13 minutes and 31.14 seconds, total_count=37332, gpu_max_cached_mem_GB=30.428, [valid] loss_att=75.176, acc=0.884, cer=0.151, wer=0.355, loss=75.176, time=4 minutes and 52.51 seconds, total_count=3168, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 27.16 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:07:54,286 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:07:54,296 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/27epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:07:54,297 (trainer:272) INFO: 37/60epoch started. Estimated time to finish: 9 hours, 32 minutes and 57.39 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:10:56,151 (trainer:732) INFO: 37epoch:train:1-51batch: iter_time=0.010, forward_time=0.202, loss_att=16.777, acc=0.968, loss=16.777, backward_time=0.275, grad_norm=27.325, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.331e-04, train_time=15.010
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:11:29,121 (trainer:732) INFO: 37epoch:train:52-102batch: iter_time=3.198e-04, forward_time=0.204, loss_att=17.498, acc=0.968, loss=17.498, backward_time=0.284, grad_norm=27.827, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.344e-04, train_time=2.586
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:12:01,855 (trainer:732) INFO: 37epoch:train:103-153batch: iter_time=3.457e-04, forward_time=0.203, loss_att=18.033, acc=0.966, loss=18.033, backward_time=0.282, grad_norm=28.304, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.357e-04, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:12:34,317 (trainer:732) INFO: 37epoch:train:154-204batch: iter_time=3.284e-04, forward_time=0.199, loss_att=16.784, acc=0.968, loss=16.784, backward_time=0.277, grad_norm=27.594, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.370e-04, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:13:06,678 (trainer:732) INFO: 37epoch:train:205-255batch: iter_time=3.635e-04, forward_time=0.202, loss_att=17.794, acc=0.966, loss=17.794, backward_time=0.279, grad_norm=29.747, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.382e-04, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:13:39,242 (trainer:732) INFO: 37epoch:train:256-306batch: iter_time=3.401e-04, forward_time=0.202, loss_att=17.250, acc=0.968, loss=17.250, backward_time=0.280, grad_norm=27.595, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.395e-04, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:14:11,368 (trainer:732) INFO: 37epoch:train:307-357batch: iter_time=3.493e-04, forward_time=0.199, loss_att=17.429, acc=0.967, loss=17.429, backward_time=0.275, grad_norm=26.532, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.408e-04, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:14:43,588 (trainer:732) INFO: 37epoch:train:358-408batch: iter_time=3.460e-04, forward_time=0.201, loss_att=16.785, acc=0.967, loss=16.785, backward_time=0.275, grad_norm=24.893, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.421e-04, train_time=2.517
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:15:16,124 (trainer:732) INFO: 37epoch:train:409-459batch: iter_time=3.537e-04, forward_time=0.202, loss_att=17.067, acc=0.966, loss=17.067, backward_time=0.279, grad_norm=27.679, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.433e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:15:49,017 (trainer:732) INFO: 37epoch:train:460-510batch: iter_time=3.383e-04, forward_time=0.204, loss_att=18.431, acc=0.965, loss=18.431, backward_time=0.283, grad_norm=27.941, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.446e-04, train_time=2.596
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:16:22,065 (trainer:732) INFO: 37epoch:train:511-561batch: iter_time=3.318e-04, forward_time=0.204, loss_att=19.508, acc=0.965, loss=19.508, backward_time=0.284, grad_norm=29.377, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.459e-04, train_time=2.589
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:16:54,412 (trainer:732) INFO: 37epoch:train:562-612batch: iter_time=2.954e-04, forward_time=0.201, loss_att=17.787, acc=0.967, loss=17.787, backward_time=0.278, grad_norm=26.996, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.472e-04, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:17:26,629 (trainer:732) INFO: 37epoch:train:613-663batch: iter_time=3.607e-04, forward_time=0.201, loss_att=16.946, acc=0.967, loss=16.946, backward_time=0.276, grad_norm=30.364, clip=100.000, loss_scale=1.000, optim_step_time=0.067, optim0_lr0=9.484e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:17:59,005 (trainer:732) INFO: 37epoch:train:664-714batch: iter_time=3.291e-04, forward_time=0.201, loss_att=17.926, acc=0.966, loss=17.926, backward_time=0.277, grad_norm=27.628, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.497e-04, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:18:31,585 (trainer:732) INFO: 37epoch:train:715-765batch: iter_time=3.344e-04, forward_time=0.203, loss_att=19.797, acc=0.964, loss=19.797, backward_time=0.280, grad_norm=29.766, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.510e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:19:03,813 (trainer:732) INFO: 37epoch:train:766-816batch: iter_time=3.354e-04, forward_time=0.200, loss_att=16.459, acc=0.968, loss=16.459, backward_time=0.277, grad_norm=26.905, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.523e-04, train_time=2.515
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:19:36,117 (trainer:732) INFO: 37epoch:train:817-867batch: iter_time=3.332e-04, forward_time=0.201, loss_att=17.869, acc=0.966, loss=17.869, backward_time=0.278, grad_norm=28.447, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.535e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:20:09,018 (trainer:732) INFO: 37epoch:train:868-918batch: iter_time=3.604e-04, forward_time=0.204, loss_att=18.324, acc=0.966, loss=18.324, backward_time=0.283, grad_norm=30.207, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=9.548e-04, train_time=2.576
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:20:41,311 (trainer:732) INFO: 37epoch:train:919-969batch: iter_time=3.429e-04, forward_time=0.200, loss_att=17.990, acc=0.966, loss=17.990, backward_time=0.276, grad_norm=27.497, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.561e-04, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:21:13,693 (trainer:732) INFO: 37epoch:train:970-1020batch: iter_time=3.010e-04, forward_time=0.200, loss_att=16.978, acc=0.967, loss=16.978, backward_time=0.277, grad_norm=26.920, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.574e-04, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:31:36,085 (trainer:338) INFO: 37epoch results: [train] iter_time=8.042e-04, forward_time=0.202, loss_att=17.636, acc=0.966, loss=17.636, backward_time=0.279, grad_norm=27.957, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.455e-04, train_time=3.125, time=13 minutes and 31.35 seconds, total_count=38369, gpu_max_cached_mem_GB=30.428, [valid] loss_att=76.367, acc=0.884, cer=0.144, wer=0.355, loss=76.367, time=4 minutes and 40.97 seconds, total_count=3256, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 29.47 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:31:40,657 (trainer:384) INFO: There are no improvements in this epoch
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:31:40,671 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/26epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:31:40,671 (trainer:272) INFO: 38/60epoch started. Estimated time to finish: 9 hours, 9 minutes and 1.26 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:34:43,714 (trainer:732) INFO: 38epoch:train:1-51batch: iter_time=0.018, forward_time=0.203, loss_att=16.895, acc=0.968, loss=16.895, backward_time=0.279, grad_norm=26.904, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=9.591e-04, train_time=15.109
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:35:16,229 (trainer:732) INFO: 38epoch:train:52-102batch: iter_time=3.537e-04, forward_time=0.202, loss_att=16.914, acc=0.968, loss=16.914, backward_time=0.278, grad_norm=31.446, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=9.603e-04, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:35:48,888 (trainer:732) INFO: 38epoch:train:103-153batch: iter_time=3.647e-04, forward_time=0.202, loss_att=16.031, acc=0.970, loss=16.031, backward_time=0.279, grad_norm=28.223, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.616e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:36:21,633 (trainer:732) INFO: 38epoch:train:154-204batch: iter_time=3.252e-04, forward_time=0.202, loss_att=16.404, acc=0.968, loss=16.404, backward_time=0.281, grad_norm=30.763, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.629e-04, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:36:54,272 (trainer:732) INFO: 38epoch:train:205-255batch: iter_time=3.582e-04, forward_time=0.203, loss_att=17.822, acc=0.968, loss=17.822, backward_time=0.282, grad_norm=30.295, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.641e-04, train_time=2.578
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:37:26,980 (trainer:732) INFO: 38epoch:train:256-306batch: iter_time=3.198e-04, forward_time=0.202, loss_att=17.202, acc=0.968, loss=17.202, backward_time=0.280, grad_norm=26.920, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.654e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:37:59,455 (trainer:732) INFO: 38epoch:train:307-357batch: iter_time=3.664e-04, forward_time=0.202, loss_att=16.033, acc=0.969, loss=16.033, backward_time=0.280, grad_norm=26.712, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.667e-04, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:38:32,112 (trainer:732) INFO: 38epoch:train:358-408batch: iter_time=3.066e-04, forward_time=0.203, loss_att=18.014, acc=0.966, loss=18.014, backward_time=0.279, grad_norm=28.738, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.680e-04, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:39:04,342 (trainer:732) INFO: 38epoch:train:409-459batch: iter_time=3.575e-04, forward_time=0.200, loss_att=16.574, acc=0.968, loss=16.574, backward_time=0.277, grad_norm=28.805, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=9.692e-04, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:39:36,863 (trainer:732) INFO: 38epoch:train:460-510batch: iter_time=4.275e-04, forward_time=0.202, loss_att=16.322, acc=0.968, loss=16.322, backward_time=0.278, grad_norm=25.982, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.705e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:40:09,445 (trainer:732) INFO: 38epoch:train:511-561batch: iter_time=3.544e-04, forward_time=0.201, loss_att=17.371, acc=0.967, loss=17.371, backward_time=0.279, grad_norm=26.701, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.718e-04, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:40:42,117 (trainer:732) INFO: 38epoch:train:562-612batch: iter_time=3.050e-04, forward_time=0.202, loss_att=16.782, acc=0.968, loss=16.782, backward_time=0.280, grad_norm=26.290, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.731e-04, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:41:14,280 (trainer:732) INFO: 38epoch:train:613-663batch: iter_time=3.660e-04, forward_time=0.201, loss_att=16.001, acc=0.969, loss=16.001, backward_time=0.277, grad_norm=25.721, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.744e-04, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:41:47,167 (trainer:732) INFO: 38epoch:train:664-714batch: iter_time=3.327e-04, forward_time=0.204, loss_att=17.797, acc=0.966, loss=17.797, backward_time=0.283, grad_norm=29.400, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.756e-04, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:42:19,399 (trainer:732) INFO: 38epoch:train:715-765batch: iter_time=2.873e-04, forward_time=0.200, loss_att=17.541, acc=0.966, loss=17.541, backward_time=0.277, grad_norm=27.425, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.769e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:42:51,712 (trainer:732) INFO: 38epoch:train:766-816batch: iter_time=3.482e-04, forward_time=0.201, loss_att=15.735, acc=0.969, loss=15.735, backward_time=0.277, grad_norm=24.753, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.782e-04, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:43:24,058 (trainer:732) INFO: 38epoch:train:817-867batch: iter_time=3.409e-04, forward_time=0.202, loss_att=16.895, acc=0.968, loss=16.895, backward_time=0.278, grad_norm=28.346, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=9.794e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:43:56,410 (trainer:732) INFO: 38epoch:train:868-918batch: iter_time=3.626e-04, forward_time=0.201, loss_att=17.884, acc=0.966, loss=17.884, backward_time=0.279, grad_norm=30.278, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.807e-04, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:44:28,755 (trainer:732) INFO: 38epoch:train:919-969batch: iter_time=3.207e-04, forward_time=0.200, loss_att=17.097, acc=0.968, loss=17.097, backward_time=0.277, grad_norm=27.996, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.820e-04, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:45:00,803 (trainer:732) INFO: 38epoch:train:970-1020batch: iter_time=2.815e-04, forward_time=0.199, loss_att=16.157, acc=0.968, loss=16.157, backward_time=0.274, grad_norm=25.227, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.833e-04, train_time=2.503

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 99)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946847:946957 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 99)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:55:33,264 (trainer:338) INFO: 38epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=16.845, acc=0.968, loss=16.845, backward_time=0.279, grad_norm=27.818, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.714e-04, train_time=3.128, time=13 minutes and 32.34 seconds, total_count=39406, gpu_max_cached_mem_GB=30.428, [valid] loss_att=76.250, acc=0.885, cer=0.154, wer=0.353, loss=76.250, time=4 minutes and 49.9 seconds, total_count=3344, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 30.35 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:55:37,657 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:55:37,668 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/28epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:55:37,669 (trainer:272) INFO: 39/60epoch started. Estimated time to finish: 8 hours, 45 minutes and 11.79 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:58:43,319 (trainer:732) INFO: 39epoch:train:1-51batch: iter_time=0.011, forward_time=0.207, loss_att=16.558, acc=0.969, loss=16.558, backward_time=0.284, grad_norm=30.134, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=9.850e-04, train_time=15.318
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:59:15,877 (trainer:732) INFO: 39epoch:train:52-102batch: iter_time=2.912e-04, forward_time=0.202, loss_att=16.446, acc=0.970, loss=16.446, backward_time=0.279, grad_norm=25.435, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.862e-04, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 06:59:48,516 (trainer:732) INFO: 39epoch:train:103-153batch: iter_time=3.595e-04, forward_time=0.203, loss_att=15.602, acc=0.970, loss=15.602, backward_time=0.280, grad_norm=27.186, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.875e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:00:20,780 (trainer:732) INFO: 39epoch:train:154-204batch: iter_time=3.333e-04, forward_time=0.200, loss_att=15.888, acc=0.969, loss=15.888, backward_time=0.276, grad_norm=26.636, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.888e-04, train_time=2.520
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:00:52,782 (trainer:732) INFO: 39epoch:train:205-255batch: iter_time=3.190e-04, forward_time=0.200, loss_att=17.449, acc=0.967, loss=17.449, backward_time=0.278, grad_norm=26.433, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=9.900e-04, train_time=2.516
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:01:25,292 (trainer:732) INFO: 39epoch:train:256-306batch: iter_time=3.353e-04, forward_time=0.202, loss_att=15.876, acc=0.970, loss=15.876, backward_time=0.279, grad_norm=26.841, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.913e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:01:57,734 (trainer:732) INFO: 39epoch:train:307-357batch: iter_time=3.604e-04, forward_time=0.202, loss_att=16.529, acc=0.968, loss=16.529, backward_time=0.278, grad_norm=28.333, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.926e-04, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:02:30,422 (trainer:732) INFO: 39epoch:train:358-408batch: iter_time=3.324e-04, forward_time=0.202, loss_att=16.013, acc=0.969, loss=16.013, backward_time=0.280, grad_norm=26.655, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.939e-04, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:03:02,464 (trainer:732) INFO: 39epoch:train:409-459batch: iter_time=3.585e-04, forward_time=0.200, loss_att=14.802, acc=0.971, loss=14.802, backward_time=0.276, grad_norm=25.996, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.951e-04, train_time=2.520
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:03:35,009 (trainer:732) INFO: 39epoch:train:460-510batch: iter_time=3.210e-04, forward_time=0.201, loss_att=16.342, acc=0.969, loss=16.342, backward_time=0.279, grad_norm=26.625, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.964e-04, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:04:07,130 (trainer:732) INFO: 39epoch:train:511-561batch: iter_time=3.538e-04, forward_time=0.200, loss_att=15.156, acc=0.970, loss=15.156, backward_time=0.276, grad_norm=24.049, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=9.977e-04, train_time=2.514
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:04:39,808 (trainer:732) INFO: 39epoch:train:562-612batch: iter_time=3.227e-04, forward_time=0.201, loss_att=16.985, acc=0.969, loss=16.985, backward_time=0.279, grad_norm=28.060, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.990e-04, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:05:12,173 (trainer:732) INFO: 39epoch:train:613-663batch: iter_time=3.572e-04, forward_time=0.202, loss_att=17.082, acc=0.968, loss=17.082, backward_time=0.279, grad_norm=28.357, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:05:44,925 (trainer:732) INFO: 39epoch:train:664-714batch: iter_time=3.370e-04, forward_time=0.202, loss_att=16.463, acc=0.969, loss=16.463, backward_time=0.282, grad_norm=27.850, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:06:17,133 (trainer:732) INFO: 39epoch:train:715-765batch: iter_time=3.094e-04, forward_time=0.200, loss_att=16.751, acc=0.968, loss=16.751, backward_time=0.277, grad_norm=28.435, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:06:49,830 (trainer:732) INFO: 39epoch:train:766-816batch: iter_time=2.973e-04, forward_time=0.202, loss_att=17.184, acc=0.969, loss=17.184, backward_time=0.281, grad_norm=26.491, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:07:21,738 (trainer:732) INFO: 39epoch:train:817-867batch: iter_time=3.559e-04, forward_time=0.200, loss_att=16.124, acc=0.967, loss=16.124, backward_time=0.275, grad_norm=25.448, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.519
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:07:54,151 (trainer:732) INFO: 39epoch:train:868-918batch: iter_time=3.405e-04, forward_time=0.202, loss_att=15.974, acc=0.969, loss=15.974, backward_time=0.279, grad_norm=25.547, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:08:26,836 (trainer:732) INFO: 39epoch:train:919-969batch: iter_time=3.405e-04, forward_time=0.203, loss_att=17.008, acc=0.967, loss=17.008, backward_time=0.280, grad_norm=30.690, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:08:59,353 (trainer:732) INFO: 39epoch:train:970-1020batch: iter_time=3.076e-04, forward_time=0.201, loss_att=16.310, acc=0.969, loss=16.310, backward_time=0.278, grad_norm=27.667, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:19:31,933 (trainer:338) INFO: 39epoch results: [train] iter_time=8.523e-04, forward_time=0.202, loss_att=16.315, acc=0.969, loss=16.315, backward_time=0.279, grad_norm=27.200, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.973e-04, train_time=3.134, time=13 minutes and 33.62 seconds, total_count=40443, gpu_max_cached_mem_GB=30.428, [valid] loss_att=74.945, acc=0.887, cer=0.144, wer=0.347, loss=74.945, time=4 minutes and 53.15 seconds, total_count=3432, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 27.49 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:19:36,049 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:19:36,061 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/29epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:19:36,062 (trainer:272) INFO: 40/60epoch started. Estimated time to finish: 8 hours, 21 minutes and 22.69 seconds

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 111)

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory'
de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] NCCL INFO bootstrap.cc:231 -> 1

de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:946846:946958 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 111)
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:22:39,217 (trainer:732) INFO: 40epoch:train:1-51batch: iter_time=0.012, forward_time=0.201, loss_att=16.215, acc=0.969, loss=16.215, backward_time=0.277, grad_norm=30.596, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=15.114
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:23:11,653 (trainer:732) INFO: 40epoch:train:52-102batch: iter_time=3.372e-04, forward_time=0.202, loss_att=15.711, acc=0.971, loss=15.711, backward_time=0.280, grad_norm=27.513, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:23:44,452 (trainer:732) INFO: 40epoch:train:103-153batch: iter_time=3.757e-04, forward_time=0.203, loss_att=15.689, acc=0.970, loss=15.689, backward_time=0.282, grad_norm=26.604, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.572
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:24:17,216 (trainer:732) INFO: 40epoch:train:154-204batch: iter_time=3.376e-04, forward_time=0.202, loss_att=15.554, acc=0.971, loss=15.554, backward_time=0.280, grad_norm=26.371, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:24:49,825 (trainer:732) INFO: 40epoch:train:205-255batch: iter_time=3.561e-04, forward_time=0.204, loss_att=15.200, acc=0.971, loss=15.200, backward_time=0.282, grad_norm=28.578, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:25:22,291 (trainer:732) INFO: 40epoch:train:256-306batch: iter_time=3.555e-04, forward_time=0.202, loss_att=15.177, acc=0.971, loss=15.177, backward_time=0.279, grad_norm=29.856, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:25:54,903 (trainer:732) INFO: 40epoch:train:307-357batch: iter_time=3.601e-04, forward_time=0.203, loss_att=15.129, acc=0.972, loss=15.129, backward_time=0.280, grad_norm=26.711, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:26:27,363 (trainer:732) INFO: 40epoch:train:358-408batch: iter_time=3.345e-04, forward_time=0.201, loss_att=14.582, acc=0.972, loss=14.582, backward_time=0.278, grad_norm=24.696, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:26:59,513 (trainer:732) INFO: 40epoch:train:409-459batch: iter_time=3.579e-04, forward_time=0.201, loss_att=15.776, acc=0.969, loss=15.776, backward_time=0.277, grad_norm=25.801, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:27:32,290 (trainer:732) INFO: 40epoch:train:460-510batch: iter_time=3.333e-04, forward_time=0.203, loss_att=15.560, acc=0.971, loss=15.560, backward_time=0.281, grad_norm=29.896, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:28:05,359 (trainer:732) INFO: 40epoch:train:511-561batch: iter_time=3.444e-04, forward_time=0.205, loss_att=15.977, acc=0.971, loss=15.977, backward_time=0.285, grad_norm=28.015, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.582
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:28:37,558 (trainer:732) INFO: 40epoch:train:562-612batch: iter_time=3.215e-04, forward_time=0.200, loss_att=15.263, acc=0.970, loss=15.263, backward_time=0.274, grad_norm=24.936, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:29:10,120 (trainer:732) INFO: 40epoch:train:613-663batch: iter_time=3.523e-04, forward_time=0.202, loss_att=16.230, acc=0.970, loss=16.230, backward_time=0.281, grad_norm=27.548, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:29:42,634 (trainer:732) INFO: 40epoch:train:664-714batch: iter_time=3.512e-04, forward_time=0.201, loss_att=15.217, acc=0.971, loss=15.217, backward_time=0.278, grad_norm=25.977, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:30:15,103 (trainer:732) INFO: 40epoch:train:715-765batch: iter_time=3.174e-04, forward_time=0.202, loss_att=15.658, acc=0.970, loss=15.658, backward_time=0.278, grad_norm=25.986, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:30:47,397 (trainer:732) INFO: 40epoch:train:766-816batch: iter_time=3.743e-04, forward_time=0.201, loss_att=15.054, acc=0.971, loss=15.054, backward_time=0.276, grad_norm=25.732, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.520
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:31:19,414 (trainer:732) INFO: 40epoch:train:817-867batch: iter_time=3.205e-04, forward_time=0.200, loss_att=15.966, acc=0.969, loss=15.966, backward_time=0.276, grad_norm=27.038, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.519
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:31:51,904 (trainer:732) INFO: 40epoch:train:868-918batch: iter_time=3.634e-04, forward_time=0.201, loss_att=16.308, acc=0.969, loss=16.308, backward_time=0.278, grad_norm=27.722, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:32:24,260 (trainer:732) INFO: 40epoch:train:919-969batch: iter_time=3.063e-04, forward_time=0.201, loss_att=15.942, acc=0.970, loss=15.942, backward_time=0.278, grad_norm=25.859, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:32:56,596 (trainer:732) INFO: 40epoch:train:970-1020batch: iter_time=3.078e-04, forward_time=0.201, loss_att=15.946, acc=0.970, loss=15.946, backward_time=0.276, grad_norm=25.287, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:43:30,039 (trainer:338) INFO: 40epoch results: [train] iter_time=9.053e-04, forward_time=0.202, loss_att=15.591, acc=0.970, loss=15.591, backward_time=0.279, grad_norm=26.990, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.130, time=13 minutes and 32.61 seconds, total_count=41480, gpu_max_cached_mem_GB=30.428, [valid] loss_att=74.570, acc=0.888, cer=0.143, wer=0.348, loss=74.570, time=4 minutes and 54.21 seconds, total_count=3520, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 27.16 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:43:34,486 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:43:34,497 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/30epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:43:34,497 (trainer:272) INFO: 41/60epoch started. Estimated time to finish: 7 hours, 57 minutes and 33.15 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:46:37,379 (trainer:732) INFO: 41epoch:train:1-51batch: iter_time=0.014, forward_time=0.202, loss_att=14.647, acc=0.972, loss=14.647, backward_time=0.276, grad_norm=25.983, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=15.096
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:47:09,752 (trainer:732) INFO: 41epoch:train:52-102batch: iter_time=3.593e-04, forward_time=0.201, loss_att=15.460, acc=0.971, loss=15.460, backward_time=0.278, grad_norm=27.027, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:47:41,965 (trainer:732) INFO: 41epoch:train:103-153batch: iter_time=3.592e-04, forward_time=0.201, loss_att=14.024, acc=0.973, loss=14.024, backward_time=0.277, grad_norm=26.023, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:48:14,128 (trainer:732) INFO: 41epoch:train:154-204batch: iter_time=3.496e-04, forward_time=0.200, loss_att=13.888, acc=0.973, loss=13.888, backward_time=0.275, grad_norm=24.254, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.509
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:48:45,857 (trainer:732) INFO: 41epoch:train:205-255batch: iter_time=3.570e-04, forward_time=0.199, loss_att=13.598, acc=0.973, loss=13.598, backward_time=0.273, grad_norm=22.253, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.501
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:49:18,233 (trainer:732) INFO: 41epoch:train:256-306batch: iter_time=3.306e-04, forward_time=0.201, loss_att=14.883, acc=0.972, loss=14.883, backward_time=0.277, grad_norm=26.415, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:49:50,770 (trainer:732) INFO: 41epoch:train:307-357batch: iter_time=3.757e-04, forward_time=0.202, loss_att=15.435, acc=0.971, loss=15.435, backward_time=0.280, grad_norm=25.158, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:50:23,465 (trainer:732) INFO: 41epoch:train:358-408batch: iter_time=3.462e-04, forward_time=0.203, loss_att=14.429, acc=0.972, loss=14.429, backward_time=0.279, grad_norm=24.725, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:50:55,909 (trainer:732) INFO: 41epoch:train:409-459batch: iter_time=3.913e-04, forward_time=0.203, loss_att=15.042, acc=0.971, loss=15.042, backward_time=0.280, grad_norm=26.988, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:51:28,334 (trainer:732) INFO: 41epoch:train:460-510batch: iter_time=3.490e-04, forward_time=0.201, loss_att=15.406, acc=0.971, loss=15.406, backward_time=0.278, grad_norm=27.571, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:52:01,222 (trainer:732) INFO: 41epoch:train:511-561batch: iter_time=3.435e-04, forward_time=0.203, loss_att=17.308, acc=0.969, loss=17.308, backward_time=0.282, grad_norm=28.310, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.578
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:52:33,814 (trainer:732) INFO: 41epoch:train:562-612batch: iter_time=3.575e-04, forward_time=0.202, loss_att=14.933, acc=0.972, loss=14.933, backward_time=0.279, grad_norm=25.074, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:53:06,043 (trainer:732) INFO: 41epoch:train:613-663batch: iter_time=3.398e-04, forward_time=0.201, loss_att=14.803, acc=0.971, loss=14.803, backward_time=0.278, grad_norm=25.534, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:53:38,394 (trainer:732) INFO: 41epoch:train:664-714batch: iter_time=3.493e-04, forward_time=0.201, loss_att=14.644, acc=0.971, loss=14.644, backward_time=0.277, grad_norm=24.985, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:54:11,058 (trainer:732) INFO: 41epoch:train:715-765batch: iter_time=3.294e-04, forward_time=0.202, loss_att=15.487, acc=0.971, loss=15.487, backward_time=0.281, grad_norm=28.875, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:54:43,599 (trainer:732) INFO: 41epoch:train:766-816batch: iter_time=3.657e-04, forward_time=0.202, loss_att=15.062, acc=0.972, loss=15.062, backward_time=0.278, grad_norm=25.735, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:55:16,781 (trainer:732) INFO: 41epoch:train:817-867batch: iter_time=3.318e-04, forward_time=0.205, loss_att=15.565, acc=0.971, loss=15.565, backward_time=0.286, grad_norm=28.732, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.618
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:55:49,274 (trainer:732) INFO: 41epoch:train:868-918batch: iter_time=3.621e-04, forward_time=0.201, loss_att=14.785, acc=0.972, loss=14.785, backward_time=0.278, grad_norm=25.365, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:56:22,303 (trainer:732) INFO: 41epoch:train:919-969batch: iter_time=3.214e-04, forward_time=0.204, loss_att=16.063, acc=0.970, loss=16.063, backward_time=0.282, grad_norm=28.739, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.587
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 07:56:55,111 (trainer:732) INFO: 41epoch:train:970-1020batch: iter_time=3.165e-04, forward_time=0.202, loss_att=16.231, acc=0.970, loss=16.231, backward_time=0.281, grad_norm=26.544, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:07:22,428 (trainer:338) INFO: 41epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=15.059, acc=0.971, loss=15.059, backward_time=0.279, grad_norm=26.232, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.130, time=13 minutes and 32.24 seconds, total_count=42517, gpu_max_cached_mem_GB=30.428, [valid] loss_att=74.908, acc=0.888, cer=0.138, wer=0.347, loss=74.908, time=4 minutes and 49.9 seconds, total_count=3608, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 25.78 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:07:26,606 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:07:26,617 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/33epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:07:26,617 (trainer:272) INFO: 42/60epoch started. Estimated time to finish: 7 hours, 33 minutes and 40.24 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:10:29,808 (trainer:732) INFO: 42epoch:train:1-51batch: iter_time=0.015, forward_time=0.204, loss_att=14.131, acc=0.974, loss=14.131, backward_time=0.280, grad_norm=25.498, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=15.119
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:11:02,607 (trainer:732) INFO: 42epoch:train:52-102batch: iter_time=3.254e-04, forward_time=0.202, loss_att=14.692, acc=0.973, loss=14.692, backward_time=0.281, grad_norm=27.270, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:11:35,470 (trainer:732) INFO: 42epoch:train:103-153batch: iter_time=3.484e-04, forward_time=0.203, loss_att=14.409, acc=0.973, loss=14.409, backward_time=0.282, grad_norm=25.953, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:12:08,096 (trainer:732) INFO: 42epoch:train:154-204batch: iter_time=3.443e-04, forward_time=0.202, loss_att=14.319, acc=0.973, loss=14.319, backward_time=0.279, grad_norm=25.380, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:12:40,465 (trainer:732) INFO: 42epoch:train:205-255batch: iter_time=3.743e-04, forward_time=0.202, loss_att=15.022, acc=0.973, loss=15.022, backward_time=0.279, grad_norm=24.464, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:13:13,193 (trainer:732) INFO: 42epoch:train:256-306batch: iter_time=3.483e-04, forward_time=0.203, loss_att=14.959, acc=0.973, loss=14.959, backward_time=0.280, grad_norm=25.996, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:13:45,316 (trainer:732) INFO: 42epoch:train:307-357batch: iter_time=3.178e-04, forward_time=0.200, loss_att=14.101, acc=0.972, loss=14.101, backward_time=0.276, grad_norm=24.197, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.514
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:14:17,987 (trainer:732) INFO: 42epoch:train:358-408batch: iter_time=3.146e-04, forward_time=0.201, loss_att=14.888, acc=0.972, loss=14.888, backward_time=0.278, grad_norm=26.150, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:14:50,712 (trainer:732) INFO: 42epoch:train:409-459batch: iter_time=3.869e-04, forward_time=0.204, loss_att=14.805, acc=0.972, loss=14.805, backward_time=0.282, grad_norm=28.834, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:15:23,054 (trainer:732) INFO: 42epoch:train:460-510batch: iter_time=3.401e-04, forward_time=0.200, loss_att=14.985, acc=0.971, loss=14.985, backward_time=0.277, grad_norm=27.828, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:15:55,243 (trainer:732) INFO: 42epoch:train:511-561batch: iter_time=3.031e-04, forward_time=0.200, loss_att=14.084, acc=0.973, loss=14.084, backward_time=0.277, grad_norm=25.067, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:16:28,194 (trainer:732) INFO: 42epoch:train:562-612batch: iter_time=3.030e-04, forward_time=0.204, loss_att=15.407, acc=0.972, loss=15.407, backward_time=0.282, grad_norm=26.384, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:17:00,886 (trainer:732) INFO: 42epoch:train:613-663batch: iter_time=3.649e-04, forward_time=0.204, loss_att=15.108, acc=0.972, loss=15.108, backward_time=0.281, grad_norm=26.038, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:17:33,082 (trainer:732) INFO: 42epoch:train:664-714batch: iter_time=3.391e-04, forward_time=0.201, loss_att=13.579, acc=0.973, loss=13.579, backward_time=0.275, grad_norm=24.306, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:18:05,666 (trainer:732) INFO: 42epoch:train:715-765batch: iter_time=3.381e-04, forward_time=0.202, loss_att=14.855, acc=0.972, loss=14.855, backward_time=0.279, grad_norm=25.813, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:18:37,678 (trainer:732) INFO: 42epoch:train:766-816batch: iter_time=3.507e-04, forward_time=0.198, loss_att=13.592, acc=0.973, loss=13.592, backward_time=0.273, grad_norm=23.925, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.502
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:19:09,392 (trainer:732) INFO: 42epoch:train:817-867batch: iter_time=3.377e-04, forward_time=0.199, loss_att=13.542, acc=0.973, loss=13.542, backward_time=0.272, grad_norm=22.903, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.500
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:19:41,871 (trainer:732) INFO: 42epoch:train:868-918batch: iter_time=3.244e-04, forward_time=0.202, loss_att=14.650, acc=0.972, loss=14.650, backward_time=0.280, grad_norm=25.599, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:20:14,621 (trainer:732) INFO: 42epoch:train:919-969batch: iter_time=3.211e-04, forward_time=0.203, loss_att=14.853, acc=0.971, loss=14.853, backward_time=0.281, grad_norm=26.674, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:20:47,089 (trainer:732) INFO: 42epoch:train:970-1020batch: iter_time=3.010e-04, forward_time=0.201, loss_att=14.589, acc=0.972, loss=14.589, backward_time=0.277, grad_norm=24.227, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:31:17,210 (trainer:338) INFO: 42epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=14.523, acc=0.972, loss=14.523, backward_time=0.279, grad_norm=25.657, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.130, time=13 minutes and 32.62 seconds, total_count=43554, gpu_max_cached_mem_GB=30.428, [valid] loss_att=75.133, acc=0.888, cer=0.141, wer=0.348, loss=75.133, time=4 minutes and 49.66 seconds, total_count=3696, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 28.31 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:31:21,394 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:31:21,405 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/31epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:31:21,405 (trainer:272) INFO: 43/60epoch started. Estimated time to finish: 7 hours, 9 minutes and 48.51 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:34:25,921 (trainer:732) INFO: 43epoch:train:1-51batch: iter_time=0.014, forward_time=0.203, loss_att=13.384, acc=0.974, loss=13.384, backward_time=0.278, grad_norm=24.897, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=15.227
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:34:58,365 (trainer:732) INFO: 43epoch:train:52-102batch: iter_time=3.209e-04, forward_time=0.200, loss_att=14.206, acc=0.974, loss=14.206, backward_time=0.278, grad_norm=25.672, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:35:31,107 (trainer:732) INFO: 43epoch:train:103-153batch: iter_time=3.419e-04, forward_time=0.203, loss_att=13.698, acc=0.974, loss=13.698, backward_time=0.282, grad_norm=25.132, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:36:03,557 (trainer:732) INFO: 43epoch:train:154-204batch: iter_time=4.034e-04, forward_time=0.200, loss_att=14.162, acc=0.973, loss=14.162, backward_time=0.276, grad_norm=23.423, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:36:35,892 (trainer:732) INFO: 43epoch:train:205-255batch: iter_time=3.301e-04, forward_time=0.201, loss_att=14.006, acc=0.974, loss=14.006, backward_time=0.279, grad_norm=25.366, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:37:07,933 (trainer:732) INFO: 43epoch:train:256-306batch: iter_time=3.371e-04, forward_time=0.199, loss_att=13.206, acc=0.975, loss=13.206, backward_time=0.275, grad_norm=23.547, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.509
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:37:40,228 (trainer:732) INFO: 43epoch:train:307-357batch: iter_time=3.108e-04, forward_time=0.200, loss_att=13.732, acc=0.973, loss=13.732, backward_time=0.277, grad_norm=24.529, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:38:12,602 (trainer:732) INFO: 43epoch:train:358-408batch: iter_time=3.479e-04, forward_time=0.200, loss_att=14.998, acc=0.972, loss=14.998, backward_time=0.277, grad_norm=25.787, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:38:44,843 (trainer:732) INFO: 43epoch:train:409-459batch: iter_time=3.262e-04, forward_time=0.201, loss_att=13.349, acc=0.975, loss=13.349, backward_time=0.278, grad_norm=24.930, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:39:17,320 (trainer:732) INFO: 43epoch:train:460-510batch: iter_time=3.343e-04, forward_time=0.202, loss_att=13.999, acc=0.973, loss=13.999, backward_time=0.280, grad_norm=27.343, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:39:50,090 (trainer:732) INFO: 43epoch:train:511-561batch: iter_time=3.387e-04, forward_time=0.203, loss_att=14.228, acc=0.974, loss=14.228, backward_time=0.282, grad_norm=28.056, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:40:22,878 (trainer:732) INFO: 43epoch:train:562-612batch: iter_time=3.541e-04, forward_time=0.202, loss_att=14.036, acc=0.974, loss=14.036, backward_time=0.281, grad_norm=25.295, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:40:55,432 (trainer:732) INFO: 43epoch:train:613-663batch: iter_time=3.306e-04, forward_time=0.201, loss_att=13.916, acc=0.974, loss=13.916, backward_time=0.277, grad_norm=24.467, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:41:28,065 (trainer:732) INFO: 43epoch:train:664-714batch: iter_time=3.574e-04, forward_time=0.203, loss_att=14.767, acc=0.972, loss=14.767, backward_time=0.280, grad_norm=26.808, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:42:00,729 (trainer:732) INFO: 43epoch:train:715-765batch: iter_time=3.442e-04, forward_time=0.202, loss_att=14.360, acc=0.973, loss=14.360, backward_time=0.280, grad_norm=27.353, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:42:33,337 (trainer:732) INFO: 43epoch:train:766-816batch: iter_time=4.060e-04, forward_time=0.203, loss_att=14.675, acc=0.972, loss=14.675, backward_time=0.280, grad_norm=26.201, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:43:05,745 (trainer:732) INFO: 43epoch:train:817-867batch: iter_time=3.708e-04, forward_time=0.203, loss_att=14.678, acc=0.973, loss=14.678, backward_time=0.280, grad_norm=24.996, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:43:37,969 (trainer:732) INFO: 43epoch:train:868-918batch: iter_time=3.409e-04, forward_time=0.200, loss_att=13.906, acc=0.973, loss=13.906, backward_time=0.276, grad_norm=25.808, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:44:10,397 (trainer:732) INFO: 43epoch:train:919-969batch: iter_time=3.421e-04, forward_time=0.201, loss_att=13.839, acc=0.972, loss=13.839, backward_time=0.278, grad_norm=26.108, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:44:42,984 (trainer:732) INFO: 43epoch:train:970-1020batch: iter_time=3.293e-04, forward_time=0.202, loss_att=13.168, acc=0.974, loss=13.168, backward_time=0.278, grad_norm=25.330, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:55:15,868 (trainer:338) INFO: 43epoch results: [train] iter_time=0.001, forward_time=0.201, loss_att=14.014, acc=0.973, loss=14.014, backward_time=0.279, grad_norm=25.566, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.134, time=13 minutes and 33.75 seconds, total_count=44591, gpu_max_cached_mem_GB=30.428, [valid] loss_att=74.931, acc=0.889, cer=0.143, wer=0.343, loss=74.931, time=4 minutes and 51.39 seconds, total_count=3784, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 29.32 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:55:20,010 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:55:20,021 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/32epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:55:20,021 (trainer:272) INFO: 44/60epoch started. Estimated time to finish: 6 hours, 45 minutes and 58.16 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:58:21,300 (trainer:732) INFO: 44epoch:train:1-51batch: iter_time=0.015, forward_time=0.207, loss_att=13.016, acc=0.976, loss=13.016, backward_time=0.286, grad_norm=25.214, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=14.962
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:58:53,725 (trainer:732) INFO: 44epoch:train:52-102batch: iter_time=3.359e-04, forward_time=0.200, loss_att=13.213, acc=0.975, loss=13.213, backward_time=0.278, grad_norm=24.563, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:59:26,028 (trainer:732) INFO: 44epoch:train:103-153batch: iter_time=3.482e-04, forward_time=0.201, loss_att=12.942, acc=0.975, loss=12.942, backward_time=0.278, grad_norm=23.741, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 08:59:58,345 (trainer:732) INFO: 44epoch:train:154-204batch: iter_time=3.514e-04, forward_time=0.200, loss_att=13.381, acc=0.974, loss=13.381, backward_time=0.276, grad_norm=26.039, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:00:30,713 (trainer:732) INFO: 44epoch:train:205-255batch: iter_time=3.518e-04, forward_time=0.202, loss_att=13.418, acc=0.974, loss=13.418, backward_time=0.280, grad_norm=27.304, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:01:02,868 (trainer:732) INFO: 44epoch:train:256-306batch: iter_time=3.220e-04, forward_time=0.198, loss_att=12.727, acc=0.975, loss=12.727, backward_time=0.274, grad_norm=24.087, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:01:35,144 (trainer:732) INFO: 44epoch:train:307-357batch: iter_time=3.320e-04, forward_time=0.200, loss_att=13.298, acc=0.974, loss=13.298, backward_time=0.276, grad_norm=24.418, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:02:07,739 (trainer:732) INFO: 44epoch:train:358-408batch: iter_time=3.374e-04, forward_time=0.202, loss_att=13.878, acc=0.974, loss=13.878, backward_time=0.279, grad_norm=24.606, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:02:39,958 (trainer:732) INFO: 44epoch:train:409-459batch: iter_time=3.415e-04, forward_time=0.202, loss_att=13.674, acc=0.974, loss=13.674, backward_time=0.279, grad_norm=25.841, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:03:12,916 (trainer:732) INFO: 44epoch:train:460-510batch: iter_time=3.460e-04, forward_time=0.203, loss_att=13.970, acc=0.975, loss=13.970, backward_time=0.283, grad_norm=25.026, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.578
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:03:45,148 (trainer:732) INFO: 44epoch:train:511-561batch: iter_time=3.321e-04, forward_time=0.200, loss_att=14.003, acc=0.973, loss=14.003, backward_time=0.276, grad_norm=25.518, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:04:17,610 (trainer:732) INFO: 44epoch:train:562-612batch: iter_time=3.390e-04, forward_time=0.201, loss_att=12.892, acc=0.975, loss=12.892, backward_time=0.278, grad_norm=25.181, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:04:50,299 (trainer:732) INFO: 44epoch:train:613-663batch: iter_time=3.511e-04, forward_time=0.203, loss_att=13.593, acc=0.974, loss=13.593, backward_time=0.280, grad_norm=24.684, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:05:22,479 (trainer:732) INFO: 44epoch:train:664-714batch: iter_time=3.327e-04, forward_time=0.200, loss_att=13.503, acc=0.973, loss=13.503, backward_time=0.276, grad_norm=24.142, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.521
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:05:55,255 (trainer:732) INFO: 44epoch:train:715-765batch: iter_time=3.209e-04, forward_time=0.204, loss_att=13.951, acc=0.974, loss=13.951, backward_time=0.281, grad_norm=26.628, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:06:27,892 (trainer:732) INFO: 44epoch:train:766-816batch: iter_time=3.675e-04, forward_time=0.202, loss_att=14.089, acc=0.974, loss=14.089, backward_time=0.279, grad_norm=25.900, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:07:00,091 (trainer:732) INFO: 44epoch:train:817-867batch: iter_time=3.383e-04, forward_time=0.201, loss_att=13.252, acc=0.975, loss=13.252, backward_time=0.277, grad_norm=23.286, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:07:32,481 (trainer:732) INFO: 44epoch:train:868-918batch: iter_time=3.683e-04, forward_time=0.201, loss_att=14.149, acc=0.973, loss=14.149, backward_time=0.278, grad_norm=24.781, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:08:05,241 (trainer:732) INFO: 44epoch:train:919-969batch: iter_time=3.257e-04, forward_time=0.203, loss_att=13.350, acc=0.975, loss=13.350, backward_time=0.281, grad_norm=24.518, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:08:37,618 (trainer:732) INFO: 44epoch:train:970-1020batch: iter_time=3.243e-04, forward_time=0.201, loss_att=13.389, acc=0.974, loss=13.389, backward_time=0.278, grad_norm=23.574, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:19:07,379 (trainer:338) INFO: 44epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=13.478, acc=0.974, loss=13.478, backward_time=0.279, grad_norm=24.921, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.118, time=13 minutes and 29.54 seconds, total_count=45628, gpu_max_cached_mem_GB=30.428, [valid] loss_att=73.748, acc=0.890, cer=0.138, wer=0.344, loss=73.748, time=4 minutes and 48.99 seconds, total_count=3872, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 28.83 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:19:11,577 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:19:11,591 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/34epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:19:11,591 (trainer:272) INFO: 45/60epoch started. Estimated time to finish: 6 hours, 22 minutes and 4.87 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:22:13,620 (trainer:732) INFO: 45epoch:train:1-51batch: iter_time=0.012, forward_time=0.202, loss_att=12.739, acc=0.976, loss=12.739, backward_time=0.277, grad_norm=23.668, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=15.018
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:22:45,556 (trainer:732) INFO: 45epoch:train:52-102batch: iter_time=3.504e-04, forward_time=0.198, loss_att=12.341, acc=0.976, loss=12.341, backward_time=0.274, grad_norm=22.886, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=0.001, train_time=2.503
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:23:18,266 (trainer:732) INFO: 45epoch:train:103-153batch: iter_time=3.528e-04, forward_time=0.203, loss_att=13.327, acc=0.974, loss=13.327, backward_time=0.281, grad_norm=26.109, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:23:51,179 (trainer:732) INFO: 45epoch:train:154-204batch: iter_time=3.495e-04, forward_time=0.203, loss_att=13.762, acc=0.975, loss=13.762, backward_time=0.282, grad_norm=29.198, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:24:23,323 (trainer:732) INFO: 45epoch:train:205-255batch: iter_time=3.570e-04, forward_time=0.200, loss_att=12.266, acc=0.976, loss=12.266, backward_time=0.277, grad_norm=26.229, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:24:56,084 (trainer:732) INFO: 45epoch:train:256-306batch: iter_time=3.346e-04, forward_time=0.202, loss_att=12.516, acc=0.976, loss=12.516, backward_time=0.280, grad_norm=23.661, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:25:28,191 (trainer:732) INFO: 45epoch:train:307-357batch: iter_time=3.852e-04, forward_time=0.199, loss_att=12.554, acc=0.976, loss=12.554, backward_time=0.275, grad_norm=22.484, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.519
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:26:00,795 (trainer:732) INFO: 45epoch:train:358-408batch: iter_time=3.779e-04, forward_time=0.202, loss_att=13.041, acc=0.975, loss=13.041, backward_time=0.279, grad_norm=24.182, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:26:33,014 (trainer:732) INFO: 45epoch:train:409-459batch: iter_time=3.782e-04, forward_time=0.202, loss_att=12.505, acc=0.976, loss=12.505, backward_time=0.278, grad_norm=24.122, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:27:05,339 (trainer:732) INFO: 45epoch:train:460-510batch: iter_time=3.805e-04, forward_time=0.201, loss_att=12.570, acc=0.976, loss=12.570, backward_time=0.277, grad_norm=24.355, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:27:38,192 (trainer:732) INFO: 45epoch:train:511-561batch: iter_time=3.551e-04, forward_time=0.203, loss_att=13.465, acc=0.975, loss=13.465, backward_time=0.281, grad_norm=24.970, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.577
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:28:10,775 (trainer:732) INFO: 45epoch:train:562-612batch: iter_time=3.721e-04, forward_time=0.202, loss_att=13.953, acc=0.974, loss=13.953, backward_time=0.279, grad_norm=25.860, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:28:42,991 (trainer:732) INFO: 45epoch:train:613-663batch: iter_time=3.770e-04, forward_time=0.202, loss_att=13.308, acc=0.975, loss=13.308, backward_time=0.278, grad_norm=23.833, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=0.001, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:29:15,585 (trainer:732) INFO: 45epoch:train:664-714batch: iter_time=3.429e-04, forward_time=0.201, loss_att=12.939, acc=0.976, loss=12.939, backward_time=0.277, grad_norm=26.614, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:29:48,445 (trainer:732) INFO: 45epoch:train:715-765batch: iter_time=3.335e-04, forward_time=0.204, loss_att=13.019, acc=0.975, loss=13.019, backward_time=0.282, grad_norm=27.224, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:30:20,628 (trainer:732) INFO: 45epoch:train:766-816batch: iter_time=3.877e-04, forward_time=0.201, loss_att=13.080, acc=0.975, loss=13.080, backward_time=0.275, grad_norm=25.321, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.516
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:30:52,912 (trainer:732) INFO: 45epoch:train:817-867batch: iter_time=3.314e-04, forward_time=0.201, loss_att=13.624, acc=0.974, loss=13.624, backward_time=0.280, grad_norm=26.668, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:31:24,948 (trainer:732) INFO: 45epoch:train:868-918batch: iter_time=3.387e-04, forward_time=0.199, loss_att=11.713, acc=0.977, loss=11.713, backward_time=0.274, grad_norm=23.605, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.511
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:31:58,163 (trainer:732) INFO: 45epoch:train:919-969batch: iter_time=3.479e-04, forward_time=0.206, loss_att=14.015, acc=0.975, loss=14.015, backward_time=0.285, grad_norm=26.599, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.598
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:32:30,839 (trainer:732) INFO: 45epoch:train:970-1020batch: iter_time=4.407e-04, forward_time=0.202, loss_att=13.220, acc=0.975, loss=13.220, backward_time=0.279, grad_norm=25.924, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:43:11,012 (trainer:338) INFO: 45epoch results: [train] iter_time=9.358e-04, forward_time=0.202, loss_att=12.996, acc=0.975, loss=12.996, backward_time=0.279, grad_norm=25.220, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.126, time=13 minutes and 31.55 seconds, total_count=46665, gpu_max_cached_mem_GB=30.428, [valid] loss_att=73.925, acc=0.891, cer=0.137, wer=0.337, loss=73.925, time=4 minutes and 52.4 seconds, total_count=3960, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 35.47 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:43:15,293 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:43:15,312 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/35epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:43:15,312 (trainer:272) INFO: 46/60epoch started. Estimated time to finish: 5 hours, 58 minutes and 15.7 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:46:18,830 (trainer:732) INFO: 46epoch:train:1-51batch: iter_time=0.009, forward_time=0.204, loss_att=12.203, acc=0.977, loss=12.203, backward_time=0.281, grad_norm=23.661, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=15.138
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:46:50,769 (trainer:732) INFO: 46epoch:train:52-102batch: iter_time=4.285e-04, forward_time=0.199, loss_att=11.652, acc=0.977, loss=11.652, backward_time=0.273, grad_norm=22.522, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.511
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:47:23,566 (trainer:732) INFO: 46epoch:train:103-153batch: iter_time=3.665e-04, forward_time=0.204, loss_att=12.087, acc=0.977, loss=12.087, backward_time=0.282, grad_norm=25.911, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.569
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:47:56,177 (trainer:732) INFO: 46epoch:train:154-204batch: iter_time=3.353e-04, forward_time=0.202, loss_att=13.370, acc=0.975, loss=13.370, backward_time=0.279, grad_norm=23.900, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:48:28,258 (trainer:732) INFO: 46epoch:train:205-255batch: iter_time=3.846e-04, forward_time=0.201, loss_att=12.387, acc=0.977, loss=12.387, backward_time=0.277, grad_norm=21.336, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:49:01,070 (trainer:732) INFO: 46epoch:train:256-306batch: iter_time=3.765e-04, forward_time=0.204, loss_att=12.212, acc=0.976, loss=12.212, backward_time=0.281, grad_norm=22.887, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:49:33,177 (trainer:732) INFO: 46epoch:train:307-357batch: iter_time=3.516e-04, forward_time=0.200, loss_att=12.496, acc=0.976, loss=12.496, backward_time=0.276, grad_norm=24.298, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.516
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:50:05,574 (trainer:732) INFO: 46epoch:train:358-408batch: iter_time=3.769e-04, forward_time=0.200, loss_att=13.068, acc=0.976, loss=13.068, backward_time=0.277, grad_norm=25.922, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:50:38,115 (trainer:732) INFO: 46epoch:train:409-459batch: iter_time=3.698e-04, forward_time=0.204, loss_att=13.141, acc=0.975, loss=13.141, backward_time=0.280, grad_norm=24.959, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:51:11,067 (trainer:732) INFO: 46epoch:train:460-510batch: iter_time=3.895e-04, forward_time=0.205, loss_att=12.120, acc=0.978, loss=12.120, backward_time=0.283, grad_norm=25.359, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:51:43,521 (trainer:732) INFO: 46epoch:train:511-561batch: iter_time=3.612e-04, forward_time=0.201, loss_att=13.874, acc=0.974, loss=13.874, backward_time=0.279, grad_norm=26.365, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:52:16,119 (trainer:732) INFO: 46epoch:train:562-612batch: iter_time=3.754e-04, forward_time=0.202, loss_att=12.887, acc=0.976, loss=12.887, backward_time=0.280, grad_norm=26.670, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:52:48,788 (trainer:732) INFO: 46epoch:train:613-663batch: iter_time=3.828e-04, forward_time=0.202, loss_att=12.454, acc=0.977, loss=12.454, backward_time=0.280, grad_norm=24.890, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:53:21,185 (trainer:732) INFO: 46epoch:train:664-714batch: iter_time=3.302e-04, forward_time=0.201, loss_att=12.774, acc=0.975, loss=12.774, backward_time=0.277, grad_norm=23.901, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:53:53,645 (trainer:732) INFO: 46epoch:train:715-765batch: iter_time=3.260e-04, forward_time=0.202, loss_att=12.726, acc=0.975, loss=12.726, backward_time=0.278, grad_norm=24.753, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:54:26,087 (trainer:732) INFO: 46epoch:train:766-816batch: iter_time=3.710e-04, forward_time=0.201, loss_att=12.654, acc=0.976, loss=12.654, backward_time=0.278, grad_norm=23.636, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:54:58,589 (trainer:732) INFO: 46epoch:train:817-867batch: iter_time=3.510e-04, forward_time=0.203, loss_att=12.672, acc=0.976, loss=12.672, backward_time=0.280, grad_norm=25.337, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:55:30,790 (trainer:732) INFO: 46epoch:train:868-918batch: iter_time=3.704e-04, forward_time=0.202, loss_att=12.558, acc=0.976, loss=12.558, backward_time=0.275, grad_norm=23.079, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:56:03,194 (trainer:732) INFO: 46epoch:train:919-969batch: iter_time=3.474e-04, forward_time=0.201, loss_att=11.751, acc=0.977, loss=11.751, backward_time=0.277, grad_norm=22.121, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 09:56:35,934 (trainer:732) INFO: 46epoch:train:970-1020batch: iter_time=3.126e-04, forward_time=0.201, loss_att=12.398, acc=0.977, loss=12.398, backward_time=0.281, grad_norm=24.501, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:07:10,949 (trainer:338) INFO: 46epoch results: [train] iter_time=7.997e-04, forward_time=0.202, loss_att=12.584, acc=0.976, loss=12.584, backward_time=0.279, grad_norm=24.267, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.130, time=13 minutes and 32.66 seconds, total_count=47702, gpu_max_cached_mem_GB=30.428, [valid] loss_att=72.735, acc=0.892, cer=0.139, wer=0.336, loss=72.735, time=4 minutes and 53.53 seconds, total_count=4048, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 29.44 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:07:15,177 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:07:15,190 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/37epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:07:15,190 (trainer:272) INFO: 47/60epoch started. Estimated time to finish: 5 hours, 34 minutes and 24.73 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:10:17,547 (trainer:732) INFO: 47epoch:train:1-51batch: iter_time=0.014, forward_time=0.202, loss_att=11.488, acc=0.979, loss=11.488, backward_time=0.279, grad_norm=23.881, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=15.046
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:10:49,983 (trainer:732) INFO: 47epoch:train:52-102batch: iter_time=3.248e-04, forward_time=0.202, loss_att=12.187, acc=0.977, loss=12.187, backward_time=0.279, grad_norm=22.424, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:11:22,518 (trainer:732) INFO: 47epoch:train:103-153batch: iter_time=3.236e-04, forward_time=0.201, loss_att=11.587, acc=0.978, loss=11.587, backward_time=0.279, grad_norm=22.933, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:11:54,877 (trainer:732) INFO: 47epoch:train:154-204batch: iter_time=3.142e-04, forward_time=0.199, loss_att=11.849, acc=0.977, loss=11.849, backward_time=0.276, grad_norm=22.595, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:12:27,168 (trainer:732) INFO: 47epoch:train:205-255batch: iter_time=3.385e-04, forward_time=0.200, loss_att=12.713, acc=0.976, loss=12.713, backward_time=0.279, grad_norm=24.430, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:12:59,594 (trainer:732) INFO: 47epoch:train:256-306batch: iter_time=3.079e-04, forward_time=0.200, loss_att=12.636, acc=0.976, loss=12.636, backward_time=0.280, grad_norm=25.570, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:13:31,885 (trainer:732) INFO: 47epoch:train:307-357batch: iter_time=3.254e-04, forward_time=0.199, loss_att=12.002, acc=0.977, loss=12.002, backward_time=0.274, grad_norm=22.911, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:14:04,472 (trainer:732) INFO: 47epoch:train:358-408batch: iter_time=3.148e-04, forward_time=0.202, loss_att=12.830, acc=0.977, loss=12.830, backward_time=0.280, grad_norm=23.650, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:14:36,531 (trainer:732) INFO: 47epoch:train:409-459batch: iter_time=3.395e-04, forward_time=0.200, loss_att=11.860, acc=0.978, loss=11.860, backward_time=0.276, grad_norm=23.531, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:15:08,916 (trainer:732) INFO: 47epoch:train:460-510batch: iter_time=3.399e-04, forward_time=0.201, loss_att=12.092, acc=0.977, loss=12.092, backward_time=0.278, grad_norm=25.973, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:15:41,159 (trainer:732) INFO: 47epoch:train:511-561batch: iter_time=3.422e-04, forward_time=0.201, loss_att=11.791, acc=0.977, loss=11.791, backward_time=0.277, grad_norm=23.507, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:16:13,982 (trainer:732) INFO: 47epoch:train:562-612batch: iter_time=3.568e-04, forward_time=0.204, loss_att=13.401, acc=0.976, loss=13.401, backward_time=0.282, grad_norm=25.958, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.564
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:16:46,168 (trainer:732) INFO: 47epoch:train:613-663batch: iter_time=3.611e-04, forward_time=0.201, loss_att=12.471, acc=0.977, loss=12.471, backward_time=0.277, grad_norm=23.387, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:17:18,770 (trainer:732) INFO: 47epoch:train:664-714batch: iter_time=3.737e-04, forward_time=0.203, loss_att=12.134, acc=0.977, loss=12.134, backward_time=0.279, grad_norm=23.333, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:17:51,857 (trainer:732) INFO: 47epoch:train:715-765batch: iter_time=3.343e-04, forward_time=0.205, loss_att=12.100, acc=0.977, loss=12.100, backward_time=0.284, grad_norm=26.276, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.592
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:18:24,735 (trainer:732) INFO: 47epoch:train:766-816batch: iter_time=3.654e-04, forward_time=0.204, loss_att=12.629, acc=0.977, loss=12.629, backward_time=0.282, grad_norm=24.100, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:18:56,835 (trainer:732) INFO: 47epoch:train:817-867batch: iter_time=3.671e-04, forward_time=0.201, loss_att=12.518, acc=0.976, loss=12.518, backward_time=0.277, grad_norm=23.575, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:19:29,205 (trainer:732) INFO: 47epoch:train:868-918batch: iter_time=3.881e-04, forward_time=0.202, loss_att=12.183, acc=0.977, loss=12.183, backward_time=0.278, grad_norm=23.960, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:20:01,528 (trainer:732) INFO: 47epoch:train:919-969batch: iter_time=3.501e-04, forward_time=0.201, loss_att=11.826, acc=0.977, loss=11.826, backward_time=0.276, grad_norm=23.189, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:20:34,606 (trainer:732) INFO: 47epoch:train:970-1020batch: iter_time=2.991e-04, forward_time=0.205, loss_att=12.058, acc=0.977, loss=12.058, backward_time=0.284, grad_norm=24.469, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.581
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:31:05,275 (trainer:338) INFO: 47epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=12.210, acc=0.977, loss=12.210, backward_time=0.279, grad_norm=23.967, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.125, time=13 minutes and 31.48 seconds, total_count=48739, gpu_max_cached_mem_GB=30.428, [valid] loss_att=72.630, acc=0.893, cer=0.135, wer=0.335, loss=72.630, time=4 minutes and 53.26 seconds, total_count=4136, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 25.34 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:31:09,467 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:31:09,480 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/36epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:31:09,480 (trainer:272) INFO: 48/60epoch started. Estimated time to finish: 5 hours, 10 minutes and 31.84 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:34:12,167 (trainer:732) INFO: 48epoch:train:1-51batch: iter_time=0.016, forward_time=0.202, loss_att=11.148, acc=0.979, loss=11.148, backward_time=0.277, grad_norm=22.491, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=15.079
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:34:44,370 (trainer:732) INFO: 48epoch:train:52-102batch: iter_time=3.391e-04, forward_time=0.200, loss_att=10.425, acc=0.980, loss=10.425, backward_time=0.277, grad_norm=21.996, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:35:16,687 (trainer:732) INFO: 48epoch:train:103-153batch: iter_time=3.280e-04, forward_time=0.200, loss_att=11.905, acc=0.977, loss=11.905, backward_time=0.277, grad_norm=25.519, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:35:49,321 (trainer:732) INFO: 48epoch:train:154-204batch: iter_time=3.208e-04, forward_time=0.201, loss_att=11.996, acc=0.978, loss=11.996, backward_time=0.279, grad_norm=24.120, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:36:22,088 (trainer:732) INFO: 48epoch:train:205-255batch: iter_time=3.215e-04, forward_time=0.204, loss_att=11.505, acc=0.979, loss=11.505, backward_time=0.284, grad_norm=22.967, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.589
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:36:54,782 (trainer:732) INFO: 48epoch:train:256-306batch: iter_time=3.011e-04, forward_time=0.202, loss_att=12.224, acc=0.977, loss=12.224, backward_time=0.280, grad_norm=23.632, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:37:27,435 (trainer:732) INFO: 48epoch:train:307-357batch: iter_time=3.256e-04, forward_time=0.203, loss_att=11.152, acc=0.978, loss=11.152, backward_time=0.281, grad_norm=23.411, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:37:59,807 (trainer:732) INFO: 48epoch:train:358-408batch: iter_time=3.070e-04, forward_time=0.200, loss_att=12.340, acc=0.976, loss=12.340, backward_time=0.277, grad_norm=25.267, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:38:32,236 (trainer:732) INFO: 48epoch:train:409-459batch: iter_time=3.230e-04, forward_time=0.202, loss_att=12.716, acc=0.976, loss=12.716, backward_time=0.280, grad_norm=24.714, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:39:04,547 (trainer:732) INFO: 48epoch:train:460-510batch: iter_time=3.033e-04, forward_time=0.200, loss_att=10.946, acc=0.979, loss=10.946, backward_time=0.275, grad_norm=22.806, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:39:37,304 (trainer:732) INFO: 48epoch:train:511-561batch: iter_time=3.496e-04, forward_time=0.202, loss_att=11.814, acc=0.978, loss=11.814, backward_time=0.280, grad_norm=24.288, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:40:09,669 (trainer:732) INFO: 48epoch:train:562-612batch: iter_time=3.375e-04, forward_time=0.201, loss_att=11.633, acc=0.978, loss=11.633, backward_time=0.277, grad_norm=22.792, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:40:42,168 (trainer:732) INFO: 48epoch:train:613-663batch: iter_time=3.714e-04, forward_time=0.202, loss_att=12.525, acc=0.977, loss=12.525, backward_time=0.279, grad_norm=25.455, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:41:14,794 (trainer:732) INFO: 48epoch:train:664-714batch: iter_time=3.315e-04, forward_time=0.202, loss_att=12.074, acc=0.977, loss=12.074, backward_time=0.278, grad_norm=25.398, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:41:47,256 (trainer:732) INFO: 48epoch:train:715-765batch: iter_time=2.959e-04, forward_time=0.202, loss_att=11.138, acc=0.978, loss=11.138, backward_time=0.278, grad_norm=23.651, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:42:19,912 (trainer:732) INFO: 48epoch:train:766-816batch: iter_time=3.431e-04, forward_time=0.202, loss_att=11.771, acc=0.977, loss=11.771, backward_time=0.279, grad_norm=24.363, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:42:52,270 (trainer:732) INFO: 48epoch:train:817-867batch: iter_time=3.189e-04, forward_time=0.202, loss_att=12.044, acc=0.978, loss=12.044, backward_time=0.280, grad_norm=23.604, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:43:24,605 (trainer:732) INFO: 48epoch:train:868-918batch: iter_time=3.330e-04, forward_time=0.201, loss_att=12.467, acc=0.976, loss=12.467, backward_time=0.277, grad_norm=25.374, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:43:57,154 (trainer:732) INFO: 48epoch:train:919-969batch: iter_time=3.331e-04, forward_time=0.202, loss_att=11.952, acc=0.977, loss=11.952, backward_time=0.280, grad_norm=24.807, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:44:29,723 (trainer:732) INFO: 48epoch:train:970-1020batch: iter_time=3.185e-04, forward_time=0.202, loss_att=11.763, acc=0.978, loss=11.763, backward_time=0.279, grad_norm=22.532, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:54:52,508 (trainer:338) INFO: 48epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=11.785, acc=0.978, loss=11.785, backward_time=0.279, grad_norm=23.979, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.129, time=13 minutes and 32.3 seconds, total_count=49776, gpu_max_cached_mem_GB=30.428, [valid] loss_att=73.195, acc=0.892, cer=0.138, wer=0.336, loss=73.195, time=4 minutes and 38.34 seconds, total_count=4224, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.39 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:54:56,739 (trainer:384) INFO: There are no improvements in this epoch
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:54:56,753 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/38epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:54:56,754 (trainer:272) INFO: 49/60epoch started. Estimated time to finish: 4 hours, 46 minutes and 37.14 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:57:58,574 (trainer:732) INFO: 49epoch:train:1-51batch: iter_time=0.013, forward_time=0.206, loss_att=10.571, acc=0.980, loss=10.571, backward_time=0.279, grad_norm=23.756, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=15.007
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:58:31,188 (trainer:732) INFO: 49epoch:train:52-102batch: iter_time=3.488e-04, forward_time=0.203, loss_att=10.624, acc=0.980, loss=10.624, backward_time=0.279, grad_norm=22.718, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:59:03,330 (trainer:732) INFO: 49epoch:train:103-153batch: iter_time=3.776e-04, forward_time=0.200, loss_att=11.642, acc=0.977, loss=11.642, backward_time=0.276, grad_norm=21.778, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.518
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 10:59:36,083 (trainer:732) INFO: 49epoch:train:154-204batch: iter_time=3.606e-04, forward_time=0.203, loss_att=11.450, acc=0.979, loss=11.450, backward_time=0.280, grad_norm=23.357, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:00:08,293 (trainer:732) INFO: 49epoch:train:205-255batch: iter_time=3.383e-04, forward_time=0.201, loss_att=10.936, acc=0.979, loss=10.936, backward_time=0.277, grad_norm=21.532, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:00:40,958 (trainer:732) INFO: 49epoch:train:256-306batch: iter_time=3.728e-04, forward_time=0.202, loss_att=10.900, acc=0.979, loss=10.900, backward_time=0.279, grad_norm=21.469, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:01:13,510 (trainer:732) INFO: 49epoch:train:307-357batch: iter_time=3.934e-04, forward_time=0.202, loss_att=11.377, acc=0.979, loss=11.377, backward_time=0.279, grad_norm=20.844, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:01:45,817 (trainer:732) INFO: 49epoch:train:358-408batch: iter_time=3.877e-04, forward_time=0.201, loss_att=11.472, acc=0.978, loss=11.472, backward_time=0.276, grad_norm=22.616, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:02:18,161 (trainer:732) INFO: 49epoch:train:409-459batch: iter_time=3.855e-04, forward_time=0.202, loss_att=11.804, acc=0.978, loss=11.804, backward_time=0.279, grad_norm=22.516, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:02:51,139 (trainer:732) INFO: 49epoch:train:460-510batch: iter_time=3.665e-04, forward_time=0.203, loss_att=11.495, acc=0.978, loss=11.495, backward_time=0.281, grad_norm=25.495, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=0.001, train_time=2.586
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:03:23,398 (trainer:732) INFO: 49epoch:train:511-561batch: iter_time=3.805e-04, forward_time=0.201, loss_att=11.551, acc=0.978, loss=11.551, backward_time=0.276, grad_norm=21.437, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:03:55,853 (trainer:732) INFO: 49epoch:train:562-612batch: iter_time=3.771e-04, forward_time=0.202, loss_att=11.075, acc=0.979, loss=11.075, backward_time=0.277, grad_norm=23.003, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:04:28,177 (trainer:732) INFO: 49epoch:train:613-663batch: iter_time=3.865e-04, forward_time=0.203, loss_att=11.978, acc=0.977, loss=11.978, backward_time=0.279, grad_norm=24.450, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:05:00,269 (trainer:732) INFO: 49epoch:train:664-714batch: iter_time=3.422e-04, forward_time=0.201, loss_att=11.144, acc=0.978, loss=11.144, backward_time=0.275, grad_norm=21.788, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.511
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:05:32,928 (trainer:732) INFO: 49epoch:train:715-765batch: iter_time=3.590e-04, forward_time=0.203, loss_att=11.546, acc=0.977, loss=11.546, backward_time=0.281, grad_norm=24.555, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:06:05,888 (trainer:732) INFO: 49epoch:train:766-816batch: iter_time=3.837e-04, forward_time=0.204, loss_att=12.566, acc=0.977, loss=12.566, backward_time=0.283, grad_norm=26.633, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:06:38,215 (trainer:732) INFO: 49epoch:train:817-867batch: iter_time=3.558e-04, forward_time=0.202, loss_att=11.964, acc=0.978, loss=11.964, backward_time=0.280, grad_norm=24.371, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:07:11,043 (trainer:732) INFO: 49epoch:train:868-918batch: iter_time=3.689e-04, forward_time=0.203, loss_att=12.397, acc=0.978, loss=12.397, backward_time=0.282, grad_norm=24.254, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.572
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:07:43,435 (trainer:732) INFO: 49epoch:train:919-969batch: iter_time=3.746e-04, forward_time=0.201, loss_att=11.072, acc=0.979, loss=11.072, backward_time=0.277, grad_norm=22.709, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:08:15,918 (trainer:732) INFO: 49epoch:train:970-1020batch: iter_time=3.198e-04, forward_time=0.201, loss_att=12.055, acc=0.978, loss=12.055, backward_time=0.277, grad_norm=22.747, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:18:53,291 (trainer:338) INFO: 49epoch results: [train] iter_time=9.816e-04, forward_time=0.202, loss_att=11.500, acc=0.978, loss=11.500, backward_time=0.279, grad_norm=23.111, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.124, time=13 minutes and 31.32 seconds, total_count=50813, gpu_max_cached_mem_GB=30.428, [valid] loss_att=72.286, acc=0.893, cer=0.137, wer=0.329, loss=72.286, time=4 minutes and 52.9 seconds, total_count=4312, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.31 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:18:57,580 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:18:57,593 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/39epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:18:57,593 (trainer:272) INFO: 50/60epoch started. Estimated time to finish: 4 hours, 22 minutes and 45.78 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:22:02,124 (trainer:732) INFO: 50epoch:train:1-51batch: iter_time=0.015, forward_time=0.202, loss_att=10.388, acc=0.980, loss=10.388, backward_time=0.275, grad_norm=20.900, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=15.232
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:22:34,181 (trainer:732) INFO: 50epoch:train:52-102batch: iter_time=3.529e-04, forward_time=0.200, loss_att=10.684, acc=0.979, loss=10.684, backward_time=0.275, grad_norm=22.071, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.511
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:23:07,090 (trainer:732) INFO: 50epoch:train:103-153batch: iter_time=3.453e-04, forward_time=0.204, loss_att=11.048, acc=0.980, loss=11.048, backward_time=0.283, grad_norm=22.895, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.578
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:23:39,739 (trainer:732) INFO: 50epoch:train:154-204batch: iter_time=3.366e-04, forward_time=0.201, loss_att=11.134, acc=0.979, loss=11.134, backward_time=0.281, grad_norm=24.787, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:24:11,477 (trainer:732) INFO: 50epoch:train:205-255batch: iter_time=3.231e-04, forward_time=0.198, loss_att=10.280, acc=0.980, loss=10.280, backward_time=0.273, grad_norm=20.093, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.501
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:24:44,399 (trainer:732) INFO: 50epoch:train:256-306batch: iter_time=3.777e-04, forward_time=0.204, loss_att=11.984, acc=0.979, loss=11.984, backward_time=0.281, grad_norm=23.048, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.580
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:25:16,658 (trainer:732) INFO: 50epoch:train:307-357batch: iter_time=3.274e-04, forward_time=0.201, loss_att=10.761, acc=0.980, loss=10.761, backward_time=0.278, grad_norm=21.056, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:25:49,309 (trainer:732) INFO: 50epoch:train:358-408batch: iter_time=3.316e-04, forward_time=0.201, loss_att=11.303, acc=0.979, loss=11.303, backward_time=0.278, grad_norm=22.560, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:26:21,479 (trainer:732) INFO: 50epoch:train:409-459batch: iter_time=3.451e-04, forward_time=0.201, loss_att=11.276, acc=0.979, loss=11.276, backward_time=0.278, grad_norm=21.764, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:26:54,648 (trainer:732) INFO: 50epoch:train:460-510batch: iter_time=3.383e-04, forward_time=0.204, loss_att=11.771, acc=0.979, loss=11.771, backward_time=0.283, grad_norm=22.739, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.597
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:27:27,186 (trainer:732) INFO: 50epoch:train:511-561batch: iter_time=3.536e-04, forward_time=0.202, loss_att=11.439, acc=0.979, loss=11.439, backward_time=0.278, grad_norm=23.067, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:28:00,005 (trainer:732) INFO: 50epoch:train:562-612batch: iter_time=3.651e-04, forward_time=0.203, loss_att=12.151, acc=0.977, loss=12.151, backward_time=0.280, grad_norm=26.094, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:28:32,210 (trainer:732) INFO: 50epoch:train:613-663batch: iter_time=3.573e-04, forward_time=0.202, loss_att=12.057, acc=0.976, loss=12.057, backward_time=0.279, grad_norm=22.690, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:29:04,351 (trainer:732) INFO: 50epoch:train:664-714batch: iter_time=3.459e-04, forward_time=0.200, loss_att=10.855, acc=0.979, loss=10.855, backward_time=0.275, grad_norm=21.920, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.522
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:29:36,804 (trainer:732) INFO: 50epoch:train:715-765batch: iter_time=3.059e-04, forward_time=0.201, loss_att=11.161, acc=0.979, loss=11.161, backward_time=0.278, grad_norm=22.917, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:30:09,258 (trainer:732) INFO: 50epoch:train:766-816batch: iter_time=3.660e-04, forward_time=0.203, loss_att=10.557, acc=0.980, loss=10.557, backward_time=0.278, grad_norm=21.009, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:30:41,532 (trainer:732) INFO: 50epoch:train:817-867batch: iter_time=3.508e-04, forward_time=0.202, loss_att=11.200, acc=0.979, loss=11.200, backward_time=0.278, grad_norm=21.846, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:31:13,876 (trainer:732) INFO: 50epoch:train:868-918batch: iter_time=3.660e-04, forward_time=0.202, loss_att=11.351, acc=0.978, loss=11.351, backward_time=0.277, grad_norm=23.939, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:31:46,726 (trainer:732) INFO: 50epoch:train:919-969batch: iter_time=3.574e-04, forward_time=0.204, loss_att=11.737, acc=0.978, loss=11.737, backward_time=0.282, grad_norm=24.967, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.578
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:32:19,447 (trainer:732) INFO: 50epoch:train:970-1020batch: iter_time=3.146e-04, forward_time=0.203, loss_att=11.626, acc=0.979, loss=11.626, backward_time=0.281, grad_norm=23.244, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:42:57,968 (trainer:338) INFO: 50epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=11.221, acc=0.979, loss=11.221, backward_time=0.279, grad_norm=22.721, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.135, time=13 minutes and 34.02 seconds, total_count=51850, gpu_max_cached_mem_GB=30.428, [valid] loss_att=74.209, acc=0.892, cer=0.133, wer=0.335, loss=74.209, time=4 minutes and 55.46 seconds, total_count=4400, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 30.9 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:43:02,568 (trainer:384) INFO: There are no improvements in this epoch
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:43:02,586 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/40epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:43:02,586 (trainer:272) INFO: 51/60epoch started. Estimated time to finish: 3 hours, 58 minutes and 54.88 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:46:03,930 (trainer:732) INFO: 51epoch:train:1-51batch: iter_time=0.013, forward_time=0.202, loss_att=10.900, acc=0.980, loss=10.900, backward_time=0.278, grad_norm=21.743, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=14.963
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:46:35,909 (trainer:732) INFO: 51epoch:train:52-102batch: iter_time=3.338e-04, forward_time=0.199, loss_att=10.226, acc=0.981, loss=10.226, backward_time=0.275, grad_norm=21.414, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.503
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:47:08,695 (trainer:732) INFO: 51epoch:train:103-153batch: iter_time=3.522e-04, forward_time=0.202, loss_att=11.206, acc=0.980, loss=11.206, backward_time=0.281, grad_norm=22.625, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:47:40,994 (trainer:732) INFO: 51epoch:train:154-204batch: iter_time=3.474e-04, forward_time=0.200, loss_att=10.089, acc=0.981, loss=10.089, backward_time=0.276, grad_norm=22.287, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:48:13,302 (trainer:732) INFO: 51epoch:train:205-255batch: iter_time=3.360e-04, forward_time=0.201, loss_att=10.456, acc=0.980, loss=10.456, backward_time=0.278, grad_norm=21.331, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:48:45,838 (trainer:732) INFO: 51epoch:train:256-306batch: iter_time=3.554e-04, forward_time=0.202, loss_att=11.239, acc=0.979, loss=11.239, backward_time=0.279, grad_norm=21.759, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.546
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:49:17,788 (trainer:732) INFO: 51epoch:train:307-357batch: iter_time=3.480e-04, forward_time=0.198, loss_att=9.973, acc=0.979, loss=9.973, backward_time=0.273, grad_norm=21.071, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.506
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:49:50,045 (trainer:732) INFO: 51epoch:train:358-408batch: iter_time=3.414e-04, forward_time=0.200, loss_att=10.593, acc=0.980, loss=10.593, backward_time=0.275, grad_norm=23.550, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.519
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:50:22,466 (trainer:732) INFO: 51epoch:train:409-459batch: iter_time=3.557e-04, forward_time=0.202, loss_att=10.612, acc=0.980, loss=10.612, backward_time=0.279, grad_norm=22.542, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:50:55,389 (trainer:732) INFO: 51epoch:train:460-510batch: iter_time=3.394e-04, forward_time=0.203, loss_att=11.030, acc=0.980, loss=11.030, backward_time=0.281, grad_norm=22.367, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.584
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:51:27,842 (trainer:732) INFO: 51epoch:train:511-561batch: iter_time=3.955e-04, forward_time=0.201, loss_att=11.073, acc=0.979, loss=11.073, backward_time=0.278, grad_norm=23.151, clip=100.000, loss_scale=1.000, optim_step_time=0.067, optim0_lr0=0.001, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:52:00,513 (trainer:732) INFO: 51epoch:train:562-612batch: iter_time=3.289e-04, forward_time=0.202, loss_att=10.696, acc=0.980, loss=10.696, backward_time=0.280, grad_norm=20.957, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:52:32,949 (trainer:732) INFO: 51epoch:train:613-663batch: iter_time=4.005e-04, forward_time=0.202, loss_att=10.975, acc=0.979, loss=10.975, backward_time=0.279, grad_norm=22.550, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:53:05,926 (trainer:732) INFO: 51epoch:train:664-714batch: iter_time=3.118e-04, forward_time=0.204, loss_att=11.155, acc=0.979, loss=11.155, backward_time=0.282, grad_norm=21.829, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.584
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:53:38,644 (trainer:732) INFO: 51epoch:train:715-765batch: iter_time=3.459e-04, forward_time=0.204, loss_att=11.138, acc=0.979, loss=11.138, backward_time=0.281, grad_norm=22.744, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:54:11,483 (trainer:732) INFO: 51epoch:train:766-816batch: iter_time=3.954e-04, forward_time=0.204, loss_att=11.075, acc=0.980, loss=11.075, backward_time=0.282, grad_norm=22.627, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:54:43,882 (trainer:732) INFO: 51epoch:train:817-867batch: iter_time=3.794e-04, forward_time=0.203, loss_att=11.062, acc=0.979, loss=11.062, backward_time=0.280, grad_norm=23.611, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.551
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:55:16,391 (trainer:732) INFO: 51epoch:train:868-918batch: iter_time=3.896e-04, forward_time=0.203, loss_att=10.372, acc=0.980, loss=10.372, backward_time=0.279, grad_norm=21.572, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:55:48,634 (trainer:732) INFO: 51epoch:train:919-969batch: iter_time=3.760e-04, forward_time=0.201, loss_att=10.560, acc=0.980, loss=10.560, backward_time=0.277, grad_norm=21.413, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 11:56:21,541 (trainer:732) INFO: 51epoch:train:970-1020batch: iter_time=3.537e-04, forward_time=0.203, loss_att=11.301, acc=0.979, loss=11.301, backward_time=0.282, grad_norm=23.090, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.570
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:06:59,464 (trainer:338) INFO: 51epoch results: [train] iter_time=9.646e-04, forward_time=0.202, loss_att=10.766, acc=0.980, loss=10.766, backward_time=0.279, grad_norm=22.200, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.123, time=13 minutes and 30.98 seconds, total_count=52887, gpu_max_cached_mem_GB=30.428, [valid] loss_att=73.244, acc=0.894, cer=0.134, wer=0.329, loss=73.244, time=4 minutes and 53.52 seconds, total_count=4488, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.37 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:07:03,687 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:07:03,703 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/41epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:07:03,703 (trainer:272) INFO: 52/60epoch started. Estimated time to finish: 3 hours, 35 minutes and 2.73 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:10:06,393 (trainer:732) INFO: 52epoch:train:1-51batch: iter_time=0.013, forward_time=0.205, loss_att=10.799, acc=0.980, loss=10.799, backward_time=0.282, grad_norm=22.757, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=15.068
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:10:39,295 (trainer:732) INFO: 52epoch:train:52-102batch: iter_time=3.248e-04, forward_time=0.204, loss_att=10.460, acc=0.980, loss=10.460, backward_time=0.283, grad_norm=23.003, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.584
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:11:11,668 (trainer:732) INFO: 52epoch:train:103-153batch: iter_time=3.312e-04, forward_time=0.200, loss_att=11.013, acc=0.980, loss=11.013, backward_time=0.278, grad_norm=22.548, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:11:44,026 (trainer:732) INFO: 52epoch:train:154-204batch: iter_time=3.551e-04, forward_time=0.200, loss_att=10.248, acc=0.981, loss=10.248, backward_time=0.277, grad_norm=20.973, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:12:16,706 (trainer:732) INFO: 52epoch:train:205-255batch: iter_time=3.404e-04, forward_time=0.202, loss_att=10.197, acc=0.981, loss=10.197, backward_time=0.279, grad_norm=20.404, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.577
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:12:48,837 (trainer:732) INFO: 52epoch:train:256-306batch: iter_time=3.580e-04, forward_time=0.200, loss_att=9.784, acc=0.981, loss=9.784, backward_time=0.275, grad_norm=21.108, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:13:21,524 (trainer:732) INFO: 52epoch:train:307-357batch: iter_time=4.022e-04, forward_time=0.203, loss_att=10.832, acc=0.980, loss=10.832, backward_time=0.280, grad_norm=22.262, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:13:54,116 (trainer:732) INFO: 52epoch:train:358-408batch: iter_time=3.590e-04, forward_time=0.202, loss_att=10.320, acc=0.980, loss=10.320, backward_time=0.278, grad_norm=22.722, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:14:26,876 (trainer:732) INFO: 52epoch:train:409-459batch: iter_time=3.748e-04, forward_time=0.205, loss_att=10.956, acc=0.980, loss=10.956, backward_time=0.283, grad_norm=22.512, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.590
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:14:59,217 (trainer:732) INFO: 52epoch:train:460-510batch: iter_time=3.365e-04, forward_time=0.200, loss_att=10.820, acc=0.980, loss=10.820, backward_time=0.278, grad_norm=22.938, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:15:32,136 (trainer:732) INFO: 52epoch:train:511-561batch: iter_time=3.801e-04, forward_time=0.204, loss_att=11.141, acc=0.979, loss=11.141, backward_time=0.281, grad_norm=23.973, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:16:04,585 (trainer:732) INFO: 52epoch:train:562-612batch: iter_time=3.701e-04, forward_time=0.201, loss_att=10.240, acc=0.981, loss=10.240, backward_time=0.278, grad_norm=22.061, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:16:36,824 (trainer:732) INFO: 52epoch:train:613-663batch: iter_time=3.774e-04, forward_time=0.202, loss_att=10.687, acc=0.979, loss=10.687, backward_time=0.279, grad_norm=20.714, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:17:09,417 (trainer:732) INFO: 52epoch:train:664-714batch: iter_time=3.593e-04, forward_time=0.202, loss_att=10.713, acc=0.980, loss=10.713, backward_time=0.279, grad_norm=22.986, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:17:41,949 (trainer:732) INFO: 52epoch:train:715-765batch: iter_time=3.413e-04, forward_time=0.202, loss_att=10.596, acc=0.980, loss=10.596, backward_time=0.280, grad_norm=20.939, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:18:14,332 (trainer:732) INFO: 52epoch:train:766-816batch: iter_time=3.812e-04, forward_time=0.201, loss_att=9.886, acc=0.981, loss=9.886, backward_time=0.277, grad_norm=20.327, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:18:47,003 (trainer:732) INFO: 52epoch:train:817-867batch: iter_time=3.584e-04, forward_time=0.203, loss_att=11.077, acc=0.980, loss=11.077, backward_time=0.282, grad_norm=21.772, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.575
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:19:19,287 (trainer:732) INFO: 52epoch:train:868-918batch: iter_time=3.758e-04, forward_time=0.201, loss_att=10.209, acc=0.980, loss=10.209, backward_time=0.276, grad_norm=20.902, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.529
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:19:51,316 (trainer:732) INFO: 52epoch:train:919-969batch: iter_time=3.249e-04, forward_time=0.199, loss_att=11.297, acc=0.979, loss=11.297, backward_time=0.276, grad_norm=25.213, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=2.510
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:20:23,703 (trainer:732) INFO: 52epoch:train:970-1020batch: iter_time=3.178e-04, forward_time=0.200, loss_att=10.450, acc=0.980, loss=10.450, backward_time=0.276, grad_norm=21.876, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:31:02,371 (trainer:338) INFO: 52epoch results: [train] iter_time=9.752e-04, forward_time=0.202, loss_att=10.561, acc=0.980, loss=10.561, backward_time=0.279, grad_norm=22.126, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.128, time=13 minutes and 32.02 seconds, total_count=53924, gpu_max_cached_mem_GB=30.428, [valid] loss_att=72.319, acc=0.895, cer=0.129, wer=0.327, loss=72.319, time=4 minutes and 52.73 seconds, total_count=4576, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 33.91 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:31:06,682 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:31:06,694 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/42epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:31:06,695 (trainer:272) INFO: 53/60epoch started. Estimated time to finish: 3 hours, 11 minutes and 10.54 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:34:09,283 (trainer:732) INFO: 53epoch:train:1-51batch: iter_time=0.015, forward_time=0.202, loss_att=9.442, acc=0.982, loss=9.442, backward_time=0.277, grad_norm=21.731, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=15.068
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:34:41,750 (trainer:732) INFO: 53epoch:train:52-102batch: iter_time=3.590e-04, forward_time=0.202, loss_att=9.440, acc=0.982, loss=9.440, backward_time=0.279, grad_norm=23.426, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:35:14,178 (trainer:732) INFO: 53epoch:train:103-153batch: iter_time=3.622e-04, forward_time=0.201, loss_att=9.364, acc=0.982, loss=9.364, backward_time=0.279, grad_norm=21.408, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:35:46,666 (trainer:732) INFO: 53epoch:train:154-204batch: iter_time=3.645e-04, forward_time=0.202, loss_att=9.854, acc=0.981, loss=9.854, backward_time=0.278, grad_norm=22.833, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:36:19,053 (trainer:732) INFO: 53epoch:train:205-255batch: iter_time=3.487e-04, forward_time=0.202, loss_att=10.040, acc=0.982, loss=10.040, backward_time=0.280, grad_norm=22.174, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:36:51,796 (trainer:732) INFO: 53epoch:train:256-306batch: iter_time=3.733e-04, forward_time=0.202, loss_att=9.452, acc=0.982, loss=9.452, backward_time=0.281, grad_norm=21.629, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.563
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:37:24,109 (trainer:732) INFO: 53epoch:train:307-357batch: iter_time=3.564e-04, forward_time=0.199, loss_att=10.060, acc=0.981, loss=10.060, backward_time=0.275, grad_norm=22.035, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:37:56,774 (trainer:732) INFO: 53epoch:train:358-408batch: iter_time=3.500e-04, forward_time=0.203, loss_att=9.855, acc=0.981, loss=9.855, backward_time=0.280, grad_norm=20.811, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:38:28,574 (trainer:732) INFO: 53epoch:train:409-459batch: iter_time=3.265e-04, forward_time=0.198, loss_att=10.087, acc=0.981, loss=10.087, backward_time=0.275, grad_norm=21.391, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.508
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:39:01,369 (trainer:732) INFO: 53epoch:train:460-510batch: iter_time=3.613e-04, forward_time=0.203, loss_att=10.484, acc=0.981, loss=10.484, backward_time=0.280, grad_norm=21.208, clip=100.000, loss_scale=1.000, optim_step_time=0.067, optim0_lr0=0.001, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:39:33,980 (trainer:732) INFO: 53epoch:train:511-561batch: iter_time=3.806e-04, forward_time=0.203, loss_att=10.885, acc=0.980, loss=10.885, backward_time=0.279, grad_norm=22.427, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:40:06,478 (trainer:732) INFO: 53epoch:train:562-612batch: iter_time=3.231e-04, forward_time=0.202, loss_att=10.542, acc=0.980, loss=10.542, backward_time=0.279, grad_norm=21.275, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:40:38,818 (trainer:732) INFO: 53epoch:train:613-663batch: iter_time=3.502e-04, forward_time=0.202, loss_att=10.416, acc=0.980, loss=10.416, backward_time=0.279, grad_norm=21.222, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:41:11,579 (trainer:732) INFO: 53epoch:train:664-714batch: iter_time=3.450e-04, forward_time=0.202, loss_att=10.216, acc=0.981, loss=10.216, backward_time=0.281, grad_norm=21.548, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:41:44,265 (trainer:732) INFO: 53epoch:train:715-765batch: iter_time=2.966e-04, forward_time=0.202, loss_att=10.405, acc=0.980, loss=10.405, backward_time=0.281, grad_norm=22.288, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:42:16,821 (trainer:732) INFO: 53epoch:train:766-816batch: iter_time=3.709e-04, forward_time=0.201, loss_att=10.269, acc=0.981, loss=10.269, backward_time=0.277, grad_norm=21.316, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:42:49,190 (trainer:732) INFO: 53epoch:train:817-867batch: iter_time=3.555e-04, forward_time=0.202, loss_att=10.710, acc=0.980, loss=10.710, backward_time=0.278, grad_norm=22.848, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:43:21,561 (trainer:732) INFO: 53epoch:train:868-918batch: iter_time=3.410e-04, forward_time=0.201, loss_att=10.307, acc=0.981, loss=10.307, backward_time=0.279, grad_norm=22.012, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:43:53,890 (trainer:732) INFO: 53epoch:train:919-969batch: iter_time=3.511e-04, forward_time=0.201, loss_att=10.086, acc=0.981, loss=10.086, backward_time=0.277, grad_norm=21.360, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:44:27,061 (trainer:732) INFO: 53epoch:train:970-1020batch: iter_time=2.905e-04, forward_time=0.204, loss_att=11.302, acc=0.980, loss=11.302, backward_time=0.285, grad_norm=23.799, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.592
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:55:10,961 (trainer:338) INFO: 53epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=10.142, acc=0.981, loss=10.142, backward_time=0.279, grad_norm=21.885, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.128, time=13 minutes and 32.23 seconds, total_count=54961, gpu_max_cached_mem_GB=30.428, [valid] loss_att=71.806, acc=0.896, cer=0.130, wer=0.328, loss=71.806, time=4 minutes and 54.57 seconds, total_count=4664, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 37.47 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:55:15,238 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:55:15,252 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/43epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:55:15,252 (trainer:272) INFO: 54/60epoch started. Estimated time to finish: 2 hours, 47 minutes and 18.67 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:58:16,710 (trainer:732) INFO: 54epoch:train:1-51batch: iter_time=0.016, forward_time=0.204, loss_att=9.669, acc=0.982, loss=9.669, backward_time=0.278, grad_norm=19.605, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=14.976
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:58:48,645 (trainer:732) INFO: 54epoch:train:52-102batch: iter_time=3.421e-04, forward_time=0.199, loss_att=9.492, acc=0.982, loss=9.492, backward_time=0.275, grad_norm=20.499, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.505
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:59:21,350 (trainer:732) INFO: 54epoch:train:103-153batch: iter_time=3.673e-04, forward_time=0.201, loss_att=10.396, acc=0.981, loss=10.396, backward_time=0.280, grad_norm=22.133, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.560
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 12:59:53,815 (trainer:732) INFO: 54epoch:train:154-204batch: iter_time=3.317e-04, forward_time=0.201, loss_att=9.655, acc=0.981, loss=9.655, backward_time=0.277, grad_norm=20.994, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:00:25,919 (trainer:732) INFO: 54epoch:train:205-255batch: iter_time=3.475e-04, forward_time=0.201, loss_att=10.017, acc=0.981, loss=10.017, backward_time=0.277, grad_norm=20.356, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:00:58,186 (trainer:732) INFO: 54epoch:train:256-306batch: iter_time=3.339e-04, forward_time=0.200, loss_att=9.733, acc=0.982, loss=9.733, backward_time=0.277, grad_norm=20.674, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:01:30,511 (trainer:732) INFO: 54epoch:train:307-357batch: iter_time=3.640e-04, forward_time=0.202, loss_att=9.090, acc=0.983, loss=9.090, backward_time=0.278, grad_norm=19.778, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:02:02,820 (trainer:732) INFO: 54epoch:train:358-408batch: iter_time=3.750e-04, forward_time=0.201, loss_att=9.680, acc=0.981, loss=9.680, backward_time=0.276, grad_norm=20.761, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:02:35,018 (trainer:732) INFO: 54epoch:train:409-459batch: iter_time=3.556e-04, forward_time=0.201, loss_att=10.730, acc=0.980, loss=10.730, backward_time=0.277, grad_norm=22.322, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:03:08,078 (trainer:732) INFO: 54epoch:train:460-510batch: iter_time=3.556e-04, forward_time=0.205, loss_att=9.954, acc=0.981, loss=9.954, backward_time=0.284, grad_norm=21.448, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.583
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:03:40,634 (trainer:732) INFO: 54epoch:train:511-561batch: iter_time=3.481e-04, forward_time=0.201, loss_att=10.916, acc=0.980, loss=10.916, backward_time=0.279, grad_norm=22.879, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:04:13,321 (trainer:732) INFO: 54epoch:train:562-612batch: iter_time=3.405e-04, forward_time=0.202, loss_att=10.159, acc=0.981, loss=10.159, backward_time=0.280, grad_norm=23.801, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.554
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:04:46,066 (trainer:732) INFO: 54epoch:train:613-663batch: iter_time=3.294e-04, forward_time=0.204, loss_att=10.065, acc=0.981, loss=10.065, backward_time=0.283, grad_norm=21.679, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.583
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:05:18,442 (trainer:732) INFO: 54epoch:train:664-714batch: iter_time=3.437e-04, forward_time=0.201, loss_att=9.611, acc=0.981, loss=9.611, backward_time=0.278, grad_norm=20.027, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:05:50,598 (trainer:732) INFO: 54epoch:train:715-765batch: iter_time=3.028e-04, forward_time=0.200, loss_att=10.016, acc=0.981, loss=10.016, backward_time=0.276, grad_norm=22.113, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.523
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:06:22,972 (trainer:732) INFO: 54epoch:train:766-816batch: iter_time=3.580e-04, forward_time=0.201, loss_att=9.630, acc=0.982, loss=9.630, backward_time=0.277, grad_norm=19.923, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:06:55,411 (trainer:732) INFO: 54epoch:train:817-867batch: iter_time=3.310e-04, forward_time=0.203, loss_att=10.489, acc=0.980, loss=10.489, backward_time=0.280, grad_norm=21.371, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:07:28,381 (trainer:732) INFO: 54epoch:train:868-918batch: iter_time=3.749e-04, forward_time=0.205, loss_att=10.752, acc=0.981, loss=10.752, backward_time=0.285, grad_norm=24.240, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.585
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:08:00,743 (trainer:732) INFO: 54epoch:train:919-969batch: iter_time=3.229e-04, forward_time=0.200, loss_att=9.562, acc=0.982, loss=9.562, backward_time=0.277, grad_norm=20.730, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:08:33,492 (trainer:732) INFO: 54epoch:train:970-1020batch: iter_time=3.545e-04, forward_time=0.204, loss_att=10.137, acc=0.981, loss=10.137, backward_time=0.280, grad_norm=21.006, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:19:18,023 (trainer:338) INFO: 54epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=9.995, acc=0.981, loss=9.995, backward_time=0.279, grad_norm=21.424, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.122, time=13 minutes and 30.56 seconds, total_count=55998, gpu_max_cached_mem_GB=30.428, [valid] loss_att=71.708, acc=0.896, cer=0.130, wer=0.328, loss=71.708, time=4 minutes and 58.31 seconds, total_count=4752, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 33.89 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:19:22,493 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:19:22,506 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/44epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:19:22,506 (trainer:272) INFO: 55/60epoch started. Estimated time to finish: 2 hours, 23 minutes and 26.03 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:22:23,724 (trainer:732) INFO: 55epoch:train:1-51batch: iter_time=0.014, forward_time=0.202, loss_att=9.438, acc=0.982, loss=9.438, backward_time=0.277, grad_norm=20.818, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=14.957
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:22:56,092 (trainer:732) INFO: 55epoch:train:52-102batch: iter_time=3.391e-04, forward_time=0.201, loss_att=8.953, acc=0.983, loss=8.953, backward_time=0.278, grad_norm=18.895, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:23:28,453 (trainer:732) INFO: 55epoch:train:103-153batch: iter_time=3.476e-04, forward_time=0.200, loss_att=9.358, acc=0.982, loss=9.358, backward_time=0.277, grad_norm=21.732, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:24:01,401 (trainer:732) INFO: 55epoch:train:154-204batch: iter_time=3.441e-04, forward_time=0.204, loss_att=10.093, acc=0.982, loss=10.093, backward_time=0.283, grad_norm=21.121, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.577
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:24:33,624 (trainer:732) INFO: 55epoch:train:205-255batch: iter_time=3.701e-04, forward_time=0.201, loss_att=9.572, acc=0.982, loss=9.572, backward_time=0.278, grad_norm=20.890, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:25:06,238 (trainer:732) INFO: 55epoch:train:256-306batch: iter_time=3.587e-04, forward_time=0.202, loss_att=9.299, acc=0.983, loss=9.299, backward_time=0.280, grad_norm=20.945, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:25:39,001 (trainer:732) INFO: 55epoch:train:307-357batch: iter_time=3.460e-04, forward_time=0.202, loss_att=9.555, acc=0.982, loss=9.555, backward_time=0.281, grad_norm=21.830, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.572
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:26:11,460 (trainer:732) INFO: 55epoch:train:358-408batch: iter_time=3.130e-04, forward_time=0.201, loss_att=10.065, acc=0.982, loss=10.065, backward_time=0.278, grad_norm=22.041, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:26:43,713 (trainer:732) INFO: 55epoch:train:409-459batch: iter_time=3.689e-04, forward_time=0.201, loss_att=9.931, acc=0.982, loss=9.931, backward_time=0.278, grad_norm=21.219, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:27:16,203 (trainer:732) INFO: 55epoch:train:460-510batch: iter_time=3.578e-04, forward_time=0.201, loss_att=10.217, acc=0.981, loss=10.217, backward_time=0.277, grad_norm=22.776, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:27:48,807 (trainer:732) INFO: 55epoch:train:511-561batch: iter_time=3.850e-04, forward_time=0.202, loss_att=9.365, acc=0.983, loss=9.365, backward_time=0.280, grad_norm=20.307, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:28:21,325 (trainer:732) INFO: 55epoch:train:562-612batch: iter_time=3.705e-04, forward_time=0.202, loss_att=9.663, acc=0.982, loss=9.663, backward_time=0.279, grad_norm=20.624, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:28:54,071 (trainer:732) INFO: 55epoch:train:613-663batch: iter_time=4.862e-04, forward_time=0.205, loss_att=9.446, acc=0.982, loss=9.446, backward_time=0.283, grad_norm=20.223, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.588
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:29:26,140 (trainer:732) INFO: 55epoch:train:664-714batch: iter_time=3.614e-04, forward_time=0.199, loss_att=9.540, acc=0.981, loss=9.540, backward_time=0.275, grad_norm=19.707, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.506
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:29:58,541 (trainer:732) INFO: 55epoch:train:715-765batch: iter_time=3.389e-04, forward_time=0.201, loss_att=9.850, acc=0.981, loss=9.850, backward_time=0.278, grad_norm=21.707, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:30:31,058 (trainer:732) INFO: 55epoch:train:766-816batch: iter_time=3.548e-04, forward_time=0.202, loss_att=9.888, acc=0.981, loss=9.888, backward_time=0.278, grad_norm=23.394, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:31:03,438 (trainer:732) INFO: 55epoch:train:817-867batch: iter_time=3.448e-04, forward_time=0.202, loss_att=9.271, acc=0.983, loss=9.271, backward_time=0.280, grad_norm=20.242, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:31:36,314 (trainer:732) INFO: 55epoch:train:868-918batch: iter_time=3.501e-04, forward_time=0.204, loss_att=10.436, acc=0.981, loss=10.436, backward_time=0.283, grad_norm=23.281, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.576
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:32:09,033 (trainer:732) INFO: 55epoch:train:919-969batch: iter_time=3.476e-04, forward_time=0.203, loss_att=10.056, acc=0.982, loss=10.056, backward_time=0.281, grad_norm=22.158, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:32:41,463 (trainer:732) INFO: 55epoch:train:970-1020batch: iter_time=3.357e-04, forward_time=0.201, loss_att=9.528, acc=0.982, loss=9.528, backward_time=0.277, grad_norm=19.981, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:43:19,473 (trainer:338) INFO: 55epoch results: [train] iter_time=0.001, forward_time=0.202, loss_att=9.667, acc=0.982, loss=9.667, backward_time=0.279, grad_norm=21.158, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.123, time=13 minutes and 30.85 seconds, total_count=57035, gpu_max_cached_mem_GB=30.428, [valid] loss_att=72.008, acc=0.896, cer=0.129, wer=0.328, loss=72.008, time=4 minutes and 53.86 seconds, total_count=4840, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.25 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:43:23,986 (trainer:384) INFO: There are no improvements in this epoch
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:43:24,024 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/45epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:43:24,024 (trainer:272) INFO: 56/60epoch started. Estimated time to finish: 1 hour, 59 minutes and 32.35 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:46:24,720 (trainer:732) INFO: 56epoch:train:1-51batch: iter_time=0.010, forward_time=0.202, loss_att=8.644, acc=0.983, loss=8.644, backward_time=0.276, grad_norm=19.727, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=14.913
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:46:57,171 (trainer:732) INFO: 56epoch:train:52-102batch: iter_time=3.411e-04, forward_time=0.202, loss_att=9.108, acc=0.983, loss=9.108, backward_time=0.279, grad_norm=22.323, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:47:29,668 (trainer:732) INFO: 56epoch:train:103-153batch: iter_time=3.585e-04, forward_time=0.202, loss_att=8.814, acc=0.984, loss=8.814, backward_time=0.279, grad_norm=20.050, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:48:01,974 (trainer:732) INFO: 56epoch:train:154-204batch: iter_time=3.599e-04, forward_time=0.200, loss_att=9.436, acc=0.982, loss=9.436, backward_time=0.277, grad_norm=21.281, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:48:34,300 (trainer:732) INFO: 56epoch:train:205-255batch: iter_time=3.874e-04, forward_time=0.202, loss_att=9.789, acc=0.982, loss=9.789, backward_time=0.277, grad_norm=21.994, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:49:07,044 (trainer:732) INFO: 56epoch:train:256-306batch: iter_time=3.757e-04, forward_time=0.203, loss_att=9.344, acc=0.982, loss=9.344, backward_time=0.280, grad_norm=21.586, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.571
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:49:39,464 (trainer:732) INFO: 56epoch:train:307-357batch: iter_time=3.876e-04, forward_time=0.202, loss_att=8.763, acc=0.983, loss=8.763, backward_time=0.278, grad_norm=20.155, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:50:11,691 (trainer:732) INFO: 56epoch:train:358-408batch: iter_time=3.819e-04, forward_time=0.200, loss_att=8.959, acc=0.983, loss=8.959, backward_time=0.275, grad_norm=20.767, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.517
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:50:43,809 (trainer:732) INFO: 56epoch:train:409-459batch: iter_time=3.703e-04, forward_time=0.201, loss_att=9.077, acc=0.983, loss=9.077, backward_time=0.278, grad_norm=18.969, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.530
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:51:16,299 (trainer:732) INFO: 56epoch:train:460-510batch: iter_time=3.590e-04, forward_time=0.202, loss_att=9.957, acc=0.982, loss=9.957, backward_time=0.279, grad_norm=20.400, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.548
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:51:48,817 (trainer:732) INFO: 56epoch:train:511-561batch: iter_time=3.896e-04, forward_time=0.202, loss_att=9.736, acc=0.982, loss=9.736, backward_time=0.279, grad_norm=21.680, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:52:21,500 (trainer:732) INFO: 56epoch:train:562-612batch: iter_time=3.776e-04, forward_time=0.202, loss_att=9.665, acc=0.982, loss=9.665, backward_time=0.279, grad_norm=19.917, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:52:54,033 (trainer:732) INFO: 56epoch:train:613-663batch: iter_time=3.709e-04, forward_time=0.203, loss_att=9.142, acc=0.983, loss=9.142, backward_time=0.281, grad_norm=20.515, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:53:26,466 (trainer:732) INFO: 56epoch:train:664-714batch: iter_time=3.324e-04, forward_time=0.201, loss_att=9.732, acc=0.982, loss=9.732, backward_time=0.279, grad_norm=21.630, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:53:58,829 (trainer:732) INFO: 56epoch:train:715-765batch: iter_time=3.442e-04, forward_time=0.202, loss_att=9.070, acc=0.983, loss=9.070, backward_time=0.279, grad_norm=21.877, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:54:31,716 (trainer:732) INFO: 56epoch:train:766-816batch: iter_time=3.672e-04, forward_time=0.204, loss_att=9.917, acc=0.982, loss=9.917, backward_time=0.282, grad_norm=21.143, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.566
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:55:04,511 (trainer:732) INFO: 56epoch:train:817-867batch: iter_time=3.618e-04, forward_time=0.205, loss_att=9.236, acc=0.983, loss=9.236, backward_time=0.284, grad_norm=20.855, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.588
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:55:37,042 (trainer:732) INFO: 56epoch:train:868-918batch: iter_time=3.600e-04, forward_time=0.202, loss_att=9.882, acc=0.982, loss=9.882, backward_time=0.280, grad_norm=20.738, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:56:09,467 (trainer:732) INFO: 56epoch:train:919-969batch: iter_time=3.596e-04, forward_time=0.201, loss_att=9.538, acc=0.982, loss=9.538, backward_time=0.277, grad_norm=22.559, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 13:56:41,546 (trainer:732) INFO: 56epoch:train:970-1020batch: iter_time=3.115e-04, forward_time=0.199, loss_att=9.265, acc=0.982, loss=9.265, backward_time=0.274, grad_norm=20.862, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.505
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:07:25,617 (trainer:338) INFO: 56epoch results: [train] iter_time=8.567e-04, forward_time=0.202, loss_att=9.354, acc=0.982, loss=9.354, backward_time=0.279, grad_norm=20.973, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.119, time=13 minutes and 29.78 seconds, total_count=58072, gpu_max_cached_mem_GB=30.428, [valid] loss_att=71.103, acc=0.898, cer=0.126, wer=0.319, loss=71.103, time=4 minutes and 58.39 seconds, total_count=4928, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 33.42 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:07:30,052 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:07:30,066 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/46epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:07:30,067 (trainer:272) INFO: 57/60epoch started. Estimated time to finish: 1 hour, 35 minutes and 38.7 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:10:34,491 (trainer:732) INFO: 57epoch:train:1-51batch: iter_time=0.011, forward_time=0.205, loss_att=8.555, acc=0.984, loss=8.555, backward_time=0.280, grad_norm=19.024, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=15.218
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:11:06,826 (trainer:732) INFO: 57epoch:train:52-102batch: iter_time=3.181e-04, forward_time=0.201, loss_att=8.912, acc=0.982, loss=8.912, backward_time=0.276, grad_norm=20.074, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:11:39,424 (trainer:732) INFO: 57epoch:train:103-153batch: iter_time=3.488e-04, forward_time=0.202, loss_att=9.495, acc=0.983, loss=9.495, backward_time=0.281, grad_norm=22.338, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:12:11,829 (trainer:732) INFO: 57epoch:train:154-204batch: iter_time=3.175e-04, forward_time=0.200, loss_att=8.503, acc=0.984, loss=8.503, backward_time=0.276, grad_norm=19.826, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:12:44,020 (trainer:732) INFO: 57epoch:train:205-255batch: iter_time=3.074e-04, forward_time=0.201, loss_att=8.670, acc=0.983, loss=8.670, backward_time=0.279, grad_norm=21.775, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:13:16,294 (trainer:732) INFO: 57epoch:train:256-306batch: iter_time=3.004e-04, forward_time=0.201, loss_att=8.648, acc=0.984, loss=8.648, backward_time=0.276, grad_norm=20.803, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:13:49,057 (trainer:732) INFO: 57epoch:train:307-357batch: iter_time=3.406e-04, forward_time=0.203, loss_att=9.816, acc=0.982, loss=9.816, backward_time=0.281, grad_norm=20.672, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.562
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:14:21,994 (trainer:732) INFO: 57epoch:train:358-408batch: iter_time=3.028e-04, forward_time=0.203, loss_att=8.827, acc=0.984, loss=8.827, backward_time=0.282, grad_norm=19.593, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:14:54,084 (trainer:732) INFO: 57epoch:train:409-459batch: iter_time=3.336e-04, forward_time=0.200, loss_att=9.062, acc=0.983, loss=9.062, backward_time=0.278, grad_norm=21.588, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:15:26,494 (trainer:732) INFO: 57epoch:train:460-510batch: iter_time=2.998e-04, forward_time=0.200, loss_att=9.341, acc=0.983, loss=9.341, backward_time=0.278, grad_norm=21.451, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:15:58,963 (trainer:732) INFO: 57epoch:train:511-561batch: iter_time=3.204e-04, forward_time=0.201, loss_att=8.922, acc=0.983, loss=8.922, backward_time=0.279, grad_norm=19.854, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:16:31,287 (trainer:732) INFO: 57epoch:train:562-612batch: iter_time=3.054e-04, forward_time=0.200, loss_att=8.551, acc=0.983, loss=8.551, backward_time=0.276, grad_norm=20.654, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:17:03,465 (trainer:732) INFO: 57epoch:train:613-663batch: iter_time=3.209e-04, forward_time=0.201, loss_att=8.960, acc=0.983, loss=8.960, backward_time=0.277, grad_norm=18.767, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:17:35,721 (trainer:732) INFO: 57epoch:train:664-714batch: iter_time=2.903e-04, forward_time=0.199, loss_att=8.951, acc=0.983, loss=8.951, backward_time=0.277, grad_norm=19.624, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:18:08,720 (trainer:732) INFO: 57epoch:train:715-765batch: iter_time=3.004e-04, forward_time=0.204, loss_att=9.535, acc=0.982, loss=9.535, backward_time=0.283, grad_norm=20.226, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.586
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:18:40,779 (trainer:732) INFO: 57epoch:train:766-816batch: iter_time=3.200e-04, forward_time=0.199, loss_att=9.074, acc=0.983, loss=9.074, backward_time=0.273, grad_norm=19.963, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.502
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:19:13,050 (trainer:732) INFO: 57epoch:train:817-867batch: iter_time=3.029e-04, forward_time=0.201, loss_att=9.512, acc=0.982, loss=9.512, backward_time=0.278, grad_norm=21.880, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.547
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:19:45,979 (trainer:732) INFO: 57epoch:train:868-918batch: iter_time=3.110e-04, forward_time=0.204, loss_att=10.243, acc=0.982, loss=10.243, backward_time=0.283, grad_norm=20.411, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:20:18,321 (trainer:732) INFO: 57epoch:train:919-969batch: iter_time=2.872e-04, forward_time=0.201, loss_att=9.327, acc=0.983, loss=9.327, backward_time=0.277, grad_norm=19.630, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:20:51,422 (trainer:732) INFO: 57epoch:train:970-1020batch: iter_time=2.792e-04, forward_time=0.202, loss_att=9.520, acc=0.983, loss=9.520, backward_time=0.282, grad_norm=20.366, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=2.583
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:31:28,156 (trainer:338) INFO: 57epoch results: [train] iter_time=8.125e-04, forward_time=0.201, loss_att=9.096, acc=0.983, loss=9.096, backward_time=0.279, grad_norm=20.374, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.133, time=13 minutes and 33.29 seconds, total_count=59109, gpu_max_cached_mem_GB=30.428, [valid] loss_att=73.232, acc=0.895, cer=0.129, wer=0.329, loss=73.232, time=4 minutes and 52.36 seconds, total_count=5016, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 32.43 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:31:32,627 (trainer:384) INFO: There are no improvements in this epoch
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:31:32,642 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/48epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:31:32,643 (trainer:272) INFO: 58/60epoch started. Estimated time to finish: 1 hour, 11 minutes and 44.44 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:34:33,602 (trainer:732) INFO: 58epoch:train:1-51batch: iter_time=0.009, forward_time=0.203, loss_att=9.106, acc=0.983, loss=9.106, backward_time=0.279, grad_norm=21.979, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=14.928
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:35:05,668 (trainer:732) INFO: 58epoch:train:52-102batch: iter_time=3.175e-04, forward_time=0.200, loss_att=8.538, acc=0.983, loss=8.538, backward_time=0.274, grad_norm=19.744, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.518
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:35:38,011 (trainer:732) INFO: 58epoch:train:103-153batch: iter_time=3.469e-04, forward_time=0.200, loss_att=9.148, acc=0.983, loss=9.148, backward_time=0.277, grad_norm=20.703, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.538
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:36:10,342 (trainer:732) INFO: 58epoch:train:154-204batch: iter_time=3.098e-04, forward_time=0.200, loss_att=9.246, acc=0.983, loss=9.246, backward_time=0.277, grad_norm=20.425, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.524
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:36:42,711 (trainer:732) INFO: 58epoch:train:205-255batch: iter_time=3.453e-04, forward_time=0.201, loss_att=8.692, acc=0.984, loss=8.692, backward_time=0.280, grad_norm=20.246, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.556
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:37:15,089 (trainer:732) INFO: 58epoch:train:256-306batch: iter_time=3.215e-04, forward_time=0.200, loss_att=9.285, acc=0.983, loss=9.285, backward_time=0.278, grad_norm=20.069, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=2.536
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:37:47,900 (trainer:732) INFO: 58epoch:train:307-357batch: iter_time=3.311e-04, forward_time=0.204, loss_att=9.290, acc=0.983, loss=9.290, backward_time=0.281, grad_norm=20.067, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:38:20,873 (trainer:732) INFO: 58epoch:train:358-408batch: iter_time=3.383e-04, forward_time=0.204, loss_att=9.396, acc=0.982, loss=9.396, backward_time=0.283, grad_norm=19.272, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.577
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:38:52,997 (trainer:732) INFO: 58epoch:train:409-459batch: iter_time=3.449e-04, forward_time=0.200, loss_att=9.443, acc=0.982, loss=9.443, backward_time=0.277, grad_norm=21.321, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:39:25,910 (trainer:732) INFO: 58epoch:train:460-510batch: iter_time=3.377e-04, forward_time=0.205, loss_att=10.092, acc=0.982, loss=10.092, backward_time=0.283, grad_norm=21.864, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:39:58,133 (trainer:732) INFO: 58epoch:train:511-561batch: iter_time=3.414e-04, forward_time=0.200, loss_att=8.646, acc=0.983, loss=8.646, backward_time=0.276, grad_norm=18.004, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.527
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:40:30,689 (trainer:732) INFO: 58epoch:train:562-612batch: iter_time=3.535e-04, forward_time=0.201, loss_att=9.368, acc=0.982, loss=9.368, backward_time=0.278, grad_norm=19.858, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:41:03,319 (trainer:732) INFO: 58epoch:train:613-663batch: iter_time=3.496e-04, forward_time=0.202, loss_att=9.539, acc=0.982, loss=9.539, backward_time=0.280, grad_norm=19.425, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:41:35,977 (trainer:732) INFO: 58epoch:train:664-714batch: iter_time=3.153e-04, forward_time=0.203, loss_att=9.363, acc=0.983, loss=9.363, backward_time=0.281, grad_norm=19.852, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=2.558
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:42:08,093 (trainer:732) INFO: 58epoch:train:715-765batch: iter_time=3.083e-04, forward_time=0.200, loss_att=8.850, acc=0.983, loss=8.850, backward_time=0.275, grad_norm=18.614, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.516
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:42:40,695 (trainer:732) INFO: 58epoch:train:766-816batch: iter_time=3.280e-04, forward_time=0.202, loss_att=8.774, acc=0.983, loss=8.774, backward_time=0.279, grad_norm=20.369, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:43:13,166 (trainer:732) INFO: 58epoch:train:817-867batch: iter_time=3.547e-04, forward_time=0.202, loss_att=9.531, acc=0.982, loss=9.531, backward_time=0.279, grad_norm=20.491, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=0.001, train_time=2.561
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:43:45,694 (trainer:732) INFO: 58epoch:train:868-918batch: iter_time=3.414e-04, forward_time=0.200, loss_att=9.225, acc=0.983, loss=9.225, backward_time=0.277, grad_norm=19.158, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=2.550
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:44:17,863 (trainer:732) INFO: 58epoch:train:919-969batch: iter_time=3.577e-04, forward_time=0.201, loss_att=9.090, acc=0.983, loss=9.090, backward_time=0.276, grad_norm=20.401, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.519
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:44:50,844 (trainer:732) INFO: 58epoch:train:970-1020batch: iter_time=3.197e-04, forward_time=0.204, loss_att=9.015, acc=0.983, loss=9.015, backward_time=0.282, grad_norm=19.968, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.578
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:55:27,493 (trainer:338) INFO: 58epoch results: [train] iter_time=7.476e-04, forward_time=0.202, loss_att=9.167, acc=0.983, loss=9.167, backward_time=0.279, grad_norm=20.072, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.121, time=13 minutes and 30.44 seconds, total_count=60146, gpu_max_cached_mem_GB=30.428, [valid] loss_att=70.401, acc=0.899, cer=0.127, wer=0.318, loss=70.401, time=4 minutes and 51.24 seconds, total_count=5104, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 33.17 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:55:31,701 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:55:31,731 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/50epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:55:31,732 (trainer:272) INFO: 59/60epoch started. Estimated time to finish: 47 minutes and 49.78 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:58:33,057 (trainer:732) INFO: 59epoch:train:1-51batch: iter_time=0.012, forward_time=0.203, loss_att=8.026, acc=0.985, loss=8.026, backward_time=0.277, grad_norm=18.388, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=14.963
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:59:05,929 (trainer:732) INFO: 59epoch:train:52-102batch: iter_time=3.364e-04, forward_time=0.203, loss_att=8.868, acc=0.984, loss=8.868, backward_time=0.283, grad_norm=20.693, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.579
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 14:59:38,286 (trainer:732) INFO: 59epoch:train:103-153batch: iter_time=3.407e-04, forward_time=0.201, loss_att=8.653, acc=0.984, loss=8.653, backward_time=0.278, grad_norm=20.212, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:00:10,880 (trainer:732) INFO: 59epoch:train:154-204batch: iter_time=3.591e-04, forward_time=0.201, loss_att=7.911, acc=0.985, loss=7.911, backward_time=0.279, grad_norm=19.378, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:00:43,347 (trainer:732) INFO: 59epoch:train:205-255batch: iter_time=3.396e-04, forward_time=0.203, loss_att=8.709, acc=0.984, loss=8.709, backward_time=0.281, grad_norm=20.163, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.559
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:01:15,736 (trainer:732) INFO: 59epoch:train:256-306batch: iter_time=3.133e-04, forward_time=0.201, loss_att=8.662, acc=0.984, loss=8.662, backward_time=0.278, grad_norm=20.169, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:01:48,116 (trainer:732) INFO: 59epoch:train:307-357batch: iter_time=3.609e-04, forward_time=0.201, loss_att=9.023, acc=0.983, loss=9.023, backward_time=0.278, grad_norm=20.694, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.535
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:02:21,063 (trainer:732) INFO: 59epoch:train:358-408batch: iter_time=3.268e-04, forward_time=0.203, loss_att=8.762, acc=0.984, loss=8.762, backward_time=0.281, grad_norm=18.993, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=2.574
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:02:53,596 (trainer:732) INFO: 59epoch:train:409-459batch: iter_time=3.712e-04, forward_time=0.202, loss_att=8.420, acc=0.984, loss=8.420, backward_time=0.278, grad_norm=18.166, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=2.565
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:03:25,917 (trainer:732) INFO: 59epoch:train:460-510batch: iter_time=3.649e-04, forward_time=0.202, loss_att=8.388, acc=0.984, loss=8.388, backward_time=0.276, grad_norm=18.501, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.532
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:03:58,136 (trainer:732) INFO: 59epoch:train:511-561batch: iter_time=3.958e-04, forward_time=0.200, loss_att=8.707, acc=0.983, loss=8.707, backward_time=0.275, grad_norm=19.675, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:04:30,842 (trainer:732) INFO: 59epoch:train:562-612batch: iter_time=3.579e-04, forward_time=0.201, loss_att=8.919, acc=0.983, loss=8.919, backward_time=0.278, grad_norm=20.332, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.557
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:05:03,393 (trainer:732) INFO: 59epoch:train:613-663batch: iter_time=3.738e-04, forward_time=0.203, loss_att=9.054, acc=0.983, loss=9.054, backward_time=0.280, grad_norm=20.520, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:05:35,829 (trainer:732) INFO: 59epoch:train:664-714batch: iter_time=3.473e-04, forward_time=0.201, loss_att=8.627, acc=0.984, loss=8.627, backward_time=0.278, grad_norm=18.394, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:06:08,266 (trainer:732) INFO: 59epoch:train:715-765batch: iter_time=3.324e-04, forward_time=0.202, loss_att=9.058, acc=0.982, loss=9.058, backward_time=0.278, grad_norm=20.092, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:06:40,790 (trainer:732) INFO: 59epoch:train:766-816batch: iter_time=3.609e-04, forward_time=0.202, loss_att=9.406, acc=0.982, loss=9.406, backward_time=0.279, grad_norm=19.843, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.539
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:07:13,065 (trainer:732) INFO: 59epoch:train:817-867batch: iter_time=3.558e-04, forward_time=0.202, loss_att=8.704, acc=0.984, loss=8.704, backward_time=0.278, grad_norm=19.083, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.544
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:07:45,685 (trainer:732) INFO: 59epoch:train:868-918batch: iter_time=3.727e-04, forward_time=0.202, loss_att=9.263, acc=0.983, loss=9.263, backward_time=0.281, grad_norm=21.156, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:08:18,080 (trainer:732) INFO: 59epoch:train:919-969batch: iter_time=3.428e-04, forward_time=0.202, loss_att=9.363, acc=0.983, loss=9.363, backward_time=0.278, grad_norm=19.357, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=2.541
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:08:50,564 (trainer:732) INFO: 59epoch:train:970-1020batch: iter_time=2.755e-04, forward_time=0.200, loss_att=8.958, acc=0.983, loss=8.958, backward_time=0.279, grad_norm=20.056, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=2.537
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:19:30,586 (trainer:338) INFO: 59epoch results: [train] iter_time=9.016e-04, forward_time=0.202, loss_att=8.781, acc=0.984, loss=8.781, backward_time=0.279, grad_norm=19.734, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.123, time=13 minutes and 30.93 seconds, total_count=61183, gpu_max_cached_mem_GB=30.428, [valid] loss_att=71.666, acc=0.897, cer=0.133, wer=0.318, loss=71.666, time=4 minutes and 58.76 seconds, total_count=5192, gpu_max_cached_mem_GB=30.428, [att_plot] time=5 minutes and 29.16 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:19:34,765 (trainer:384) INFO: There are no improvements in this epoch
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:19:34,914 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/47epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:19:34,915 (trainer:272) INFO: 60/60epoch started. Estimated time to finish: 23 minutes and 55.03 seconds
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:22:37,352 (trainer:732) INFO: 60epoch:train:1-51batch: iter_time=0.011, forward_time=0.204, loss_att=8.332, acc=0.984, loss=8.332, backward_time=0.279, grad_norm=19.720, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=15.058
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:23:09,629 (trainer:732) INFO: 60epoch:train:52-102batch: iter_time=3.386e-04, forward_time=0.201, loss_att=8.326, acc=0.985, loss=8.326, backward_time=0.277, grad_norm=19.003, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.525
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:23:41,992 (trainer:732) INFO: 60epoch:train:103-153batch: iter_time=4.552e-04, forward_time=0.202, loss_att=8.438, acc=0.985, loss=8.438, backward_time=0.279, grad_norm=20.184, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.540
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:24:14,258 (trainer:732) INFO: 60epoch:train:154-204batch: iter_time=3.397e-04, forward_time=0.201, loss_att=8.419, acc=0.984, loss=8.419, backward_time=0.276, grad_norm=19.165, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.520
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:24:46,676 (trainer:732) INFO: 60epoch:train:205-255batch: iter_time=3.453e-04, forward_time=0.201, loss_att=7.874, acc=0.985, loss=7.874, backward_time=0.276, grad_norm=19.763, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:25:18,878 (trainer:732) INFO: 60epoch:train:256-306batch: iter_time=3.393e-04, forward_time=0.201, loss_att=7.789, acc=0.985, loss=7.789, backward_time=0.276, grad_norm=19.589, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=0.002, train_time=2.526
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:25:51,151 (trainer:732) INFO: 60epoch:train:307-357batch: iter_time=3.420e-04, forward_time=0.200, loss_att=8.331, acc=0.984, loss=8.331, backward_time=0.276, grad_norm=17.875, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.528
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:26:24,283 (trainer:732) INFO: 60epoch:train:358-408batch: iter_time=3.351e-04, forward_time=0.205, loss_att=8.986, acc=0.983, loss=8.986, backward_time=0.284, grad_norm=20.808, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.002, train_time=2.587
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:26:56,736 (trainer:732) INFO: 60epoch:train:409-459batch: iter_time=3.661e-04, forward_time=0.203, loss_att=8.442, acc=0.984, loss=8.442, backward_time=0.280, grad_norm=20.204, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.002, train_time=2.552
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:27:28,940 (trainer:732) INFO: 60epoch:train:460-510batch: iter_time=2.974e-04, forward_time=0.201, loss_att=8.471, acc=0.984, loss=8.471, backward_time=0.278, grad_norm=18.824, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=2.534
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:28:01,379 (trainer:732) INFO: 60epoch:train:511-561batch: iter_time=3.364e-04, forward_time=0.201, loss_att=8.250, acc=0.985, loss=8.250, backward_time=0.279, grad_norm=20.291, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.542
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:28:34,288 (trainer:732) INFO: 60epoch:train:562-612batch: iter_time=3.712e-04, forward_time=0.203, loss_att=8.477, acc=0.984, loss=8.477, backward_time=0.280, grad_norm=20.642, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=2.567
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:29:06,695 (trainer:732) INFO: 60epoch:train:613-663batch: iter_time=3.397e-04, forward_time=0.202, loss_att=8.501, acc=0.984, loss=8.501, backward_time=0.280, grad_norm=20.110, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.549
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:29:39,254 (trainer:732) INFO: 60epoch:train:664-714batch: iter_time=3.349e-04, forward_time=0.202, loss_att=9.071, acc=0.983, loss=9.071, backward_time=0.279, grad_norm=19.688, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=2.553
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:30:11,571 (trainer:732) INFO: 60epoch:train:715-765batch: iter_time=2.815e-04, forward_time=0.201, loss_att=8.464, acc=0.984, loss=8.464, backward_time=0.279, grad_norm=19.156, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.533
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:30:44,459 (trainer:732) INFO: 60epoch:train:766-816batch: iter_time=3.676e-04, forward_time=0.203, loss_att=8.576, acc=0.984, loss=8.576, backward_time=0.282, grad_norm=19.509, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.573
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:31:16,677 (trainer:732) INFO: 60epoch:train:817-867batch: iter_time=3.338e-04, forward_time=0.201, loss_att=9.053, acc=0.983, loss=9.053, backward_time=0.278, grad_norm=21.226, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.543
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:31:48,954 (trainer:732) INFO: 60epoch:train:868-918batch: iter_time=3.539e-04, forward_time=0.201, loss_att=8.684, acc=0.984, loss=8.684, backward_time=0.276, grad_norm=20.242, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=2.531
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:32:21,596 (trainer:732) INFO: 60epoch:train:919-969batch: iter_time=3.426e-04, forward_time=0.202, loss_att=8.606, acc=0.984, loss=8.606, backward_time=0.279, grad_norm=20.265, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.555
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:32:54,211 (trainer:732) INFO: 60epoch:train:970-1020batch: iter_time=3.287e-04, forward_time=0.202, loss_att=8.531, acc=0.984, loss=8.531, backward_time=0.278, grad_norm=20.157, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=2.545
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:33,885 (trainer:338) INFO: 60epoch results: [train] iter_time=8.790e-04, forward_time=0.202, loss_att=8.484, acc=0.984, loss=8.484, backward_time=0.279, grad_norm=19.823, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.126, time=13 minutes and 31.74 seconds, total_count=62220, gpu_max_cached_mem_GB=30.428, [valid] loss_att=70.705, acc=0.899, cer=0.129, wer=0.316, loss=70.705, time=4 minutes and 54.96 seconds, total_count=5280, gpu_max_cached_mem_GB=30.428, [att_plot] time=3 minutes and 32.27 seconds, total_count=0, gpu_max_cached_mem_GB=30.428
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:38,213 (trainer:386) INFO: The best model has been updated: valid.acc
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:38,228 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/49epoch.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:38,228 (trainer:458) INFO: The training was finished at 60 epochs 
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:38,294 (average_nbest_models:69) INFO: Averaging 10best models: criterion="valid.acc": exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr_only/valid.acc.ave_10best.pth
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,016 (average_nbest_models:96) INFO: Accumulating encoder.encoders.0.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,018 (average_nbest_models:96) INFO: Accumulating encoder.encoders.1.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,021 (average_nbest_models:96) INFO: Accumulating encoder.encoders.2.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,024 (average_nbest_models:96) INFO: Accumulating encoder.encoders.3.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,026 (average_nbest_models:96) INFO: Accumulating encoder.encoders.4.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,029 (average_nbest_models:96) INFO: Accumulating encoder.encoders.5.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,031 (average_nbest_models:96) INFO: Accumulating encoder.encoders.6.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,034 (average_nbest_models:96) INFO: Accumulating encoder.encoders.7.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,036 (average_nbest_models:96) INFO: Accumulating encoder.encoders.8.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,038 (average_nbest_models:96) INFO: Accumulating encoder.encoders.9.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,040 (average_nbest_models:96) INFO: Accumulating encoder.encoders.10.conv_module.norm.num_batches_tracked instead of averaging
[de-74279-k2-train-7-1218101249-5bcbfb5567-jsftr:0/2] 2024-02-21 15:41:49,043 (average_nbest_models:96) INFO: Accumulating encoder.encoders.11.conv_module.norm.num_batches_tracked instead of averaging
# Accounting: time=86044 threads=1
# Ended (code 0) at Wed Feb 21 15:41:53 CST 2024, elapsed time 86044 seconds