diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/36epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/36epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..936506fdf270a1b83f034411e115cdfd2c6c4f1a --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/36epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f4217df828df017fa016adc7deafb581f60786e5797d8ff74c9442245fdabad +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/37epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/37epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..590c1c0a645749f0b8be6d92200f1f56661b324a --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/37epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdfa48edb0f3e665835b6f180c134e9e01217e2e766e3fee9280cc444bb4878c +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/38epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/38epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..9e176ad65fde95dea5fab6f75c1a9cfeb00f76a6 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/38epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f827f2ea8142d01b3cd1ded431be08fd28bfa161a9d397d570110671ba946f5b +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/39epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/39epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..1e9d1ee638470804c699bc18e8fde576fe099b2e --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/39epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cdfc45a3ce59665376017b5b466a4a46a1f6831f39a50e2980223224c47070bf +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/40epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/40epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..ce03fcc3910415740d6472c2f633b5d00c29a232 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/40epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37e9df078558505ce825e097b66cd056b82bd8b0953f24d94bf05c3826f1a9a8 +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/41epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/41epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..ba9cd5ef88657dbb8df53aaf5e7098bb3e931c02 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/41epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1f659297813d1c971f32058b51ce2937d08d326cedbb8472cada849c123b2432 +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/42epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/42epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..40e90af01f8fa4d762182c9e31f2dfa4908d9552 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/42epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:350674327ad9832917df12de6fc98438484a80da168e5c5be95de56a39b7733e +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/43epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/43epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..4ded15492098163b52f9aa872592874821542162 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/43epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1d8f733e6ea4a25a9bae32e3d0b816b883f21f98d324e2d688c57ec1474e716 +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/44epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/44epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..dc46683e2b25bce04710d8d8a567293614b9b209 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/44epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41f6f2f1ebc97903480a52f418e71aec4e71861db9aca9bcaa67112687533d68 +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/45epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/45epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..261caa86ba8036b31be7b22c4cb58f0bd435f10a --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/45epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27ef31ab1e2e5974a590d9f464fc233cb8c99c2e55bf7708f403dd84b2add7b +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/RESULTS.md b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/RESULTS.md new file mode 100644 index 0000000000000000000000000000000000000000..fd8cc2ad94a711f6e89d9f89e2da6b87550f3fe5 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/RESULTS.md @@ -0,0 +1,107 @@ + +# RESULTS +## Environments +- date: `Mon Mar 4 23:35:01 CST 2024` +- python version: `3.9.18 (main, Sep 11 2023, 13:41:44) [GCC 11.2.0]` +- espnet version: `espnet 202308` +- pytorch version: `pytorch 1.12.1+cu116` +- Git hash: `884659f9ee95374811015381c976fa3b4f6e01db` + - Commit date: `Thu Nov 23 00:23:29 2023 +0800` + +## exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new +### WER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +|decode_sot_asr_model_25epoch/test_clean_kaldi_fmt|961|64007|44.2|47.9|7.9|17.8|73.5|99.6| +|decode_sot_asr_model_25epoch/test_other_kaldi_fmt|992|80370|40.2|50.5|9.3|17.3|77.2|100.0| +|decode_sot_asr_model_valid.acc.best/dev_2spk|3315|226216|48.8|35.5|15.7|13.1|64.3|100.0| +|decode_sot_asr_model_valid.acc.best/dev_2spk_kaldi_fmt_mtt|2282|142104|45.8|35.1|19.1|13.1|67.3|100.0| +|decode_sot_asr_model_valid.acc.best/dev_3spk|2059|209679|34.6|45.3|20.2|9.6|75.1|100.0| +|decode_sot_asr_model_valid.acc.best/dev_3spk_kaldi_fmt|1004|124462|37.8|43.5|18.7|9.9|72.1|100.0| +|decode_sot_asr_model_valid.acc.best/dev_4spk|1467|200029|28.3|46.5|25.2|6.4|78.0|100.0| +|decode_sot_asr_model_valid.acc.best/dev_4spk_kaldi_fmt|721|119166|31.3|45.7|23.0|9.4|78.1|100.0| +|decode_sot_asr_model_valid.acc.best/sot_sdm1_dev|2382|35243|26.8|64.8|8.4|81.8|155.0|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_2spk|4570|301042|47.8|36.4|15.8|13.7|65.9|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_3spk|2072|212871|33.0|47.3|19.8|10.4|77.4|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_4spk|1326|185394|27.7|47.0|25.3|6.9|79.1|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_2spk|4663|336490|45.8|38.8|15.4|13.0|67.2|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_3spk|2453|266074|30.2|50.2|19.6|9.5|79.2|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_4spk|1795|259138|25.1|49.3|25.6|5.9|80.8|100.0| +|decode_sot_asr_model_valid.acc.best/test_clean_2spk_kaldi_fmt|2180|178761|49.8|34.6|15.6|13.0|63.2|100.0| +|decode_sot_asr_model_valid.acc.best/test_clean_3spk_kaldi_fmt|977|124741|34.3|47.1|18.5|12.0|77.7|100.0| +|decode_sot_asr_model_valid.acc.best/test_clean_4spk_kaldi_fmt|632|109072|30.3|46.8|22.9|9.8|79.5|100.0| +|decode_sot_asr_model_valid.acc.best/test_other_2spk_kaldi_fmt|2363|205496|47.8|37.3|14.9|11.8|63.9|100.0| +|decode_sot_asr_model_valid.acc.best/test_other_3spk_kaldi_fmt|1246|162996|31.6|50.6|17.8|11.8|80.2|100.0| +|decode_sot_asr_model_valid.acc.best/test_other_4spk_kaldi_fmt|901|157123|27.6|49.4|23.0|9.3|81.8|100.0| + +### CER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +|decode_sot_asr_model_25epoch/test_clean_kaldi_fmt|961|329390|67.6|20.4|12.0|18.5|50.9|99.6| +|decode_sot_asr_model_25epoch/test_other_kaldi_fmt|992|416899|64.3|21.7|14.0|18.2|53.8|100.0| +|decode_sot_asr_model_valid.acc.best/dev_2spk|3315|1230801|66.7|16.6|16.7|11.9|45.2|100.0| +|decode_sot_asr_model_valid.acc.best/dev_2spk_kaldi_fmt_mtt|2282|770287|63.3|17.0|19.7|12.2|48.9|100.0| +|decode_sot_asr_model_valid.acc.best/dev_3spk|2059|1140428|55.3|20.3|24.4|10.3|54.9|100.0| +|decode_sot_asr_model_valid.acc.best/dev_3spk_kaldi_fmt|1004|677017|58.2|19.1|22.7|10.3|52.1|100.0| +|decode_sot_asr_model_valid.acc.best/dev_4spk|1467|1087409|49.2|20.3|30.5|7.8|58.6|100.0| +|decode_sot_asr_model_valid.acc.best/dev_4spk_kaldi_fmt|721|647884|52.2|19.5|28.3|9.9|57.7|100.0| +|decode_sot_asr_model_valid.acc.best/sot_sdm1_dev|2382|169857|60.5|29.1|10.4|82.0|121.5|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_2spk|4570|1550429|66.5|17.2|16.3|12.9|46.4|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_3spk|2072|1084475|54.9|22.0|23.1|11.9|56.9|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_4spk|1326|938467|49.6|21.4|29.1|8.8|59.2|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_2spk|4663|1742136|65.5|17.9|16.5|12.4|46.9|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_3spk|2453|1381987|53.3|22.5|24.2|11.1|57.8|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_4spk|1795|1346646|47.7|21.5|30.8|7.9|60.2|100.0| +|decode_sot_asr_model_valid.acc.best/test_clean_2spk_kaldi_fmt|2180|921344|67.7|16.3|16.0|11.8|44.1|100.0| +|decode_sot_asr_model_valid.acc.best/test_clean_3spk_kaldi_fmt|977|635802|56.4|21.4|22.2|13.2|56.8|100.0| +|decode_sot_asr_model_valid.acc.best/test_clean_4spk_kaldi_fmt|632|552325|52.4|20.9|26.7|11.6|59.1|100.0| +|decode_sot_asr_model_valid.acc.best/test_other_2spk_kaldi_fmt|2363|1064868|67.0|16.9|16.1|11.0|44.0|100.0| +|decode_sot_asr_model_valid.acc.best/test_other_3spk_kaldi_fmt|1246|847159|55.0|22.6|22.4|13.0|58.0|100.0| +|decode_sot_asr_model_valid.acc.best/test_other_4spk_kaldi_fmt|901|817228|50.5|21.4|28.1|10.4|59.9|100.0| + +### TER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +## exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/decode_sot_asr_model_25epoch +### WER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +|org/dev_kaldi_fmt|605|47659|45.3|44.2|10.5|15.3|70.0|99.5| + +### CER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +|org/dev_kaldi_fmt|605|258151|66.1|19.0|14.9|15.2|49.0|99.5| + +### TER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +## exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/decode_sot_asr_model_valid.acc.best +### WER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +|org/dev_2spk_kaldi_fmt|1606|135101|50.7|33.4|15.9|12.0|61.3|100.0| +|org/sot_sdm1_dev|4142|96496|25.8|65.4|8.8|98.2|172.4|100.0| +|org/sot_sdm1_eval|2385|37529|22.6|70.1|7.3|78.7|156.1|100.0| +|org/tt_mix_clean_reverb_max_16k|3000|3000|0.0|100.0|0.0|3936.7|4036.7|100.0| + +### CER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +|org/dev_2spk_kaldi_fmt|1606|735694|67.8|15.4|16.8|10.4|42.6|100.0| +|org/sot_sdm1_dev|4142|469537|61.1|28.8|10.2|100.6|139.5|100.0| +|org/sot_sdm1_eval|2385|183036|57.6|32.1|10.3|76.3|118.7|100.0| +|org/tt_mix_clean_reverb_max_16k|3000|143026|16.9|83.0|0.1|376.6|459.6|100.0| + +### TER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/checkpoint.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/checkpoint.pth new file mode 100644 index 0000000000000000000000000000000000000000..0a23d04b7574ecc45a8c55794f2c5ca9c3af37e0 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/checkpoint.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:97b9144b902ec389690dc5de336cd2f7e72606f66a6fabd8d0be4f5bef3d32f2 +size 516804611 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/config.yaml b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2ddda592d0d5ecdf940da06659789ed4f223c975 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/config.yaml @@ -0,0 +1,227 @@ +config: conf/tuning/train_sot_asr_conformer.yaml +print_config: false +log_level: INFO +drop_last_iter: false +dry_run: false +iterator_type: sequence +valid_iterator_type: null +output_dir: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new +ngpu: 1 +seed: 0 +num_workers: 16 +num_att_plot: 3 +dist_backend: nccl +dist_init_method: env:// +dist_world_size: 2 +dist_rank: 0 +local_rank: 0 +dist_master_addr: localhost +dist_master_port: 43895 +dist_launcher: null +multiprocessing_distributed: true +unused_parameters: false +sharded_ddp: false +cudnn_enabled: true +cudnn_benchmark: false +cudnn_deterministic: true +collect_stats: false +write_collected_feats: false +max_epoch: 60 +patience: null +val_scheduler_criterion: +- valid +- loss +early_stopping_criterion: +- valid +- loss +- min +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 +nbest_averaging_interval: 0 +grad_clip: 5.0 +grad_clip_type: 2.0 +grad_noise: false +accum_grad: 4 +no_forward_run: false +resume: true +train_dtype: float32 +use_amp: false +log_interval: null +use_matplotlib: true +use_tensorboard: true +create_graph_in_tensorboard: false +use_wandb: false +wandb_project: null +wandb_id: null +wandb_entity: null +wandb_name: null +wandb_model_log_interval: -1 +detect_anomaly: false +pretrain_path: null +init_param: +- /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth +ignore_init_mismatch: false +freeze_param: [] +num_iters_per_epoch: null +batch_size: 20 +valid_batch_size: null +batch_bins: 10000000 +valid_batch_bins: null +train_shape_file: +- exp/asr_stats_raw_en_char_sp/train/speech_shape +- exp/asr_stats_raw_en_char_sp/train/text_shape.char +valid_shape_file: +- exp/asr_stats_raw_en_char_sp/valid/speech_shape +- exp/asr_stats_raw_en_char_sp/valid/text_shape.char +batch_type: numel +valid_batch_type: null +fold_length: +- 80000 +- 150 +sort_in_batch: descending +shuffle_within_batch: false +sort_batch: descending +multiple_iterator: false +chunk_length: 500 +chunk_shift_ratio: 0.5 +num_cache_chunks: 1024 +chunk_excluded_key_prefixes: [] +train_data_path_and_name_and_type: +- - dump/raw/train_small_kaldi_fmt_sp/wav.scp + - speech + - kaldi_ark +- - dump/raw/train_small_kaldi_fmt_sp/text + - text + - text +valid_data_path_and_name_and_type: +- - dump/raw/dev_kaldi_fmt/wav.scp + - speech + - kaldi_ark +- - dump/raw/dev_kaldi_fmt/text + - text + - text +allow_variable_data_keys: false +max_cache_size: 0.0 +max_cache_fd: 32 +valid_max_cache_size: null +exclude_weight_decay: false +exclude_weight_decay_conf: {} +optim: adam +optim_conf: + lr: 0.002 + weight_decay: 1.0e-06 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 20000 +token_list: +- +- +- +- +- E +- T +- A +- O +- N +- I +- H +- S +- R +- D +- L +- U +- M +- C +- W +- F +- G +- Y +- P +- B +- V +- K +- '''' +- X +- J +- Q +- Z +- +init: null +input_size: null +ctc_conf: + dropout_rate: 0.0 + ctc_type: builtin + reduce: true + ignore_nan_grad: null + zero_infinity: true +joint_net_conf: null +use_preprocessor: true +token_type: char +bpemodel: null +non_linguistic_symbols: null +cleaner: null +g2p: null +speech_volume_normalize: null +rir_scp: null +rir_apply_prob: 1.0 +noise_scp: null +noise_apply_prob: 1.0 +noise_db_range: '13_15' +short_noise_thres: 0.5 +aux_ctc_tasks: [] +frontend: default +frontend_conf: + fs: 16k +specaug: null +specaug_conf: {} +normalize: global_mvn +normalize_conf: + stats_file: exp/asr_stats_raw_en_char_sp/train/feats_stats.npz +model: espnet +model_conf: + ctc_weight: 0.0 + lsm_weight: 0.1 + length_normalized_loss: false +preencoder: null +preencoder_conf: {} +encoder: conformer +encoder_conf: + output_size: 256 + attention_heads: 4 + linear_units: 2048 + num_blocks: 12 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.1 + input_layer: conv2d + normalize_before: true + macaron_style: true + rel_pos_type: latest + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + use_cnn_module: true + cnn_module_kernel: 31 +postencoder: null +postencoder_conf: {} +decoder: transformer +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.1 + src_attention_dropout_rate: 0.1 +preprocessor: multi +preprocessor_conf: + speaker_change_symbol: + - +required: +- output_dir +- token_list +version: '202308' +distributed: true diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/acc.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/acc.png new file mode 100644 index 0000000000000000000000000000000000000000..e0736de4c6c7e1e4f4a8ee75fe593a9d21182c3b Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/acc.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/backward_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/backward_time.png new file mode 100644 index 0000000000000000000000000000000000000000..be6488e5aeffc168b390b3d86f250b91703d2d9d Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/backward_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/cer.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/cer.png new file mode 100644 index 0000000000000000000000000000000000000000..eb6b388c641694abd9c4a743bf5c19f5d0b8e4ea Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/cer.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/clip.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/clip.png new file mode 100644 index 0000000000000000000000000000000000000000..2874e85db8f1ab638f09b63bfd0d1184d11e67c1 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/clip.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/forward_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/forward_time.png new file mode 100644 index 0000000000000000000000000000000000000000..ea1a2c33488eb7f47c6c1e5d59f29b37a8a9e62f Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/forward_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/gpu_max_cached_mem_GB.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/gpu_max_cached_mem_GB.png new file mode 100644 index 0000000000000000000000000000000000000000..2590dbc3654bf8fadebacd03a18876120e56bf54 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/gpu_max_cached_mem_GB.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/grad_norm.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/grad_norm.png new file mode 100644 index 0000000000000000000000000000000000000000..bf9c31d33ae9ea87dd5739c12218ca6bf28c798d Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/grad_norm.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/iter_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/iter_time.png new file mode 100644 index 0000000000000000000000000000000000000000..7f2282ded4e28a1c50185d3254afc0f15b3cdcd5 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/iter_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/loss.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/loss.png new file mode 100644 index 0000000000000000000000000000000000000000..cda5b744c624d2ca1f870805c778b5948c2315c4 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/loss.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/loss_att.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/loss_att.png new file mode 100644 index 0000000000000000000000000000000000000000..6d614d521add989c8f255b2600e3970d0468827a Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/loss_att.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/loss_scale.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/loss_scale.png new file mode 100644 index 0000000000000000000000000000000000000000..c214f70fb0fb9b283d568fe66734f4cfcff2c25a Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/loss_scale.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/optim0_lr0.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/optim0_lr0.png new file mode 100644 index 0000000000000000000000000000000000000000..76db4b9bd9df70603e40ac0a00bea5601bed6bf5 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/optim0_lr0.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/optim_step_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/optim_step_time.png new file mode 100644 index 0000000000000000000000000000000000000000..8b8ef53a6c806b564b66bb6695c7001f8f5b7337 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/optim_step_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/train_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/train_time.png new file mode 100644 index 0000000000000000000000000000000000000000..2b4a1e0efadd8558506421a55a449cb5616bce71 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/train_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/wer.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/wer.png new file mode 100644 index 0000000000000000000000000000000000000000..2e7f897b8546f19b6af256da1ef486a580d0c489 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/images/wer.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/latest.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/latest.pth new file mode 100644 index 0000000000000000000000000000000000000000..261caa86ba8036b31be7b22c4cb58f0bd435f10a --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/latest.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27ef31ab1e2e5974a590d9f464fc233cb8c99c2e55bf7708f403dd84b2add7b +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/run.sh b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..e1f723a777834fa5b83ce86043411e48959776a0 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/run.sh @@ -0,0 +1 @@ +./asr.sh --lang en --audio_format flac.ark --stage 11 --feats_type raw --token_type char --sot_asr true --max_wav_duration 50 --speed_perturb_factors '0.9 1.0 1.1' --feats_normalize global_mvn --use_lm false --pretrained_model /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth --asr_config conf/tuning/train_sot_asr_conformer.yaml --lm_config conf/tuning/train_lm_transformer.yaml --inference_config conf/tuning/decode_sot.yaml --train_set train_small_kaldi_fmt --valid_set dev_kaldi_fmt --test_sets 'dev_kaldi_fmt test_clean_kaldi_fmt test_other_kaldi_fmt' --ngpu 2 --asr_tag train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new --lm_train_text data/local/other_text/text --bpe_train_text data/train_small_kaldi_fmt/text --stage 11 "$@"; exit $? diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/tensorboard/train/events.out.tfevents.1700139527.de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb.2101480.0 b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/tensorboard/train/events.out.tfevents.1700139527.de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb.2101480.0 new file mode 100644 index 0000000000000000000000000000000000000000..8874c1a2ffbc0079735120a5aa3858e85df2b630 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/tensorboard/train/events.out.tfevents.1700139527.de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb.2101480.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0377748d19696a7f0633ffa2127d6092745fac79a961243e1f853c951e6ba8b2 +size 625891718 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/tensorboard/valid/events.out.tfevents.1700139527.de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb.2101480.1 b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/tensorboard/valid/events.out.tfevents.1700139527.de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb.2101480.1 new file mode 100644 index 0000000000000000000000000000000000000000..e734674ae0c3ce23ef2d9637a837c85887bc89af --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/tensorboard/valid/events.out.tfevents.1700139527.de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb.2101480.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d54a186f38796fe2f11fb9d18374dd5c6dcbd2f0af0363dd45c6720138bb2cd0 +size 12730 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/train.log b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/train.log new file mode 100644 index 0000000000000000000000000000000000000000..b6c1cd256e135f7227d0782a86c6ce9160b702f7 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/train.log @@ -0,0 +1,2226 @@ +# python3 -m espnet2.bin.asr_train --use_preprocessor true --bpemodel none --token_type char --token_list data/en_token_list/char/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_kaldi_fmt/wav.scp,speech,kaldi_ark --valid_shape_file exp/asr_stats_raw_en_char_sp/valid/speech_shape --resume true --init_param /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth --ignore_init_mismatch false --fold_length 80000 --output_dir exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new --config conf/tuning/train_sot_asr_conformer.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/asr_stats_raw_en_char_sp/train/feats_stats.npz --train_data_path_and_name_and_type dump/raw/train_small_kaldi_fmt_sp/wav.scp,speech,kaldi_ark --train_shape_file exp/asr_stats_raw_en_char_sp/train/speech_shape --fold_length 150 --train_data_path_and_name_and_type dump/raw/train_small_kaldi_fmt_sp/text,text,text --train_shape_file exp/asr_stats_raw_en_char_sp/train/text_shape.char --valid_data_path_and_name_and_type dump/raw/dev_kaldi_fmt/text,text,text --valid_shape_file exp/asr_stats_raw_en_char_sp/valid/text_shape.char --ngpu 2 --multiprocessing_distributed True +# Started at Thu Nov 16 20:58:07 CST 2023 +# +/star-home/jinzengrui/lib/miniconda3/envs/dev39/bin/python3 /star-home/jinzengrui/lib/miniconda3/envs/dev39/lib/python3.9/site-packages/espnet-202308-py3.9.egg/espnet2/bin/asr_train.py --use_preprocessor true --bpemodel none --token_type char --token_list data/en_token_list/char/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/dev_kaldi_fmt/wav.scp,speech,kaldi_ark --valid_shape_file exp/asr_stats_raw_en_char_sp/valid/speech_shape --resume true --init_param /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth --ignore_init_mismatch false --fold_length 80000 --output_dir exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new --config conf/tuning/train_sot_asr_conformer.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/asr_stats_raw_en_char_sp/train/feats_stats.npz --train_data_path_and_name_and_type dump/raw/train_small_kaldi_fmt_sp/wav.scp,speech,kaldi_ark --train_shape_file exp/asr_stats_raw_en_char_sp/train/speech_shape --fold_length 150 --train_data_path_and_name_and_type dump/raw/train_small_kaldi_fmt_sp/text,text,text --train_shape_file exp/asr_stats_raw_en_char_sp/train/text_shape.char --valid_data_path_and_name_and_type dump/raw/dev_kaldi_fmt/text,text,text --valid_shape_file exp/asr_stats_raw_en_char_sp/valid/text_shape.char --ngpu 2 --multiprocessing_distributed True +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:33,408 (distributed_c10d:228) INFO: Added key: store_based_barrier_key:1 to store for rank: 0 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:33,408 (distributed_c10d:262) INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 2 nodes. +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:33,450 (asr:490) INFO: Vocabulary size: 32 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:41,726 (abs_task:1229) INFO: pytorch.version=1.11.0+cu102, cuda.available=True, cudnn.version=7605, cudnn.benchmark=False, cudnn.deterministic=True +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:41,742 (abs_task:1230) INFO: Model structure: +ESPnetASRModel( + (frontend): DefaultFrontend( + (stft): Stft(n_fft=512, win_length=512, hop_length=128, center=True, normalized=False, onesided=True) + (frontend): Frontend() + (logmel): LogMel(sr=16000, n_fft=512, n_mels=80, fmin=0, fmax=8000.0, htk=False) + ) + (normalize): GlobalMVN(stats_file=exp/asr_stats_raw_en_char_sp/train/feats_stats.npz, norm_means=True, norm_vars=True) + (encoder): ConformerEncoder( + (embed): Conv2dSubsampling( + (conv): Sequential( + (0): Conv2d(1, 256, kernel_size=(3, 3), stride=(2, 2)) + (1): ReLU() + (2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2)) + (3): ReLU() + ) + (out): Sequential( + (0): Linear(in_features=4864, out_features=256, bias=True) + (1): RelPositionalEncoding( + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + ) + (encoders): MultiSequential( + (0): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (1): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (2): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (3): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (4): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (5): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (6): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (7): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (8): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (9): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (10): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (11): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (after_norm): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + ) + (decoder): TransformerDecoder( + (embed): Sequential( + (0): Embedding(32, 256) + (1): PositionalEncoding( + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (after_norm): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (output_layer): Linear(in_features=256, out_features=32, bias=True) + (decoders): MultiSequential( + (0): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (1): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (2): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (3): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (4): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (5): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + ) + (criterion_att): LabelSmoothingLoss( + (criterion): KLDivLoss() + ) +) + +Model summary: + Class Name: ESPnetASRModel + Total Number of model parameters: 43.00 M + Number of trainable parameters: 43.00 M (100.0%) + Size: 172.01 MB + Type: torch.float32 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:41,743 (abs_task:1233) INFO: Optimizer: +Adam ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.999) + eps: 1e-08 + initial_lr: 0.002 + lr: 1e-07 + maximize: False + weight_decay: 1e-06 +) +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:41,743 (abs_task:1234) INFO: Scheduler: WarmupLR(warmup_steps=20000) +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:41,745 (abs_task:1243) INFO: Saving the configuration in exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/config.yaml +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:41,771 (abs_task:1304) INFO: Loading pretrained params from /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:43,916 (asr:461) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4') +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,094 (abs_task:1614) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/train_small_kaldi_fmt_sp/wav.scp", "type": "kaldi_ark"} + text: {"path": "dump/raw/train_small_kaldi_fmt_sp/text", "type": "text"} + preprocess: ) +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,094 (abs_task:1615) INFO: [train] Batch sampler: NumElementsBatchSampler(N-batch=1843, batch_bins=10000000, sort_in_batch=descending, sort_batch=descending) +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,095 (abs_task:1616) INFO: [train] mini-batch sizes summary: N-batch=1843, mean=34.3, min=8, max=101 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,170 (asr:461) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4') +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,213 (abs_task:1614) INFO: [valid] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/dev_kaldi_fmt/wav.scp", "type": "kaldi_ark"} + text: {"path": "dump/raw/dev_kaldi_fmt/text", "type": "text"} + preprocess: ) +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,213 (abs_task:1615) INFO: [valid] Batch sampler: NumElementsBatchSampler(N-batch=19, batch_bins=10000000, sort_in_batch=descending, sort_batch=descending) +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,213 (abs_task:1616) INFO: [valid] mini-batch sizes summary: N-batch=19, mean=31.8, min=3, max=57 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,219 (asr:461) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4') +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,246 (abs_task:1614) INFO: [plot_att] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/dev_kaldi_fmt/wav.scp", "type": "kaldi_ark"} + text: {"path": "dump/raw/dev_kaldi_fmt/text", "type": "text"} + preprocess: ) +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,247 (abs_task:1615) INFO: [plot_att] Batch sampler: UnsortedBatchSampler(N-batch=605, batch_size=1, key_file=exp/asr_stats_raw_en_char_sp/valid/speech_shape, +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:45,247 (abs_task:1616) INFO: [plot_att] mini-batch sizes summary: N-batch=3, mean=1.0, min=1, max=1 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2101480 [0] NCCL INFO Bootstrap : Using eth0:10.177.13.150<0> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2101480 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2101480 [0] NCCL INFO NET/IB : No device found. +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2101480 [0] NCCL INFO NET/Socket : Using [0]eth0:10.177.13.150<0> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2101480 [0] NCCL INFO Using network Socket +NCCL version 2.10.3+cuda10.2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2101481 [1] NCCL INFO Bootstrap : Using eth0:10.177.13.150<0> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2101481 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2101481 [1] NCCL INFO NET/IB : No device found. +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2101481 [1] NCCL INFO NET/Socket : Using [0]eth0:10.177.13.150<0> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2101481 [1] NCCL INFO Using network Socket +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO Channel 00/02 : 0 1 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103964 [1] NCCL INFO Trees [0] -1/-1/-1->1->0 [1] -1/-1/-1->1->0 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO Channel 01/02 : 0 1 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103964 [1] NCCL INFO Setting affinity for GPU 4 to 20000002 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO Setting affinity for GPU 3 to 20000002 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103964 [1] NCCL INFO Channel 00 : 1[c000] -> 0[8000] via direct shared memory +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103964 [1] NCCL INFO Channel 01 : 1[c000] -> 0[8000] via direct shared memory +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO Channel 00 : 0[8000] -> 1[c000] via direct shared memory +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO Channel 01 : 0[8000] -> 1[c000] via direct shared memory +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO Connected all rings +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO Connected all trees +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103964 [1] NCCL INFO Connected all rings +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103964 [1] NCCL INFO Connected all trees +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 8/8/512 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103964 [1] NCCL INFO threadThresholds 8/8/64 | 16/8/64 | 8/8/512 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103964 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103964 [1] NCCL INFO comm 0x7f172c001200 rank 1 nranks 2 cudaDev 1 busId c000 - Init COMPLETE +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103963 [0] NCCL INFO comm 0x7f7228001200 rank 0 nranks 2 cudaDev 0 busId 8000 - Init COMPLETE +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2101480 [0] NCCL INFO Launch mode Parallel +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 20:58:47,351 (trainer:284) INFO: 1/60epoch started +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:03:31,520 (distributed:948) INFO: Reducer buckets have been rebuilt in this iteration. +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:04:49,494 (trainer:732) INFO: 1epoch:train:1-92batch: iter_time=0.024, forward_time=0.295, loss_att=941.878, acc=0.399, loss=941.878, backward_time=0.391, grad_norm=892.265, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=1.300e-06, train_time=15.737 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:06:08,458 (trainer:732) INFO: 1epoch:train:93-184batch: iter_time=3.536e-04, forward_time=0.278, loss_att=920.987, acc=0.408, loss=920.987, backward_time=0.390, grad_norm=806.386, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=3.600e-06, train_time=3.437 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:07:26,953 (trainer:732) INFO: 1epoch:train:185-276batch: iter_time=3.527e-04, forward_time=0.279, loss_att=858.405, acc=0.431, loss=858.405, backward_time=0.389, grad_norm=688.628, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=5.900e-06, train_time=3.412 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:08:44,420 (trainer:732) INFO: 1epoch:train:277-368batch: iter_time=3.652e-04, forward_time=0.271, loss_att=767.506, acc=0.450, loss=767.506, backward_time=0.387, grad_norm=480.643, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=8.200e-06, train_time=3.367 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:10:03,699 (trainer:732) INFO: 1epoch:train:369-460batch: iter_time=3.493e-04, forward_time=0.284, loss_att=703.007, acc=0.461, loss=703.007, backward_time=0.389, grad_norm=279.519, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=1.050e-05, train_time=3.446 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:11:20,893 (trainer:732) INFO: 1epoch:train:461-552batch: iter_time=3.878e-04, forward_time=0.269, loss_att=608.731, acc=0.485, loss=608.731, backward_time=0.384, grad_norm=161.478, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=1.280e-05, train_time=3.356 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:12:39,072 (trainer:732) INFO: 1epoch:train:553-644batch: iter_time=3.566e-04, forward_time=0.280, loss_att=601.998, acc=0.508, loss=601.998, backward_time=0.388, grad_norm=148.428, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=1.510e-05, train_time=3.398 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:13:57,467 (trainer:732) INFO: 1epoch:train:645-736batch: iter_time=3.592e-04, forward_time=0.276, loss_att=595.854, acc=0.522, loss=595.854, backward_time=0.390, grad_norm=152.748, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=1.740e-05, train_time=3.408 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:15:15,477 (trainer:732) INFO: 1epoch:train:737-828batch: iter_time=3.597e-04, forward_time=0.276, loss_att=581.342, acc=0.529, loss=581.342, backward_time=0.388, grad_norm=192.037, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=1.970e-05, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:16:32,806 (trainer:732) INFO: 1epoch:train:829-920batch: iter_time=3.593e-04, forward_time=0.269, loss_att=547.701, acc=0.539, loss=547.701, backward_time=0.385, grad_norm=148.012, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=2.200e-05, train_time=3.361 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:17:51,029 (trainer:732) INFO: 1epoch:train:921-1012batch: iter_time=3.435e-04, forward_time=0.272, loss_att=549.148, acc=0.547, loss=549.148, backward_time=0.390, grad_norm=142.484, clip=100.000, loss_scale=1.000, optim_step_time=0.097, optim0_lr0=2.430e-05, train_time=3.400 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:19:08,771 (trainer:732) INFO: 1epoch:train:1013-1104batch: iter_time=3.349e-04, forward_time=0.268, loss_att=532.527, acc=0.554, loss=532.527, backward_time=0.389, grad_norm=128.053, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=2.660e-05, train_time=3.379 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:20:27,262 (trainer:732) INFO: 1epoch:train:1105-1196batch: iter_time=3.116e-04, forward_time=0.271, loss_att=550.607, acc=0.554, loss=550.607, backward_time=0.394, grad_norm=157.146, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=2.890e-05, train_time=3.412 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:21:45,583 (trainer:732) INFO: 1epoch:train:1197-1288batch: iter_time=3.344e-04, forward_time=0.267, loss_att=536.024, acc=0.564, loss=536.024, backward_time=0.391, grad_norm=128.063, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=3.120e-05, train_time=3.405 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:23:03,152 (trainer:732) INFO: 1epoch:train:1289-1380batch: iter_time=3.327e-04, forward_time=0.268, loss_att=523.637, acc=0.571, loss=523.637, backward_time=0.389, grad_norm=110.526, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=3.350e-05, train_time=3.372 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:24:19,941 (trainer:732) INFO: 1epoch:train:1381-1472batch: iter_time=3.320e-04, forward_time=0.264, loss_att=490.963, acc=0.573, loss=490.963, backward_time=0.387, grad_norm=112.746, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=3.580e-05, train_time=3.338 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:25:36,907 (trainer:732) INFO: 1epoch:train:1473-1564batch: iter_time=3.518e-04, forward_time=0.265, loss_att=482.758, acc=0.578, loss=482.758, backward_time=0.385, grad_norm=106.980, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=3.810e-05, train_time=3.346 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:26:54,418 (trainer:732) INFO: 1epoch:train:1565-1656batch: iter_time=3.567e-04, forward_time=0.271, loss_att=479.227, acc=0.586, loss=479.227, backward_time=0.386, grad_norm=99.923, clip=100.000, loss_scale=1.000, optim_step_time=0.097, optim0_lr0=4.040e-05, train_time=3.369 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:28:11,783 (trainer:732) INFO: 1epoch:train:1657-1748batch: iter_time=3.450e-04, forward_time=0.268, loss_att=486.275, acc=0.581, loss=486.275, backward_time=0.387, grad_norm=111.078, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=4.270e-05, train_time=3.363 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:29:29,464 (trainer:732) INFO: 1epoch:train:1749-1840batch: iter_time=3.259e-04, forward_time=0.267, loss_att=489.602, acc=0.589, loss=489.602, backward_time=0.388, grad_norm=108.016, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=4.500e-05, train_time=3.377 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:38:22,884 (trainer:338) INFO: 1epoch results: [train] iter_time=0.002, forward_time=0.273, loss_att=611.900, acc=0.522, loss=611.900, backward_time=0.388, grad_norm=257.758, clip=100.000, loss_scale=1.000, optim_step_time=0.095, optim0_lr0=2.315e-05, train_time=4.004, time=30 minutes and 47.51 seconds, total_count=1843, gpu_max_cached_mem_GB=29.846, [valid] loss_att=437.428, acc=0.623, cer=0.448, wer=0.779, loss=437.428, time=4 minutes and 11.67 seconds, total_count=19, gpu_max_cached_mem_GB=29.846, [att_plot] time=4 minutes and 36.32 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:38:29,224 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:38:29,226 (trainer:272) INFO: 2/60epoch started. Estimated time to finish: 1 day, 15 hours and 2 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:42:08,492 (trainer:732) INFO: 2epoch:train:1-92batch: iter_time=0.009, forward_time=0.262, loss_att=491.984, acc=0.593, loss=491.984, backward_time=0.390, grad_norm=119.174, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=4.730e-05, train_time=9.532 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:43:25,165 (trainer:732) INFO: 2epoch:train:93-184batch: iter_time=4.121e-04, forward_time=0.258, loss_att=473.501, acc=0.598, loss=473.501, backward_time=0.387, grad_norm=102.135, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=4.960e-05, train_time=3.333 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:44:41,610 (trainer:732) INFO: 2epoch:train:185-276batch: iter_time=3.851e-04, forward_time=0.260, loss_att=473.923, acc=0.599, loss=473.923, backward_time=0.390, grad_norm=100.188, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=5.190e-05, train_time=3.323 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:45:57,027 (trainer:732) INFO: 2epoch:train:277-368batch: iter_time=3.534e-04, forward_time=0.256, loss_att=451.859, acc=0.599, loss=451.859, backward_time=0.383, grad_norm=98.697, clip=100.000, loss_scale=1.000, optim_step_time=0.087, optim0_lr0=5.420e-05, train_time=3.278 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:47:13,616 (trainer:732) INFO: 2epoch:train:369-460batch: iter_time=3.553e-04, forward_time=0.260, loss_att=473.091, acc=0.601, loss=473.091, backward_time=0.390, grad_norm=88.355, clip=100.000, loss_scale=1.000, optim_step_time=0.088, optim0_lr0=5.650e-05, train_time=3.329 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:48:30,051 (trainer:732) INFO: 2epoch:train:461-552batch: iter_time=3.627e-04, forward_time=0.260, loss_att=454.789, acc=0.604, loss=454.789, backward_time=0.388, grad_norm=101.950, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=5.880e-05, train_time=3.323 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:49:46,804 (trainer:732) INFO: 2epoch:train:553-644batch: iter_time=3.638e-04, forward_time=0.261, loss_att=460.266, acc=0.608, loss=460.266, backward_time=0.389, grad_norm=89.822, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=6.110e-05, train_time=3.336 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:51:03,082 (trainer:732) INFO: 2epoch:train:645-736batch: iter_time=4.236e-04, forward_time=0.259, loss_att=449.938, acc=0.607, loss=449.938, backward_time=0.385, grad_norm=77.012, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=6.340e-05, train_time=3.316 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:52:19,855 (trainer:732) INFO: 2epoch:train:737-828batch: iter_time=4.224e-04, forward_time=0.260, loss_att=468.194, acc=0.611, loss=468.194, backward_time=0.388, grad_norm=89.587, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=6.570e-05, train_time=3.337 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:53:35,873 (trainer:732) INFO: 2epoch:train:829-920batch: iter_time=3.840e-04, forward_time=0.258, loss_att=452.410, acc=0.617, loss=452.410, backward_time=0.385, grad_norm=107.355, clip=100.000, loss_scale=1.000, optim_step_time=0.095, optim0_lr0=6.800e-05, train_time=3.305 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:54:52,182 (trainer:732) INFO: 2epoch:train:921-1012batch: iter_time=4.139e-04, forward_time=0.260, loss_att=446.564, acc=0.616, loss=446.564, backward_time=0.387, grad_norm=92.849, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=7.030e-05, train_time=3.317 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:56:08,267 (trainer:732) INFO: 2epoch:train:1013-1104batch: iter_time=3.810e-04, forward_time=0.259, loss_att=439.785, acc=0.615, loss=439.785, backward_time=0.386, grad_norm=76.190, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=7.260e-05, train_time=3.307 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:57:24,890 (trainer:732) INFO: 2epoch:train:1105-1196batch: iter_time=3.380e-04, forward_time=0.260, loss_att=447.287, acc=0.617, loss=447.287, backward_time=0.387, grad_norm=90.045, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=7.490e-05, train_time=3.331 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:58:41,845 (trainer:732) INFO: 2epoch:train:1197-1288batch: iter_time=3.757e-04, forward_time=0.261, loss_att=472.561, acc=0.623, loss=472.561, backward_time=0.391, grad_norm=93.746, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=7.720e-05, train_time=3.345 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 21:59:59,431 (trainer:732) INFO: 2epoch:train:1289-1380batch: iter_time=4.211e-04, forward_time=0.261, loss_att=456.163, acc=0.625, loss=456.163, backward_time=0.390, grad_norm=86.394, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=7.950e-05, train_time=3.373 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:01:16,217 (trainer:732) INFO: 2epoch:train:1381-1472batch: iter_time=4.719e-04, forward_time=0.261, loss_att=445.974, acc=0.628, loss=445.974, backward_time=0.388, grad_norm=82.980, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=8.180e-05, train_time=3.338 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:02:32,860 (trainer:732) INFO: 2epoch:train:1473-1564batch: iter_time=4.192e-04, forward_time=0.261, loss_att=444.274, acc=0.629, loss=444.274, backward_time=0.389, grad_norm=86.458, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=8.410e-05, train_time=3.332 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:03:49,439 (trainer:732) INFO: 2epoch:train:1565-1656batch: iter_time=3.903e-04, forward_time=0.260, loss_att=446.283, acc=0.629, loss=446.283, backward_time=0.390, grad_norm=88.921, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=8.640e-05, train_time=3.329 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:05:05,156 (trainer:732) INFO: 2epoch:train:1657-1748batch: iter_time=3.728e-04, forward_time=0.257, loss_att=424.861, acc=0.622, loss=424.861, backward_time=0.383, grad_norm=80.672, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=8.870e-05, train_time=3.291 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:06:20,962 (trainer:732) INFO: 2epoch:train:1749-1840batch: iter_time=3.115e-04, forward_time=0.258, loss_att=431.515, acc=0.629, loss=431.515, backward_time=0.385, grad_norm=84.235, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=9.100e-05, train_time=3.295 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:15:33,529 (trainer:338) INFO: 2epoch results: [train] iter_time=7.958e-04, forward_time=0.260, loss_att=455.063, acc=0.613, loss=455.063, backward_time=0.388, grad_norm=91.838, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=6.915e-05, train_time=3.634, time=27 minutes and 56.23 seconds, total_count=3686, gpu_max_cached_mem_GB=29.846, [valid] loss_att=389.426, acc=0.661, cer=0.410, wer=0.747, loss=389.426, time=3 minutes and 57.23 seconds, total_count=38, gpu_max_cached_mem_GB=29.846, [att_plot] time=5 minutes and 10.84 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:15:39,765 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:15:39,767 (trainer:272) INFO: 3/60epoch started. Estimated time to finish: 1 day, 13 hours and 9 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:19:30,921 (trainer:732) INFO: 3epoch:train:1-92batch: iter_time=0.013, forward_time=0.265, loss_att=446.562, acc=0.638, loss=446.562, backward_time=0.391, grad_norm=91.071, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=9.330e-05, train_time=10.049 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:20:47,351 (trainer:732) INFO: 3epoch:train:93-184batch: iter_time=4.660e-04, forward_time=0.259, loss_att=405.526, acc=0.635, loss=405.526, backward_time=0.386, grad_norm=82.651, clip=100.000, loss_scale=1.000, optim_step_time=0.097, optim0_lr0=9.560e-05, train_time=3.322 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:22:03,317 (trainer:732) INFO: 3epoch:train:185-276batch: iter_time=3.534e-04, forward_time=0.258, loss_att=419.676, acc=0.637, loss=419.676, backward_time=0.385, grad_norm=74.275, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=9.790e-05, train_time=3.302 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:23:20,331 (trainer:732) INFO: 3epoch:train:277-368batch: iter_time=3.600e-04, forward_time=0.260, loss_att=427.559, acc=0.640, loss=427.559, backward_time=0.390, grad_norm=90.732, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=1.002e-04, train_time=3.348 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:24:37,399 (trainer:732) INFO: 3epoch:train:369-460batch: iter_time=3.850e-04, forward_time=0.261, loss_att=429.949, acc=0.642, loss=429.949, backward_time=0.390, grad_norm=87.095, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=1.025e-04, train_time=3.350 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:25:53,755 (trainer:732) INFO: 3epoch:train:461-552batch: iter_time=3.628e-04, forward_time=0.259, loss_att=414.754, acc=0.641, loss=414.754, backward_time=0.387, grad_norm=95.547, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=1.048e-04, train_time=3.319 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:27:10,499 (trainer:732) INFO: 3epoch:train:553-644batch: iter_time=4.026e-04, forward_time=0.261, loss_att=431.943, acc=0.643, loss=431.943, backward_time=0.388, grad_norm=82.500, clip=100.000, loss_scale=1.000, optim_step_time=0.095, optim0_lr0=1.071e-04, train_time=3.336 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:28:27,054 (trainer:732) INFO: 3epoch:train:645-736batch: iter_time=3.902e-04, forward_time=0.260, loss_att=418.332, acc=0.648, loss=418.332, backward_time=0.388, grad_norm=80.831, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=1.094e-04, train_time=3.328 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:29:43,328 (trainer:732) INFO: 3epoch:train:737-828batch: iter_time=3.900e-04, forward_time=0.258, loss_att=410.037, acc=0.646, loss=410.037, backward_time=0.384, grad_norm=79.701, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=1.117e-04, train_time=3.316 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:30:59,494 (trainer:732) INFO: 3epoch:train:829-920batch: iter_time=4.301e-04, forward_time=0.259, loss_att=410.956, acc=0.647, loss=410.956, backward_time=0.387, grad_norm=85.396, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=1.140e-04, train_time=3.311 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:32:15,073 (trainer:732) INFO: 3epoch:train:921-1012batch: iter_time=3.872e-04, forward_time=0.257, loss_att=394.924, acc=0.648, loss=394.924, backward_time=0.383, grad_norm=84.243, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=1.163e-04, train_time=3.286 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:33:31,473 (trainer:732) INFO: 3epoch:train:1013-1104batch: iter_time=3.815e-04, forward_time=0.258, loss_att=402.108, acc=0.648, loss=402.108, backward_time=0.386, grad_norm=100.589, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=1.186e-04, train_time=3.321 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:34:47,768 (trainer:732) INFO: 3epoch:train:1105-1196batch: iter_time=3.501e-04, forward_time=0.259, loss_att=418.072, acc=0.649, loss=418.072, backward_time=0.387, grad_norm=77.350, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=1.209e-04, train_time=3.317 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:36:04,309 (trainer:732) INFO: 3epoch:train:1197-1288batch: iter_time=3.895e-04, forward_time=0.260, loss_att=405.344, acc=0.654, loss=405.344, backward_time=0.389, grad_norm=81.389, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=1.232e-04, train_time=3.327 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:37:21,366 (trainer:732) INFO: 3epoch:train:1289-1380batch: iter_time=4.281e-04, forward_time=0.262, loss_att=418.769, acc=0.656, loss=418.769, backward_time=0.391, grad_norm=87.959, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=1.255e-04, train_time=3.350 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:38:38,459 (trainer:732) INFO: 3epoch:train:1381-1472batch: iter_time=3.962e-04, forward_time=0.261, loss_att=410.565, acc=0.653, loss=410.565, backward_time=0.390, grad_norm=82.446, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=1.278e-04, train_time=3.351 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:39:53,934 (trainer:732) INFO: 3epoch:train:1473-1564batch: iter_time=4.048e-04, forward_time=0.257, loss_att=381.653, acc=0.649, loss=381.653, backward_time=0.382, grad_norm=85.934, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=1.301e-04, train_time=3.281 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:41:12,579 (trainer:732) INFO: 3epoch:train:1565-1656batch: iter_time=4.142e-04, forward_time=0.265, loss_att=423.523, acc=0.656, loss=423.523, backward_time=0.393, grad_norm=93.283, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=1.324e-04, train_time=3.419 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:42:30,286 (trainer:732) INFO: 3epoch:train:1657-1748batch: iter_time=4.174e-04, forward_time=0.263, loss_att=394.223, acc=0.657, loss=394.223, backward_time=0.388, grad_norm=86.070, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=1.347e-04, train_time=3.378 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:43:47,322 (trainer:732) INFO: 3epoch:train:1749-1840batch: iter_time=3.729e-04, forward_time=0.263, loss_att=398.627, acc=0.657, loss=398.627, backward_time=0.386, grad_norm=77.743, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=1.370e-04, train_time=3.349 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:56:20,635 (trainer:338) INFO: 3epoch results: [train] iter_time=0.001, forward_time=0.260, loss_att=412.589, acc=0.647, loss=412.589, backward_time=0.388, grad_norm=85.340, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=1.151e-04, train_time=3.668, time=28 minutes and 13.08 seconds, total_count=5529, gpu_max_cached_mem_GB=29.846, [valid] loss_att=359.243, acc=0.686, cer=0.384, wer=0.728, loss=359.243, time=5 minutes and 22.01 seconds, total_count=57, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 5.78 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:56:30,099 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 22:56:30,104 (trainer:272) INFO: 4/60epoch started. Estimated time to finish: 1 day, 13 hours and 16 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:02:28,428 (trainer:732) INFO: 4epoch:train:1-92batch: iter_time=0.014, forward_time=0.283, loss_att=395.550, acc=0.664, loss=395.550, backward_time=0.395, grad_norm=87.220, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=1.393e-04, train_time=15.577 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:03:48,319 (trainer:732) INFO: 4epoch:train:93-184batch: iter_time=5.115e-04, forward_time=0.272, loss_att=392.822, acc=0.665, loss=392.822, backward_time=0.390, grad_norm=83.644, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=1.416e-04, train_time=3.473 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:05:07,895 (trainer:732) INFO: 4epoch:train:185-276batch: iter_time=4.925e-04, forward_time=0.273, loss_att=395.235, acc=0.660, loss=395.235, backward_time=0.392, grad_norm=83.264, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=1.439e-04, train_time=3.459 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:06:27,130 (trainer:732) INFO: 4epoch:train:277-368batch: iter_time=5.609e-04, forward_time=0.271, loss_att=379.614, acc=0.660, loss=379.614, backward_time=0.388, grad_norm=78.257, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=1.462e-04, train_time=3.444 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:07:48,057 (trainer:732) INFO: 4epoch:train:369-460batch: iter_time=5.647e-04, forward_time=0.281, loss_att=406.639, acc=0.666, loss=406.639, backward_time=0.398, grad_norm=91.189, clip=100.000, loss_scale=1.000, optim_step_time=0.127, optim0_lr0=1.485e-04, train_time=3.518 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:09:07,133 (trainer:732) INFO: 4epoch:train:461-552batch: iter_time=4.571e-04, forward_time=0.271, loss_att=394.303, acc=0.670, loss=394.303, backward_time=0.390, grad_norm=93.230, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=1.508e-04, train_time=3.437 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:10:27,675 (trainer:732) INFO: 4epoch:train:553-644batch: iter_time=4.543e-04, forward_time=0.275, loss_att=403.794, acc=0.669, loss=403.794, backward_time=0.392, grad_norm=91.601, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=1.531e-04, train_time=3.501 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:11:47,193 (trainer:732) INFO: 4epoch:train:645-736batch: iter_time=5.377e-04, forward_time=0.277, loss_att=375.524, acc=0.671, loss=375.524, backward_time=0.390, grad_norm=84.828, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=1.554e-04, train_time=3.457 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:13:06,872 (trainer:732) INFO: 4epoch:train:737-828batch: iter_time=5.831e-04, forward_time=0.276, loss_att=389.410, acc=0.670, loss=389.410, backward_time=0.390, grad_norm=90.953, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=1.577e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:14:27,551 (trainer:732) INFO: 4epoch:train:829-920batch: iter_time=7.001e-04, forward_time=0.275, loss_att=378.464, acc=0.676, loss=378.464, backward_time=0.393, grad_norm=88.638, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=1.600e-04, train_time=3.507 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:15:47,634 (trainer:732) INFO: 4epoch:train:921-1012batch: iter_time=4.791e-04, forward_time=0.276, loss_att=375.535, acc=0.674, loss=375.535, backward_time=0.390, grad_norm=79.491, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=1.623e-04, train_time=3.481 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:17:06,849 (trainer:732) INFO: 4epoch:train:1013-1104batch: iter_time=5.549e-04, forward_time=0.273, loss_att=364.024, acc=0.675, loss=364.024, backward_time=0.387, grad_norm=82.168, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=1.646e-04, train_time=3.443 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:18:26,005 (trainer:732) INFO: 4epoch:train:1105-1196batch: iter_time=5.715e-04, forward_time=0.273, loss_att=368.039, acc=0.668, loss=368.039, backward_time=0.389, grad_norm=88.219, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=1.669e-04, train_time=3.441 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:19:47,025 (trainer:732) INFO: 4epoch:train:1197-1288batch: iter_time=4.741e-04, forward_time=0.278, loss_att=403.477, acc=0.681, loss=403.477, backward_time=0.398, grad_norm=90.258, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=1.692e-04, train_time=3.522 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:21:07,192 (trainer:732) INFO: 4epoch:train:1289-1380batch: iter_time=5.060e-04, forward_time=0.277, loss_att=385.285, acc=0.679, loss=385.285, backward_time=0.395, grad_norm=107.398, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=1.715e-04, train_time=3.485 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:22:27,113 (trainer:732) INFO: 4epoch:train:1381-1472batch: iter_time=4.775e-04, forward_time=0.272, loss_att=373.051, acc=0.675, loss=373.051, backward_time=0.390, grad_norm=103.915, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=1.738e-04, train_time=3.474 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:23:46,354 (trainer:732) INFO: 4epoch:train:1473-1564batch: iter_time=4.800e-04, forward_time=0.272, loss_att=375.903, acc=0.676, loss=375.903, backward_time=0.390, grad_norm=81.805, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=1.761e-04, train_time=3.445 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:25:03,698 (trainer:732) INFO: 4epoch:train:1565-1656batch: iter_time=4.301e-04, forward_time=0.265, loss_att=364.463, acc=0.681, loss=364.463, backward_time=0.385, grad_norm=91.910, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=1.784e-04, train_time=3.362 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:26:21,519 (trainer:732) INFO: 4epoch:train:1657-1748batch: iter_time=4.673e-04, forward_time=0.267, loss_att=360.856, acc=0.679, loss=360.856, backward_time=0.384, grad_norm=82.084, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=1.807e-04, train_time=3.383 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:27:39,578 (trainer:732) INFO: 4epoch:train:1749-1840batch: iter_time=3.606e-04, forward_time=0.266, loss_att=357.292, acc=0.683, loss=357.292, backward_time=0.386, grad_norm=85.395, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=1.830e-04, train_time=3.393 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:40:03,652 (trainer:338) INFO: 4epoch results: [train] iter_time=0.001, forward_time=0.274, loss_att=381.677, acc=0.672, loss=381.677, backward_time=0.391, grad_norm=88.273, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=1.611e-04, train_time=4.063, time=31 minutes and 15.81 seconds, total_count=7372, gpu_max_cached_mem_GB=29.846, [valid] loss_att=338.790, acc=0.703, cer=0.365, wer=0.707, loss=338.790, time=5 minutes and 21.12 seconds, total_count=76, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 56.6 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:40:12,258 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:40:12,261 (trainer:272) INFO: 5/60epoch started. Estimated time to finish: 1 day, 13 hours and 39 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:46:17,753 (trainer:732) INFO: 5epoch:train:1-92batch: iter_time=0.006, forward_time=0.291, loss_att=369.932, acc=0.684, loss=369.932, backward_time=0.394, grad_norm=97.831, clip=100.000, loss_scale=1.000, optim_step_time=0.134, optim0_lr0=1.853e-04, train_time=15.889 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:47:39,265 (trainer:732) INFO: 5epoch:train:93-184batch: iter_time=5.495e-04, forward_time=0.280, loss_att=371.687, acc=0.684, loss=371.687, backward_time=0.392, grad_norm=95.149, clip=100.000, loss_scale=1.000, optim_step_time=0.149, optim0_lr0=1.876e-04, train_time=3.542 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:48:58,071 (trainer:732) INFO: 5epoch:train:185-276batch: iter_time=4.974e-04, forward_time=0.270, loss_att=351.523, acc=0.682, loss=351.523, backward_time=0.389, grad_norm=85.637, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=1.899e-04, train_time=3.427 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:50:19,178 (trainer:732) INFO: 5epoch:train:277-368batch: iter_time=5.573e-04, forward_time=0.278, loss_att=367.653, acc=0.689, loss=367.653, backward_time=0.394, grad_norm=85.919, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=1.922e-04, train_time=3.526 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:51:38,863 (trainer:732) INFO: 5epoch:train:369-460batch: iter_time=5.276e-04, forward_time=0.272, loss_att=346.492, acc=0.688, loss=346.492, backward_time=0.389, grad_norm=90.708, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=1.945e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:52:59,375 (trainer:732) INFO: 5epoch:train:461-552batch: iter_time=5.275e-04, forward_time=0.275, loss_att=357.806, acc=0.689, loss=357.806, backward_time=0.394, grad_norm=94.561, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=1.968e-04, train_time=3.500 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:54:18,341 (trainer:732) INFO: 5epoch:train:553-644batch: iter_time=4.294e-04, forward_time=0.271, loss_att=364.707, acc=0.690, loss=364.707, backward_time=0.390, grad_norm=94.799, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=1.991e-04, train_time=3.433 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:55:37,749 (trainer:732) INFO: 5epoch:train:645-736batch: iter_time=5.066e-04, forward_time=0.275, loss_att=361.075, acc=0.687, loss=361.075, backward_time=0.390, grad_norm=97.542, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=2.014e-04, train_time=3.452 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:56:58,722 (trainer:732) INFO: 5epoch:train:737-828batch: iter_time=5.702e-04, forward_time=0.275, loss_att=365.067, acc=0.696, loss=365.067, backward_time=0.395, grad_norm=87.361, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=2.037e-04, train_time=3.520 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:58:18,684 (trainer:732) INFO: 5epoch:train:829-920batch: iter_time=9.188e-04, forward_time=0.276, loss_att=342.448, acc=0.691, loss=342.448, backward_time=0.389, grad_norm=84.371, clip=100.000, loss_scale=1.000, optim_step_time=0.135, optim0_lr0=2.060e-04, train_time=3.476 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-16 23:59:38,046 (trainer:732) INFO: 5epoch:train:921-1012batch: iter_time=6.891e-04, forward_time=0.274, loss_att=347.839, acc=0.691, loss=347.839, backward_time=0.387, grad_norm=90.194, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=2.083e-04, train_time=3.450 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:00:56,479 (trainer:732) INFO: 5epoch:train:1013-1104batch: iter_time=4.538e-04, forward_time=0.269, loss_att=337.572, acc=0.692, loss=337.572, backward_time=0.383, grad_norm=83.565, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=2.106e-04, train_time=3.409 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:02:16,454 (trainer:732) INFO: 5epoch:train:1105-1196batch: iter_time=4.752e-04, forward_time=0.277, loss_att=358.172, acc=0.696, loss=358.172, backward_time=0.394, grad_norm=91.488, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=2.129e-04, train_time=3.476 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:03:37,061 (trainer:732) INFO: 5epoch:train:1197-1288batch: iter_time=6.341e-04, forward_time=0.278, loss_att=355.488, acc=0.692, loss=355.488, backward_time=0.392, grad_norm=89.308, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=2.152e-04, train_time=3.504 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:04:56,562 (trainer:732) INFO: 5epoch:train:1289-1380batch: iter_time=5.129e-04, forward_time=0.275, loss_att=350.804, acc=0.696, loss=350.804, backward_time=0.391, grad_norm=86.425, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=2.175e-04, train_time=3.456 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:06:17,254 (trainer:732) INFO: 5epoch:train:1381-1472batch: iter_time=4.597e-04, forward_time=0.275, loss_att=362.019, acc=0.701, loss=362.019, backward_time=0.394, grad_norm=89.241, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=2.198e-04, train_time=3.507 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:07:37,589 (trainer:732) INFO: 5epoch:train:1473-1564batch: iter_time=5.801e-04, forward_time=0.277, loss_att=356.513, acc=0.695, loss=356.513, backward_time=0.395, grad_norm=91.688, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=2.221e-04, train_time=3.493 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:08:57,739 (trainer:732) INFO: 5epoch:train:1565-1656batch: iter_time=6.854e-04, forward_time=0.278, loss_att=353.561, acc=0.700, loss=353.561, backward_time=0.390, grad_norm=85.831, clip=100.000, loss_scale=1.000, optim_step_time=0.129, optim0_lr0=2.244e-04, train_time=3.484 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:10:19,775 (trainer:732) INFO: 5epoch:train:1657-1748batch: iter_time=4.762e-04, forward_time=0.281, loss_att=363.020, acc=0.699, loss=363.020, backward_time=0.393, grad_norm=89.792, clip=100.000, loss_scale=1.000, optim_step_time=0.160, optim0_lr0=2.267e-04, train_time=3.566 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:11:39,232 (trainer:732) INFO: 5epoch:train:1749-1840batch: iter_time=3.995e-04, forward_time=0.271, loss_att=362.606, acc=0.702, loss=362.606, backward_time=0.392, grad_norm=94.564, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=2.290e-04, train_time=3.454 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:23:46,842 (trainer:338) INFO: 5epoch results: [train] iter_time=8.129e-04, forward_time=0.276, loss_att=357.045, acc=0.692, loss=357.045, backward_time=0.391, grad_norm=90.299, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=2.071e-04, train_time=4.101, time=31 minutes and 37.35 seconds, total_count=9215, gpu_max_cached_mem_GB=29.846, [valid] loss_att=321.993, acc=0.717, cer=0.350, wer=0.684, loss=321.993, time=5 minutes and 9.24 seconds, total_count=95, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 47.99 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:23:55,838 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:23:55,849 (trainer:272) INFO: 6/60epoch started. Estimated time to finish: 1 day, 13 hours and 36 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:29:54,954 (trainer:732) INFO: 6epoch:train:1-92batch: iter_time=0.014, forward_time=0.279, loss_att=335.008, acc=0.702, loss=335.008, backward_time=0.389, grad_norm=83.626, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=2.313e-04, train_time=15.612 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:31:14,809 (trainer:732) INFO: 6epoch:train:93-184batch: iter_time=5.346e-04, forward_time=0.268, loss_att=348.784, acc=0.702, loss=348.784, backward_time=0.390, grad_norm=103.711, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=2.336e-04, train_time=3.471 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:32:34,146 (trainer:732) INFO: 6epoch:train:185-276batch: iter_time=4.810e-04, forward_time=0.271, loss_att=331.950, acc=0.706, loss=331.950, backward_time=0.388, grad_norm=91.035, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=2.359e-04, train_time=3.449 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:33:52,506 (trainer:732) INFO: 6epoch:train:277-368batch: iter_time=5.257e-04, forward_time=0.269, loss_att=337.335, acc=0.704, loss=337.335, backward_time=0.390, grad_norm=102.559, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=2.382e-04, train_time=3.406 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:35:13,634 (trainer:732) INFO: 6epoch:train:369-460batch: iter_time=6.043e-04, forward_time=0.279, loss_att=358.551, acc=0.705, loss=358.551, backward_time=0.396, grad_norm=104.663, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=2.405e-04, train_time=3.526 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:36:33,338 (trainer:732) INFO: 6epoch:train:461-552batch: iter_time=4.863e-04, forward_time=0.274, loss_att=340.427, acc=0.706, loss=340.427, backward_time=0.391, grad_norm=89.881, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=2.428e-04, train_time=3.465 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:37:53,027 (trainer:732) INFO: 6epoch:train:553-644batch: iter_time=5.124e-04, forward_time=0.272, loss_att=337.296, acc=0.709, loss=337.296, backward_time=0.389, grad_norm=82.894, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=2.451e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:39:11,888 (trainer:732) INFO: 6epoch:train:645-736batch: iter_time=5.736e-04, forward_time=0.271, loss_att=329.458, acc=0.708, loss=329.458, backward_time=0.388, grad_norm=85.718, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=2.474e-04, train_time=3.428 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:40:31,518 (trainer:732) INFO: 6epoch:train:737-828batch: iter_time=5.143e-04, forward_time=0.274, loss_att=347.518, acc=0.709, loss=347.518, backward_time=0.396, grad_norm=91.714, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=2.497e-04, train_time=3.461 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:41:50,538 (trainer:732) INFO: 6epoch:train:829-920batch: iter_time=4.988e-04, forward_time=0.267, loss_att=346.501, acc=0.708, loss=346.501, backward_time=0.389, grad_norm=92.575, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=2.520e-04, train_time=3.435 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:43:10,138 (trainer:732) INFO: 6epoch:train:921-1012batch: iter_time=5.386e-04, forward_time=0.273, loss_att=326.112, acc=0.711, loss=326.112, backward_time=0.388, grad_norm=86.340, clip=100.000, loss_scale=1.000, optim_step_time=0.133, optim0_lr0=2.543e-04, train_time=3.460 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:44:29,429 (trainer:732) INFO: 6epoch:train:1013-1104batch: iter_time=6.323e-04, forward_time=0.273, loss_att=327.556, acc=0.708, loss=327.556, backward_time=0.390, grad_norm=97.475, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=2.566e-04, train_time=3.447 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:45:49,505 (trainer:732) INFO: 6epoch:train:1105-1196batch: iter_time=5.086e-04, forward_time=0.275, loss_att=344.253, acc=0.709, loss=344.253, backward_time=0.391, grad_norm=103.827, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=2.589e-04, train_time=3.481 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:47:09,846 (trainer:732) INFO: 6epoch:train:1197-1288batch: iter_time=5.120e-04, forward_time=0.277, loss_att=342.073, acc=0.715, loss=342.073, backward_time=0.395, grad_norm=93.314, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=2.612e-04, train_time=3.492 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:48:30,006 (trainer:732) INFO: 6epoch:train:1289-1380batch: iter_time=6.022e-04, forward_time=0.278, loss_att=332.750, acc=0.713, loss=332.750, backward_time=0.393, grad_norm=101.489, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=2.635e-04, train_time=3.484 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:49:48,859 (trainer:732) INFO: 6epoch:train:1381-1472batch: iter_time=6.347e-04, forward_time=0.270, loss_att=325.547, acc=0.712, loss=325.547, backward_time=0.387, grad_norm=100.364, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=2.658e-04, train_time=3.428 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:51:08,775 (trainer:732) INFO: 6epoch:train:1473-1564batch: iter_time=6.860e-04, forward_time=0.276, loss_att=330.705, acc=0.718, loss=330.705, backward_time=0.392, grad_norm=92.359, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=2.681e-04, train_time=3.474 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:52:28,473 (trainer:732) INFO: 6epoch:train:1565-1656batch: iter_time=4.827e-04, forward_time=0.274, loss_att=330.093, acc=0.714, loss=330.093, backward_time=0.389, grad_norm=88.628, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=2.704e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:53:48,014 (trainer:732) INFO: 6epoch:train:1657-1748batch: iter_time=5.315e-04, forward_time=0.273, loss_att=317.105, acc=0.713, loss=317.105, backward_time=0.388, grad_norm=90.059, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=2.727e-04, train_time=3.458 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 00:55:07,657 (trainer:732) INFO: 6epoch:train:1749-1840batch: iter_time=4.648e-04, forward_time=0.274, loss_att=335.393, acc=0.715, loss=335.393, backward_time=0.391, grad_norm=87.324, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=2.750e-04, train_time=3.462 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:07:23,504 (trainer:338) INFO: 6epoch results: [train] iter_time=0.001, forward_time=0.273, loss_att=336.004, acc=0.709, loss=336.004, backward_time=0.391, grad_norm=93.478, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=2.532e-04, train_time=4.068, time=31 minutes and 18.09 seconds, total_count=11058, gpu_max_cached_mem_GB=29.846, [valid] loss_att=307.829, acc=0.729, cer=0.335, wer=0.668, loss=307.829, time=5 minutes and 13.04 seconds, total_count=114, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 56.51 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:07:33,160 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:07:33,167 (trainer:272) INFO: 7/60epoch started. Estimated time to finish: 1 day, 13 hours and 18 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:13:27,374 (trainer:732) INFO: 7epoch:train:1-92batch: iter_time=0.009, forward_time=0.281, loss_att=316.862, acc=0.717, loss=316.862, backward_time=0.386, grad_norm=97.806, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=2.773e-04, train_time=15.399 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:14:48,165 (trainer:732) INFO: 7epoch:train:93-184batch: iter_time=6.036e-04, forward_time=0.282, loss_att=327.529, acc=0.719, loss=327.529, backward_time=0.394, grad_norm=120.478, clip=100.000, loss_scale=1.000, optim_step_time=0.130, optim0_lr0=2.796e-04, train_time=3.512 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:16:09,245 (trainer:732) INFO: 7epoch:train:185-276batch: iter_time=9.007e-04, forward_time=0.275, loss_att=328.771, acc=0.719, loss=328.771, backward_time=0.391, grad_norm=103.902, clip=100.000, loss_scale=1.000, optim_step_time=0.136, optim0_lr0=2.819e-04, train_time=3.524 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:17:30,166 (trainer:732) INFO: 7epoch:train:277-368batch: iter_time=5.944e-04, forward_time=0.279, loss_att=334.003, acc=0.726, loss=334.003, backward_time=0.398, grad_norm=96.424, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=2.842e-04, train_time=3.518 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:18:51,278 (trainer:732) INFO: 7epoch:train:369-460batch: iter_time=5.991e-04, forward_time=0.279, loss_att=330.787, acc=0.721, loss=330.787, backward_time=0.398, grad_norm=103.721, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=2.865e-04, train_time=3.526 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:20:10,950 (trainer:732) INFO: 7epoch:train:461-552batch: iter_time=5.325e-04, forward_time=0.274, loss_att=317.207, acc=0.729, loss=317.207, backward_time=0.391, grad_norm=92.441, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=2.888e-04, train_time=3.463 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:21:30,108 (trainer:732) INFO: 7epoch:train:553-644batch: iter_time=6.813e-04, forward_time=0.272, loss_att=314.757, acc=0.725, loss=314.757, backward_time=0.388, grad_norm=93.397, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=2.911e-04, train_time=3.441 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:22:49,050 (trainer:732) INFO: 7epoch:train:645-736batch: iter_time=4.990e-04, forward_time=0.271, loss_att=311.547, acc=0.721, loss=311.547, backward_time=0.388, grad_norm=96.793, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=2.934e-04, train_time=3.431 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:24:08,651 (trainer:732) INFO: 7epoch:train:737-828batch: iter_time=5.064e-04, forward_time=0.274, loss_att=313.122, acc=0.724, loss=313.122, backward_time=0.389, grad_norm=93.834, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=2.957e-04, train_time=3.460 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:25:28,144 (trainer:732) INFO: 7epoch:train:829-920batch: iter_time=4.912e-04, forward_time=0.273, loss_att=315.630, acc=0.725, loss=315.630, backward_time=0.394, grad_norm=88.322, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=2.980e-04, train_time=3.455 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:26:45,634 (trainer:732) INFO: 7epoch:train:921-1012batch: iter_time=4.621e-04, forward_time=0.265, loss_att=311.570, acc=0.720, loss=311.570, backward_time=0.383, grad_norm=87.631, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=3.003e-04, train_time=3.368 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:28:03,884 (trainer:732) INFO: 7epoch:train:1013-1104batch: iter_time=5.043e-04, forward_time=0.269, loss_att=312.468, acc=0.724, loss=312.468, backward_time=0.388, grad_norm=95.123, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=3.026e-04, train_time=3.401 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:29:23,108 (trainer:732) INFO: 7epoch:train:1105-1196batch: iter_time=4.339e-04, forward_time=0.271, loss_att=326.538, acc=0.720, loss=326.538, backward_time=0.391, grad_norm=96.710, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=3.049e-04, train_time=3.444 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:30:41,434 (trainer:732) INFO: 7epoch:train:1197-1288batch: iter_time=7.565e-04, forward_time=0.268, loss_att=315.214, acc=0.722, loss=315.214, backward_time=0.385, grad_norm=102.779, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=3.072e-04, train_time=3.404 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:32:00,279 (trainer:732) INFO: 7epoch:train:1289-1380batch: iter_time=4.661e-04, forward_time=0.272, loss_att=313.549, acc=0.726, loss=313.549, backward_time=0.389, grad_norm=90.901, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=3.095e-04, train_time=3.427 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:33:19,228 (trainer:732) INFO: 7epoch:train:1381-1472batch: iter_time=4.718e-04, forward_time=0.272, loss_att=323.579, acc=0.728, loss=323.579, backward_time=0.388, grad_norm=92.473, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=3.118e-04, train_time=3.432 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:34:38,266 (trainer:732) INFO: 7epoch:train:1473-1564batch: iter_time=4.561e-04, forward_time=0.271, loss_att=317.548, acc=0.730, loss=317.548, backward_time=0.390, grad_norm=93.413, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=3.141e-04, train_time=3.435 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:35:57,447 (trainer:732) INFO: 7epoch:train:1565-1656batch: iter_time=5.000e-04, forward_time=0.269, loss_att=300.485, acc=0.728, loss=300.485, backward_time=0.388, grad_norm=94.723, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=3.164e-04, train_time=3.442 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:37:17,078 (trainer:732) INFO: 7epoch:train:1657-1748batch: iter_time=4.794e-04, forward_time=0.271, loss_att=329.024, acc=0.734, loss=329.024, backward_time=0.396, grad_norm=101.906, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=3.187e-04, train_time=3.461 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:38:34,353 (trainer:732) INFO: 7epoch:train:1749-1840batch: iter_time=3.733e-04, forward_time=0.262, loss_att=303.206, acc=0.729, loss=303.206, backward_time=0.386, grad_norm=89.098, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=3.210e-04, train_time=3.359 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:49:37,369 (trainer:338) INFO: 7epoch results: [train] iter_time=9.407e-04, forward_time=0.272, loss_att=317.909, acc=0.724, loss=317.909, backward_time=0.390, grad_norm=96.594, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=2.991e-04, train_time=4.045, time=31 minutes and 7.7 seconds, total_count=12901, gpu_max_cached_mem_GB=29.846, [valid] loss_att=296.060, acc=0.739, cer=0.325, wer=0.651, loss=296.060, time=5 minutes and 10.06 seconds, total_count=133, gpu_max_cached_mem_GB=29.846, [att_plot] time=5 minutes and 46.43 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:49:44,433 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:49:44,437 (trainer:272) INFO: 8/60epoch started. Estimated time to finish: 1 day, 12 hours and 42 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:53:44,164 (trainer:732) INFO: 8epoch:train:1-92batch: iter_time=0.010, forward_time=0.265, loss_att=308.085, acc=0.736, loss=308.085, backward_time=0.389, grad_norm=94.456, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=3.233e-04, train_time=10.421 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:55:01,399 (trainer:732) INFO: 8epoch:train:93-184batch: iter_time=4.494e-04, forward_time=0.263, loss_att=302.631, acc=0.732, loss=302.631, backward_time=0.386, grad_norm=89.114, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=3.256e-04, train_time=3.357 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:56:18,708 (trainer:732) INFO: 8epoch:train:185-276batch: iter_time=4.905e-04, forward_time=0.262, loss_att=296.217, acc=0.734, loss=296.217, backward_time=0.387, grad_norm=91.109, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=3.279e-04, train_time=3.361 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:57:37,094 (trainer:732) INFO: 8epoch:train:277-368batch: iter_time=4.535e-04, forward_time=0.266, loss_att=312.143, acc=0.739, loss=312.143, backward_time=0.391, grad_norm=108.400, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=3.302e-04, train_time=3.407 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 01:58:54,749 (trainer:732) INFO: 8epoch:train:369-460batch: iter_time=4.541e-04, forward_time=0.263, loss_att=306.276, acc=0.732, loss=306.276, backward_time=0.389, grad_norm=107.338, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=3.325e-04, train_time=3.376 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:00:12,153 (trainer:732) INFO: 8epoch:train:461-552batch: iter_time=4.298e-04, forward_time=0.261, loss_att=303.130, acc=0.734, loss=303.130, backward_time=0.385, grad_norm=99.793, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=3.348e-04, train_time=3.365 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:01:29,976 (trainer:732) INFO: 8epoch:train:553-644batch: iter_time=3.881e-04, forward_time=0.264, loss_att=319.689, acc=0.738, loss=319.689, backward_time=0.390, grad_norm=99.812, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=3.371e-04, train_time=3.383 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:02:48,362 (trainer:732) INFO: 8epoch:train:645-736batch: iter_time=4.047e-04, forward_time=0.265, loss_att=299.691, acc=0.739, loss=299.691, backward_time=0.391, grad_norm=121.049, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=3.394e-04, train_time=3.407 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:04:05,430 (trainer:732) INFO: 8epoch:train:737-828batch: iter_time=4.151e-04, forward_time=0.261, loss_att=297.627, acc=0.735, loss=297.627, backward_time=0.384, grad_norm=95.812, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=3.417e-04, train_time=3.350 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:05:23,074 (trainer:732) INFO: 8epoch:train:829-920batch: iter_time=3.862e-04, forward_time=0.264, loss_att=307.074, acc=0.738, loss=307.074, backward_time=0.389, grad_norm=96.901, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=3.440e-04, train_time=3.375 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:06:40,430 (trainer:732) INFO: 8epoch:train:921-1012batch: iter_time=4.260e-04, forward_time=0.262, loss_att=300.759, acc=0.736, loss=300.759, backward_time=0.386, grad_norm=88.196, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=3.463e-04, train_time=3.363 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:07:58,008 (trainer:732) INFO: 8epoch:train:1013-1104batch: iter_time=4.487e-04, forward_time=0.265, loss_att=306.628, acc=0.740, loss=306.628, backward_time=0.390, grad_norm=95.447, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=3.486e-04, train_time=3.372 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:09:15,739 (trainer:732) INFO: 8epoch:train:1105-1196batch: iter_time=4.227e-04, forward_time=0.263, loss_att=294.897, acc=0.735, loss=294.897, backward_time=0.388, grad_norm=92.248, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=3.509e-04, train_time=3.379 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:10:33,170 (trainer:732) INFO: 8epoch:train:1197-1288batch: iter_time=4.968e-04, forward_time=0.264, loss_att=290.781, acc=0.739, loss=290.781, backward_time=0.389, grad_norm=89.679, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=3.532e-04, train_time=3.366 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:11:50,867 (trainer:732) INFO: 8epoch:train:1289-1380batch: iter_time=4.302e-04, forward_time=0.263, loss_att=285.424, acc=0.743, loss=285.424, backward_time=0.388, grad_norm=97.757, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=3.555e-04, train_time=3.377 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:13:09,547 (trainer:732) INFO: 8epoch:train:1381-1472batch: iter_time=4.440e-04, forward_time=0.266, loss_att=302.290, acc=0.740, loss=302.290, backward_time=0.394, grad_norm=111.018, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=3.578e-04, train_time=3.420 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:14:26,191 (trainer:732) INFO: 8epoch:train:1473-1564batch: iter_time=4.040e-04, forward_time=0.260, loss_att=300.572, acc=0.739, loss=300.572, backward_time=0.386, grad_norm=94.242, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=3.601e-04, train_time=3.332 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:15:43,369 (trainer:732) INFO: 8epoch:train:1565-1656batch: iter_time=4.261e-04, forward_time=0.262, loss_att=302.756, acc=0.739, loss=302.756, backward_time=0.388, grad_norm=94.250, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=3.624e-04, train_time=3.355 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:17:00,894 (trainer:732) INFO: 8epoch:train:1657-1748batch: iter_time=4.138e-04, forward_time=0.263, loss_att=297.465, acc=0.743, loss=297.465, backward_time=0.388, grad_norm=104.369, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=3.647e-04, train_time=3.370 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:18:19,005 (trainer:732) INFO: 8epoch:train:1749-1840batch: iter_time=3.330e-04, forward_time=0.264, loss_att=308.102, acc=0.743, loss=308.102, backward_time=0.392, grad_norm=99.980, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=3.670e-04, train_time=3.395 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:30:37,113 (trainer:338) INFO: 8epoch results: [train] iter_time=9.176e-04, forward_time=0.263, loss_att=301.965, acc=0.738, loss=301.965, backward_time=0.389, grad_norm=98.548, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=3.451e-04, train_time=3.727, time=28 minutes and 42.54 seconds, total_count=14744, gpu_max_cached_mem_GB=29.846, [valid] loss_att=287.050, acc=0.748, cer=0.316, wer=0.640, loss=287.050, time=5 minutes and 9.47 seconds, total_count=152, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 0.66 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:30:47,004 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:30:47,012 (trainer:272) INFO: 9/60epoch started. Estimated time to finish: 1 day, 11 hours and 57 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:36:48,600 (trainer:732) INFO: 9epoch:train:1-92batch: iter_time=0.013, forward_time=0.283, loss_att=293.404, acc=0.744, loss=293.404, backward_time=0.392, grad_norm=101.073, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=3.693e-04, train_time=15.718 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:38:08,874 (trainer:732) INFO: 9epoch:train:93-184batch: iter_time=6.717e-04, forward_time=0.277, loss_att=286.308, acc=0.749, loss=286.308, backward_time=0.392, grad_norm=104.038, clip=100.000, loss_scale=1.000, optim_step_time=0.126, optim0_lr0=3.716e-04, train_time=3.490 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:39:28,947 (trainer:732) INFO: 9epoch:train:185-276batch: iter_time=4.977e-04, forward_time=0.275, loss_att=281.931, acc=0.747, loss=281.931, backward_time=0.389, grad_norm=97.693, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=3.739e-04, train_time=3.481 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:40:50,289 (trainer:732) INFO: 9epoch:train:277-368batch: iter_time=5.353e-04, forward_time=0.278, loss_att=284.448, acc=0.750, loss=284.448, backward_time=0.392, grad_norm=100.757, clip=100.000, loss_scale=1.000, optim_step_time=0.132, optim0_lr0=3.762e-04, train_time=3.536 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:42:10,049 (trainer:732) INFO: 9epoch:train:369-460batch: iter_time=6.724e-04, forward_time=0.274, loss_att=287.808, acc=0.748, loss=287.808, backward_time=0.392, grad_norm=102.279, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=3.785e-04, train_time=3.467 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:43:30,120 (trainer:732) INFO: 9epoch:train:461-552batch: iter_time=4.467e-04, forward_time=0.274, loss_att=287.656, acc=0.746, loss=287.656, backward_time=0.393, grad_norm=94.518, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=3.808e-04, train_time=3.481 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:44:50,867 (trainer:732) INFO: 9epoch:train:553-644batch: iter_time=4.971e-04, forward_time=0.274, loss_att=291.182, acc=0.747, loss=291.182, backward_time=0.392, grad_norm=106.874, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=3.831e-04, train_time=3.510 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:46:10,935 (trainer:732) INFO: 9epoch:train:645-736batch: iter_time=4.758e-04, forward_time=0.274, loss_att=294.204, acc=0.748, loss=294.204, backward_time=0.392, grad_norm=109.055, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=3.854e-04, train_time=3.481 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:47:31,035 (trainer:732) INFO: 9epoch:train:737-828batch: iter_time=5.196e-04, forward_time=0.278, loss_att=295.071, acc=0.746, loss=295.071, backward_time=0.392, grad_norm=100.769, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=3.877e-04, train_time=3.482 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:48:50,068 (trainer:732) INFO: 9epoch:train:829-920batch: iter_time=4.811e-04, forward_time=0.272, loss_att=282.566, acc=0.751, loss=282.566, backward_time=0.391, grad_norm=88.573, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=3.900e-04, train_time=3.436 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:50:10,446 (trainer:732) INFO: 9epoch:train:921-1012batch: iter_time=5.522e-04, forward_time=0.279, loss_att=292.853, acc=0.749, loss=292.853, backward_time=0.389, grad_norm=101.722, clip=100.000, loss_scale=1.000, optim_step_time=0.129, optim0_lr0=3.923e-04, train_time=3.494 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:51:32,208 (trainer:732) INFO: 9epoch:train:1013-1104batch: iter_time=6.473e-04, forward_time=0.277, loss_att=288.344, acc=0.749, loss=288.344, backward_time=0.392, grad_norm=99.739, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=3.946e-04, train_time=3.554 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:52:52,709 (trainer:732) INFO: 9epoch:train:1105-1196batch: iter_time=5.141e-04, forward_time=0.273, loss_att=282.631, acc=0.753, loss=282.631, backward_time=0.393, grad_norm=100.500, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=3.969e-04, train_time=3.499 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:54:12,918 (trainer:732) INFO: 9epoch:train:1197-1288batch: iter_time=4.456e-04, forward_time=0.276, loss_att=301.572, acc=0.751, loss=301.572, backward_time=0.395, grad_norm=105.181, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=3.992e-04, train_time=3.486 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:55:32,264 (trainer:732) INFO: 9epoch:train:1289-1380batch: iter_time=4.830e-04, forward_time=0.273, loss_att=294.806, acc=0.750, loss=294.806, backward_time=0.391, grad_norm=93.704, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=4.015e-04, train_time=3.449 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:56:50,933 (trainer:732) INFO: 9epoch:train:1381-1472batch: iter_time=4.316e-04, forward_time=0.271, loss_att=282.694, acc=0.755, loss=282.694, backward_time=0.387, grad_norm=111.962, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=4.038e-04, train_time=3.420 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:58:10,180 (trainer:732) INFO: 9epoch:train:1473-1564batch: iter_time=4.622e-04, forward_time=0.271, loss_att=288.125, acc=0.756, loss=288.125, backward_time=0.390, grad_norm=96.608, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=4.061e-04, train_time=3.445 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 02:59:28,546 (trainer:732) INFO: 9epoch:train:1565-1656batch: iter_time=4.063e-04, forward_time=0.267, loss_att=280.891, acc=0.753, loss=280.891, backward_time=0.386, grad_norm=95.815, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=4.084e-04, train_time=3.407 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:00:46,587 (trainer:732) INFO: 9epoch:train:1657-1748batch: iter_time=4.177e-04, forward_time=0.269, loss_att=278.192, acc=0.747, loss=278.192, backward_time=0.383, grad_norm=93.789, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=4.107e-04, train_time=3.392 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:02:07,023 (trainer:732) INFO: 9epoch:train:1749-1840batch: iter_time=4.180e-04, forward_time=0.274, loss_att=285.979, acc=0.749, loss=285.979, backward_time=0.392, grad_norm=100.784, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=4.130e-04, train_time=3.496 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:14:33,826 (trainer:338) INFO: 9epoch results: [train] iter_time=0.001, forward_time=0.274, loss_att=287.979, acc=0.749, loss=287.979, backward_time=0.391, grad_norm=100.272, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=3.911e-04, train_time=4.086, time=31 minutes and 26.65 seconds, total_count=16587, gpu_max_cached_mem_GB=29.846, [valid] loss_att=281.164, acc=0.753, cer=0.312, wer=0.627, loss=281.164, time=5 minutes and 11.7 seconds, total_count=171, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 8.45 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:14:50,018 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:14:50,031 (trainer:272) INFO: 10/60epoch started. Estimated time to finish: 1 day, 11 hours and 30 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:21:06,909 (trainer:732) INFO: 10epoch:train:1-92batch: iter_time=0.011, forward_time=0.283, loss_att=274.394, acc=0.756, loss=274.394, backward_time=0.389, grad_norm=107.478, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=4.153e-04, train_time=16.384 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:22:27,435 (trainer:732) INFO: 10epoch:train:93-184batch: iter_time=7.018e-04, forward_time=0.278, loss_att=279.746, acc=0.756, loss=279.746, backward_time=0.391, grad_norm=91.815, clip=100.000, loss_scale=1.000, optim_step_time=0.127, optim0_lr0=4.176e-04, train_time=3.501 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:23:47,744 (trainer:732) INFO: 10epoch:train:185-276batch: iter_time=5.054e-04, forward_time=0.276, loss_att=286.885, acc=0.759, loss=286.885, backward_time=0.394, grad_norm=101.901, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=4.199e-04, train_time=3.491 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:25:07,047 (trainer:732) INFO: 10epoch:train:277-368batch: iter_time=5.764e-04, forward_time=0.274, loss_att=268.441, acc=0.759, loss=268.441, backward_time=0.386, grad_norm=91.991, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=4.222e-04, train_time=3.447 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:26:26,966 (trainer:732) INFO: 10epoch:train:369-460batch: iter_time=5.178e-04, forward_time=0.274, loss_att=284.739, acc=0.752, loss=284.739, backward_time=0.391, grad_norm=103.934, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=4.245e-04, train_time=3.474 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:27:47,174 (trainer:732) INFO: 10epoch:train:461-552batch: iter_time=8.574e-04, forward_time=0.274, loss_att=276.134, acc=0.754, loss=276.134, backward_time=0.393, grad_norm=100.015, clip=100.000, loss_scale=1.000, optim_step_time=0.130, optim0_lr0=4.268e-04, train_time=3.487 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:29:08,587 (trainer:732) INFO: 10epoch:train:553-644batch: iter_time=5.611e-04, forward_time=0.281, loss_att=267.982, acc=0.761, loss=267.982, backward_time=0.396, grad_norm=91.830, clip=100.000, loss_scale=1.000, optim_step_time=0.136, optim0_lr0=4.291e-04, train_time=3.539 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:30:28,965 (trainer:732) INFO: 10epoch:train:645-736batch: iter_time=6.418e-04, forward_time=0.279, loss_att=281.847, acc=0.758, loss=281.847, backward_time=0.392, grad_norm=101.061, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=4.314e-04, train_time=3.494 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:31:48,850 (trainer:732) INFO: 10epoch:train:737-828batch: iter_time=4.891e-04, forward_time=0.275, loss_att=271.362, acc=0.758, loss=271.362, backward_time=0.389, grad_norm=96.725, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=4.337e-04, train_time=3.472 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:33:09,360 (trainer:732) INFO: 10epoch:train:829-920batch: iter_time=6.211e-04, forward_time=0.280, loss_att=277.964, acc=0.759, loss=277.964, backward_time=0.390, grad_norm=105.917, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=4.360e-04, train_time=3.500 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:34:29,533 (trainer:732) INFO: 10epoch:train:921-1012batch: iter_time=5.349e-04, forward_time=0.274, loss_att=278.526, acc=0.761, loss=278.526, backward_time=0.393, grad_norm=100.523, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=4.383e-04, train_time=3.485 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:35:49,232 (trainer:732) INFO: 10epoch:train:1013-1104batch: iter_time=5.305e-04, forward_time=0.276, loss_att=275.289, acc=0.760, loss=275.289, backward_time=0.394, grad_norm=106.773, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=4.406e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:37:09,871 (trainer:732) INFO: 10epoch:train:1105-1196batch: iter_time=5.077e-04, forward_time=0.280, loss_att=277.765, acc=0.765, loss=277.765, backward_time=0.392, grad_norm=102.651, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=4.429e-04, train_time=3.505 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:38:28,982 (trainer:732) INFO: 10epoch:train:1197-1288batch: iter_time=4.972e-04, forward_time=0.271, loss_att=265.109, acc=0.763, loss=265.109, backward_time=0.388, grad_norm=89.812, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=4.452e-04, train_time=3.439 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:39:48,893 (trainer:732) INFO: 10epoch:train:1289-1380batch: iter_time=5.477e-04, forward_time=0.273, loss_att=286.266, acc=0.759, loss=286.266, backward_time=0.394, grad_norm=110.542, clip=100.000, loss_scale=1.000, optim_step_time=0.128, optim0_lr0=4.475e-04, train_time=3.474 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:41:09,493 (trainer:732) INFO: 10epoch:train:1381-1472batch: iter_time=5.188e-04, forward_time=0.278, loss_att=276.051, acc=0.762, loss=276.051, backward_time=0.393, grad_norm=125.569, clip=100.000, loss_scale=1.000, optim_step_time=0.132, optim0_lr0=4.498e-04, train_time=3.503 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:42:30,935 (trainer:732) INFO: 10epoch:train:1473-1564batch: iter_time=6.661e-04, forward_time=0.279, loss_att=279.916, acc=0.762, loss=279.916, backward_time=0.396, grad_norm=102.298, clip=100.000, loss_scale=1.000, optim_step_time=0.146, optim0_lr0=4.521e-04, train_time=3.540 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:43:50,922 (trainer:732) INFO: 10epoch:train:1565-1656batch: iter_time=5.757e-04, forward_time=0.275, loss_att=272.478, acc=0.763, loss=272.478, backward_time=0.393, grad_norm=107.021, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=4.544e-04, train_time=3.477 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:45:10,541 (trainer:732) INFO: 10epoch:train:1657-1748batch: iter_time=5.098e-04, forward_time=0.276, loss_att=267.257, acc=0.763, loss=267.257, backward_time=0.387, grad_norm=91.334, clip=100.000, loss_scale=1.000, optim_step_time=0.127, optim0_lr0=4.567e-04, train_time=3.461 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:46:30,214 (trainer:732) INFO: 10epoch:train:1749-1840batch: iter_time=3.994e-04, forward_time=0.276, loss_att=273.689, acc=0.761, loss=273.689, backward_time=0.392, grad_norm=103.989, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=4.590e-04, train_time=3.462 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:58:55,555 (trainer:338) INFO: 10epoch results: [train] iter_time=0.001, forward_time=0.277, loss_att=275.996, acc=0.760, loss=275.996, backward_time=0.392, grad_norm=101.659, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=4.371e-04, train_time=4.130, time=31 minutes and 47.48 seconds, total_count=18430, gpu_max_cached_mem_GB=29.846, [valid] loss_att=273.109, acc=0.760, cer=0.300, wer=0.619, loss=273.109, time=5 minutes and 10.68 seconds, total_count=190, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 7.35 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:59:05,468 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 03:59:05,475 (trainer:272) INFO: 11/60epoch started. Estimated time to finish: 1 day, 11 hours and 1 minute +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:05:16,698 (trainer:732) INFO: 11epoch:train:1-92batch: iter_time=0.017, forward_time=0.281, loss_att=270.324, acc=0.768, loss=270.324, backward_time=0.394, grad_norm=101.782, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=4.613e-04, train_time=16.138 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:06:36,749 (trainer:732) INFO: 11epoch:train:93-184batch: iter_time=5.404e-04, forward_time=0.275, loss_att=267.962, acc=0.770, loss=267.962, backward_time=0.392, grad_norm=112.826, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=4.636e-04, train_time=3.480 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:07:56,432 (trainer:732) INFO: 11epoch:train:185-276batch: iter_time=6.896e-04, forward_time=0.276, loss_att=269.554, acc=0.765, loss=269.554, backward_time=0.391, grad_norm=101.243, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=4.659e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:09:17,033 (trainer:732) INFO: 11epoch:train:277-368batch: iter_time=5.940e-04, forward_time=0.275, loss_att=271.059, acc=0.768, loss=271.059, backward_time=0.389, grad_norm=96.124, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=4.682e-04, train_time=3.504 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:10:35,845 (trainer:732) INFO: 11epoch:train:369-460batch: iter_time=4.876e-04, forward_time=0.272, loss_att=267.246, acc=0.763, loss=267.246, backward_time=0.387, grad_norm=87.894, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=4.705e-04, train_time=3.426 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:11:56,484 (trainer:732) INFO: 11epoch:train:461-552batch: iter_time=4.857e-04, forward_time=0.275, loss_att=276.777, acc=0.769, loss=276.777, backward_time=0.396, grad_norm=104.990, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=4.728e-04, train_time=3.505 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:13:17,225 (trainer:732) INFO: 11epoch:train:553-644batch: iter_time=5.427e-04, forward_time=0.280, loss_att=269.279, acc=0.770, loss=269.279, backward_time=0.398, grad_norm=97.852, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=4.751e-04, train_time=3.510 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:14:36,746 (trainer:732) INFO: 11epoch:train:645-736batch: iter_time=5.329e-04, forward_time=0.273, loss_att=266.840, acc=0.769, loss=266.840, backward_time=0.393, grad_norm=98.708, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=4.774e-04, train_time=3.457 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:15:55,749 (trainer:732) INFO: 11epoch:train:737-828batch: iter_time=4.889e-04, forward_time=0.276, loss_att=268.613, acc=0.764, loss=268.613, backward_time=0.387, grad_norm=108.627, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=4.797e-04, train_time=3.434 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:17:15,370 (trainer:732) INFO: 11epoch:train:829-920batch: iter_time=4.905e-04, forward_time=0.275, loss_att=259.892, acc=0.765, loss=259.892, backward_time=0.387, grad_norm=102.241, clip=100.000, loss_scale=1.000, optim_step_time=0.131, optim0_lr0=4.820e-04, train_time=3.461 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:18:35,639 (trainer:732) INFO: 11epoch:train:921-1012batch: iter_time=4.981e-04, forward_time=0.276, loss_att=258.122, acc=0.768, loss=258.122, backward_time=0.393, grad_norm=113.938, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=4.843e-04, train_time=3.489 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:19:55,056 (trainer:732) INFO: 11epoch:train:1013-1104batch: iter_time=5.103e-04, forward_time=0.273, loss_att=256.512, acc=0.764, loss=256.512, backward_time=0.387, grad_norm=94.866, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=4.866e-04, train_time=3.452 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:21:15,597 (trainer:732) INFO: 11epoch:train:1105-1196batch: iter_time=4.320e-04, forward_time=0.275, loss_att=264.447, acc=0.770, loss=264.447, backward_time=0.393, grad_norm=93.447, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=4.889e-04, train_time=3.501 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:22:35,331 (trainer:732) INFO: 11epoch:train:1197-1288batch: iter_time=6.657e-04, forward_time=0.275, loss_att=269.868, acc=0.770, loss=269.868, backward_time=0.391, grad_norm=97.373, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=4.912e-04, train_time=3.467 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:23:55,680 (trainer:732) INFO: 11epoch:train:1289-1380batch: iter_time=6.885e-04, forward_time=0.279, loss_att=273.001, acc=0.768, loss=273.001, backward_time=0.393, grad_norm=116.602, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=4.935e-04, train_time=3.491 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:25:15,154 (trainer:732) INFO: 11epoch:train:1381-1472batch: iter_time=6.303e-04, forward_time=0.274, loss_att=258.522, acc=0.771, loss=258.522, backward_time=0.390, grad_norm=101.714, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=4.958e-04, train_time=3.456 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:26:35,751 (trainer:732) INFO: 11epoch:train:1473-1564batch: iter_time=7.101e-04, forward_time=0.285, loss_att=258.820, acc=0.776, loss=258.820, backward_time=0.393, grad_norm=99.244, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=4.981e-04, train_time=3.503 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:27:55,410 (trainer:732) INFO: 11epoch:train:1565-1656batch: iter_time=8.021e-04, forward_time=0.274, loss_att=260.582, acc=0.772, loss=260.582, backward_time=0.388, grad_norm=98.046, clip=100.000, loss_scale=1.000, optim_step_time=0.128, optim0_lr0=5.004e-04, train_time=3.463 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:29:15,760 (trainer:732) INFO: 11epoch:train:1657-1748batch: iter_time=5.630e-04, forward_time=0.277, loss_att=269.510, acc=0.768, loss=269.510, backward_time=0.392, grad_norm=107.470, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=5.027e-04, train_time=3.493 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:30:35,036 (trainer:732) INFO: 11epoch:train:1749-1840batch: iter_time=3.908e-04, forward_time=0.270, loss_att=257.205, acc=0.769, loss=257.205, backward_time=0.388, grad_norm=106.177, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=5.050e-04, train_time=3.446 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:42:50,635 (trainer:338) INFO: 11epoch results: [train] iter_time=0.001, forward_time=0.276, loss_att=265.506, acc=0.768, loss=265.506, backward_time=0.391, grad_norm=102.058, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=4.831e-04, train_time=4.107, time=31 minutes and 36.49 seconds, total_count=20273, gpu_max_cached_mem_GB=29.846, [valid] loss_att=268.456, acc=0.765, cer=0.294, wer=0.612, loss=268.456, time=5 minutes and 12.49 seconds, total_count=209, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 56.18 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:42:59,922 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:42:59,937 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/1epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:42:59,938 (trainer:272) INFO: 12/60epoch started. Estimated time to finish: 1 day, 10 hours and 27 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:49:15,192 (trainer:732) INFO: 12epoch:train:1-92batch: iter_time=0.024, forward_time=0.280, loss_att=255.799, acc=0.776, loss=255.799, backward_time=0.394, grad_norm=98.824, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=5.073e-04, train_time=16.314 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:50:34,577 (trainer:732) INFO: 12epoch:train:93-184batch: iter_time=6.787e-04, forward_time=0.274, loss_att=255.462, acc=0.770, loss=255.462, backward_time=0.392, grad_norm=98.628, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=5.096e-04, train_time=3.451 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:51:53,269 (trainer:732) INFO: 12epoch:train:185-276batch: iter_time=4.631e-04, forward_time=0.269, loss_att=253.504, acc=0.774, loss=253.504, backward_time=0.390, grad_norm=98.136, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=5.119e-04, train_time=3.421 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:53:11,566 (trainer:732) INFO: 12epoch:train:277-368batch: iter_time=4.600e-04, forward_time=0.264, loss_att=253.994, acc=0.779, loss=253.994, backward_time=0.389, grad_norm=99.172, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=5.142e-04, train_time=3.403 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:54:29,139 (trainer:732) INFO: 12epoch:train:369-460batch: iter_time=4.331e-04, forward_time=0.263, loss_att=259.303, acc=0.779, loss=259.303, backward_time=0.389, grad_norm=100.710, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=5.165e-04, train_time=3.372 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:55:46,926 (trainer:732) INFO: 12epoch:train:461-552batch: iter_time=4.666e-04, forward_time=0.262, loss_att=248.178, acc=0.774, loss=248.178, backward_time=0.385, grad_norm=107.747, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=5.188e-04, train_time=3.381 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:57:05,319 (trainer:732) INFO: 12epoch:train:553-644batch: iter_time=4.294e-04, forward_time=0.267, loss_att=259.788, acc=0.774, loss=259.788, backward_time=0.392, grad_norm=109.945, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=5.211e-04, train_time=3.408 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:58:22,183 (trainer:732) INFO: 12epoch:train:645-736batch: iter_time=4.444e-04, forward_time=0.262, loss_att=245.258, acc=0.775, loss=245.258, backward_time=0.387, grad_norm=98.760, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=5.234e-04, train_time=3.341 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 04:59:40,269 (trainer:732) INFO: 12epoch:train:737-828batch: iter_time=4.532e-04, forward_time=0.266, loss_att=265.454, acc=0.776, loss=265.454, backward_time=0.392, grad_norm=117.251, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=5.257e-04, train_time=3.394 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:01:00,028 (trainer:732) INFO: 12epoch:train:829-920batch: iter_time=4.692e-04, forward_time=0.274, loss_att=259.339, acc=0.777, loss=259.339, backward_time=0.392, grad_norm=102.976, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=5.280e-04, train_time=3.467 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:02:18,865 (trainer:732) INFO: 12epoch:train:921-1012batch: iter_time=5.218e-04, forward_time=0.270, loss_att=257.256, acc=0.775, loss=257.256, backward_time=0.387, grad_norm=96.923, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=5.303e-04, train_time=3.427 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:03:39,049 (trainer:732) INFO: 12epoch:train:1013-1104batch: iter_time=4.994e-04, forward_time=0.270, loss_att=253.138, acc=0.776, loss=253.138, backward_time=0.390, grad_norm=95.358, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=5.326e-04, train_time=3.486 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:04:57,985 (trainer:732) INFO: 12epoch:train:1105-1196batch: iter_time=4.769e-04, forward_time=0.271, loss_att=250.822, acc=0.777, loss=250.822, backward_time=0.388, grad_norm=98.684, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=5.349e-04, train_time=3.431 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:06:17,838 (trainer:732) INFO: 12epoch:train:1197-1288batch: iter_time=5.444e-04, forward_time=0.271, loss_att=257.306, acc=0.782, loss=257.306, backward_time=0.393, grad_norm=111.430, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=5.372e-04, train_time=3.471 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:07:38,607 (trainer:732) INFO: 12epoch:train:1289-1380batch: iter_time=6.385e-04, forward_time=0.280, loss_att=256.925, acc=0.776, loss=256.925, backward_time=0.392, grad_norm=99.068, clip=100.000, loss_scale=1.000, optim_step_time=0.129, optim0_lr0=5.395e-04, train_time=3.511 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:08:58,722 (trainer:732) INFO: 12epoch:train:1381-1472batch: iter_time=4.058e-04, forward_time=0.272, loss_att=261.189, acc=0.780, loss=261.189, backward_time=0.394, grad_norm=102.922, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=5.418e-04, train_time=3.482 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:10:17,059 (trainer:732) INFO: 12epoch:train:1473-1564batch: iter_time=4.586e-04, forward_time=0.270, loss_att=257.024, acc=0.775, loss=257.024, backward_time=0.387, grad_norm=100.372, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=5.441e-04, train_time=3.405 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:11:36,099 (trainer:732) INFO: 12epoch:train:1565-1656batch: iter_time=5.269e-04, forward_time=0.270, loss_att=258.665, acc=0.777, loss=258.665, backward_time=0.392, grad_norm=96.056, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=5.464e-04, train_time=3.436 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:12:54,106 (trainer:732) INFO: 12epoch:train:1657-1748batch: iter_time=4.804e-04, forward_time=0.266, loss_att=252.628, acc=0.777, loss=252.628, backward_time=0.388, grad_norm=93.954, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=5.487e-04, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:14:12,116 (trainer:732) INFO: 12epoch:train:1749-1840batch: iter_time=4.945e-04, forward_time=0.265, loss_att=263.671, acc=0.777, loss=263.671, backward_time=0.390, grad_norm=99.877, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=5.510e-04, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:25:49,717 (trainer:338) INFO: 12epoch results: [train] iter_time=0.002, forward_time=0.269, loss_att=256.135, acc=0.776, loss=256.135, backward_time=0.390, grad_norm=101.340, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=5.291e-04, train_time=4.069, time=31 minutes and 18.19 seconds, total_count=22116, gpu_max_cached_mem_GB=29.846, [valid] loss_att=264.008, acc=0.769, cer=0.289, wer=0.606, loss=264.008, time=5 minutes and 0.39 seconds, total_count=228, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 31.2 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:25:57,772 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:25:57,782 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/2epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:25:57,783 (trainer:272) INFO: 13/60epoch started. Estimated time to finish: 1 day, 9 hours and 48 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:30:46,561 (trainer:732) INFO: 13epoch:train:1-92batch: iter_time=0.010, forward_time=0.274, loss_att=257.666, acc=0.786, loss=257.666, backward_time=0.394, grad_norm=108.867, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=5.533e-04, train_time=12.554 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:32:03,812 (trainer:732) INFO: 13epoch:train:93-184batch: iter_time=4.863e-04, forward_time=0.261, loss_att=244.478, acc=0.784, loss=244.478, backward_time=0.385, grad_norm=105.957, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=5.556e-04, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:33:22,781 (trainer:732) INFO: 13epoch:train:185-276batch: iter_time=4.246e-04, forward_time=0.267, loss_att=262.048, acc=0.781, loss=262.048, backward_time=0.394, grad_norm=98.495, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=5.579e-04, train_time=3.433 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:34:40,261 (trainer:732) INFO: 13epoch:train:277-368batch: iter_time=4.491e-04, forward_time=0.263, loss_att=243.562, acc=0.784, loss=243.562, backward_time=0.388, grad_norm=95.780, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=5.602e-04, train_time=3.368 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:35:58,163 (trainer:732) INFO: 13epoch:train:369-460batch: iter_time=4.923e-04, forward_time=0.266, loss_att=245.198, acc=0.781, loss=245.198, backward_time=0.390, grad_norm=94.401, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=5.625e-04, train_time=3.386 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:37:15,406 (trainer:732) INFO: 13epoch:train:461-552batch: iter_time=4.444e-04, forward_time=0.262, loss_att=256.554, acc=0.780, loss=256.554, backward_time=0.386, grad_norm=95.974, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=5.648e-04, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:38:32,996 (trainer:732) INFO: 13epoch:train:553-644batch: iter_time=4.565e-04, forward_time=0.264, loss_att=246.835, acc=0.780, loss=246.835, backward_time=0.386, grad_norm=100.699, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=5.671e-04, train_time=3.373 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:39:50,932 (trainer:732) INFO: 13epoch:train:645-736batch: iter_time=4.780e-04, forward_time=0.264, loss_att=250.409, acc=0.787, loss=250.409, backward_time=0.390, grad_norm=109.321, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=5.694e-04, train_time=3.388 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:41:08,904 (trainer:732) INFO: 13epoch:train:737-828batch: iter_time=6.293e-04, forward_time=0.265, loss_att=245.140, acc=0.780, loss=245.140, backward_time=0.390, grad_norm=97.878, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=5.717e-04, train_time=3.389 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:42:26,856 (trainer:732) INFO: 13epoch:train:829-920batch: iter_time=4.514e-04, forward_time=0.265, loss_att=254.455, acc=0.781, loss=254.455, backward_time=0.389, grad_norm=112.750, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=5.740e-04, train_time=3.388 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:43:44,829 (trainer:732) INFO: 13epoch:train:921-1012batch: iter_time=4.864e-04, forward_time=0.266, loss_att=239.725, acc=0.787, loss=239.725, backward_time=0.390, grad_norm=110.904, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=5.763e-04, train_time=3.389 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:45:02,483 (trainer:732) INFO: 13epoch:train:1013-1104batch: iter_time=4.147e-04, forward_time=0.264, loss_att=254.094, acc=0.788, loss=254.094, backward_time=0.389, grad_norm=93.848, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=5.786e-04, train_time=3.376 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:46:20,602 (trainer:732) INFO: 13epoch:train:1105-1196batch: iter_time=3.933e-04, forward_time=0.265, loss_att=248.658, acc=0.780, loss=248.658, backward_time=0.390, grad_norm=103.811, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=5.809e-04, train_time=3.396 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:47:38,026 (trainer:732) INFO: 13epoch:train:1197-1288batch: iter_time=4.227e-04, forward_time=0.262, loss_att=238.398, acc=0.781, loss=238.398, backward_time=0.386, grad_norm=97.058, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=5.832e-04, train_time=3.366 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:48:55,361 (trainer:732) INFO: 13epoch:train:1289-1380batch: iter_time=4.372e-04, forward_time=0.263, loss_att=243.326, acc=0.777, loss=243.326, backward_time=0.387, grad_norm=99.723, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=5.855e-04, train_time=3.362 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:50:13,328 (trainer:732) INFO: 13epoch:train:1381-1472batch: iter_time=4.636e-04, forward_time=0.264, loss_att=252.033, acc=0.784, loss=252.033, backward_time=0.391, grad_norm=109.193, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=5.878e-04, train_time=3.389 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:51:30,055 (trainer:732) INFO: 13epoch:train:1473-1564batch: iter_time=4.699e-04, forward_time=0.261, loss_att=244.595, acc=0.784, loss=244.595, backward_time=0.384, grad_norm=93.501, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=5.901e-04, train_time=3.335 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:52:47,174 (trainer:732) INFO: 13epoch:train:1565-1656batch: iter_time=4.496e-04, forward_time=0.264, loss_att=247.205, acc=0.788, loss=247.205, backward_time=0.388, grad_norm=101.634, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=5.924e-04, train_time=3.352 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:54:04,298 (trainer:732) INFO: 13epoch:train:1657-1748batch: iter_time=3.930e-04, forward_time=0.262, loss_att=240.879, acc=0.784, loss=240.879, backward_time=0.386, grad_norm=102.654, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=5.947e-04, train_time=3.352 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 05:55:21,532 (trainer:732) INFO: 13epoch:train:1749-1840batch: iter_time=3.582e-04, forward_time=0.262, loss_att=248.073, acc=0.783, loss=248.073, backward_time=0.386, grad_norm=102.190, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=5.970e-04, train_time=3.357 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:07:18,721 (trainer:338) INFO: 13epoch results: [train] iter_time=9.227e-04, forward_time=0.264, loss_att=248.088, acc=0.783, loss=248.088, backward_time=0.389, grad_norm=101.732, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=5.751e-04, train_time=3.833, time=29 minutes and 29.5 seconds, total_count=23959, gpu_max_cached_mem_GB=29.846, [valid] loss_att=261.861, acc=0.772, cer=0.287, wer=0.601, loss=261.861, time=4 minutes and 57.43 seconds, total_count=247, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 54.01 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:07:28,069 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:07:28,087 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/3epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:07:28,089 (trainer:272) INFO: 14/60epoch started. Estimated time to finish: 1 day, 9 hours and 3 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:13:29,908 (trainer:732) INFO: 14epoch:train:1-92batch: iter_time=0.008, forward_time=0.279, loss_att=233.820, acc=0.788, loss=233.820, backward_time=0.389, grad_norm=100.264, clip=100.000, loss_scale=1.000, optim_step_time=0.126, optim0_lr0=5.993e-04, train_time=15.729 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:14:49,985 (trainer:732) INFO: 14epoch:train:93-184batch: iter_time=6.642e-04, forward_time=0.277, loss_att=245.325, acc=0.789, loss=245.325, backward_time=0.393, grad_norm=105.426, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=6.016e-04, train_time=3.481 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:16:10,914 (trainer:732) INFO: 14epoch:train:185-276batch: iter_time=4.998e-04, forward_time=0.282, loss_att=237.939, acc=0.789, loss=237.939, backward_time=0.396, grad_norm=102.994, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=6.039e-04, train_time=3.518 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:17:30,910 (trainer:732) INFO: 14epoch:train:277-368batch: iter_time=5.661e-04, forward_time=0.277, loss_att=240.119, acc=0.790, loss=240.119, backward_time=0.391, grad_norm=93.650, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=6.062e-04, train_time=3.477 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:18:50,331 (trainer:732) INFO: 14epoch:train:369-460batch: iter_time=6.340e-04, forward_time=0.274, loss_att=228.513, acc=0.786, loss=228.513, backward_time=0.386, grad_norm=90.689, clip=100.000, loss_scale=1.000, optim_step_time=0.134, optim0_lr0=6.085e-04, train_time=3.452 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:20:08,345 (trainer:732) INFO: 14epoch:train:461-552batch: iter_time=4.634e-04, forward_time=0.269, loss_att=242.292, acc=0.786, loss=242.292, backward_time=0.386, grad_norm=97.196, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=6.108e-04, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:21:27,076 (trainer:732) INFO: 14epoch:train:553-644batch: iter_time=6.955e-04, forward_time=0.272, loss_att=235.322, acc=0.789, loss=235.322, backward_time=0.388, grad_norm=94.273, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=6.131e-04, train_time=3.422 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:22:46,204 (trainer:732) INFO: 14epoch:train:645-736batch: iter_time=4.962e-04, forward_time=0.269, loss_att=235.632, acc=0.790, loss=235.632, backward_time=0.387, grad_norm=89.949, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=6.154e-04, train_time=3.440 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:24:06,361 (trainer:732) INFO: 14epoch:train:737-828batch: iter_time=4.761e-04, forward_time=0.276, loss_att=242.630, acc=0.793, loss=242.630, backward_time=0.392, grad_norm=97.817, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=6.177e-04, train_time=3.484 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:25:25,692 (trainer:732) INFO: 14epoch:train:829-920batch: iter_time=6.105e-04, forward_time=0.274, loss_att=238.588, acc=0.790, loss=238.588, backward_time=0.388, grad_norm=104.524, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=6.200e-04, train_time=3.448 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:26:45,061 (trainer:732) INFO: 14epoch:train:921-1012batch: iter_time=5.019e-04, forward_time=0.273, loss_att=242.625, acc=0.788, loss=242.625, backward_time=0.387, grad_norm=105.339, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=6.223e-04, train_time=3.450 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:28:05,894 (trainer:732) INFO: 14epoch:train:1013-1104batch: iter_time=6.208e-04, forward_time=0.278, loss_att=250.995, acc=0.791, loss=250.995, backward_time=0.396, grad_norm=100.408, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=6.246e-04, train_time=3.514 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:29:28,072 (trainer:732) INFO: 14epoch:train:1105-1196batch: iter_time=5.324e-04, forward_time=0.281, loss_att=253.574, acc=0.791, loss=253.574, backward_time=0.402, grad_norm=113.643, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=6.269e-04, train_time=3.572 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:30:48,916 (trainer:732) INFO: 14epoch:train:1197-1288batch: iter_time=5.058e-04, forward_time=0.278, loss_att=251.229, acc=0.789, loss=251.229, backward_time=0.397, grad_norm=103.521, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=6.292e-04, train_time=3.514 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:32:10,196 (trainer:732) INFO: 14epoch:train:1289-1380batch: iter_time=4.991e-04, forward_time=0.280, loss_att=241.663, acc=0.790, loss=241.663, backward_time=0.392, grad_norm=92.799, clip=100.000, loss_scale=1.000, optim_step_time=0.141, optim0_lr0=6.315e-04, train_time=3.533 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:33:30,402 (trainer:732) INFO: 14epoch:train:1381-1472batch: iter_time=6.489e-04, forward_time=0.276, loss_att=241.747, acc=0.786, loss=241.747, backward_time=0.388, grad_norm=97.479, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=6.338e-04, train_time=3.486 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:34:49,824 (trainer:732) INFO: 14epoch:train:1473-1564batch: iter_time=5.415e-04, forward_time=0.275, loss_att=246.203, acc=0.793, loss=246.203, backward_time=0.393, grad_norm=94.624, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=6.361e-04, train_time=3.452 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:36:10,990 (trainer:732) INFO: 14epoch:train:1565-1656batch: iter_time=5.496e-04, forward_time=0.280, loss_att=244.422, acc=0.792, loss=244.422, backward_time=0.400, grad_norm=118.285, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=6.384e-04, train_time=3.528 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:37:30,490 (trainer:732) INFO: 14epoch:train:1657-1748batch: iter_time=5.333e-04, forward_time=0.275, loss_att=227.197, acc=0.789, loss=227.197, backward_time=0.388, grad_norm=90.856, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=6.407e-04, train_time=3.456 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:38:50,506 (trainer:732) INFO: 14epoch:train:1749-1840batch: iter_time=4.928e-04, forward_time=0.275, loss_att=240.128, acc=0.785, loss=240.128, backward_time=0.391, grad_norm=99.027, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=6.430e-04, train_time=3.478 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:51:09,745 (trainer:338) INFO: 14epoch results: [train] iter_time=9.465e-04, forward_time=0.276, loss_att=240.791, acc=0.789, loss=240.791, backward_time=0.391, grad_norm=99.638, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=6.212e-04, train_time=4.091, time=31 minutes and 28.75 seconds, total_count=25802, gpu_max_cached_mem_GB=29.846, [valid] loss_att=256.252, acc=0.776, cer=0.281, wer=0.592, loss=256.252, time=5 minutes and 9.99 seconds, total_count=266, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 2.9 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:51:19,348 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:51:19,369 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/4epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:51:19,370 (trainer:272) INFO: 15/60epoch started. Estimated time to finish: 1 day, 8 hours and 26 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:57:35,724 (trainer:732) INFO: 15epoch:train:1-92batch: iter_time=0.017, forward_time=0.282, loss_att=227.728, acc=0.793, loss=227.728, backward_time=0.388, grad_norm=92.389, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=6.453e-04, train_time=16.362 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 06:58:55,347 (trainer:732) INFO: 15epoch:train:93-184batch: iter_time=5.269e-04, forward_time=0.274, loss_att=235.692, acc=0.796, loss=235.692, backward_time=0.390, grad_norm=97.825, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=6.476e-04, train_time=3.461 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:00:15,037 (trainer:732) INFO: 15epoch:train:185-276batch: iter_time=5.141e-04, forward_time=0.277, loss_att=231.906, acc=0.793, loss=231.906, backward_time=0.388, grad_norm=99.770, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=6.499e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:01:34,438 (trainer:732) INFO: 15epoch:train:277-368batch: iter_time=5.038e-04, forward_time=0.273, loss_att=236.885, acc=0.795, loss=236.885, backward_time=0.391, grad_norm=98.531, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=6.522e-04, train_time=3.451 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:02:55,171 (trainer:732) INFO: 15epoch:train:369-460batch: iter_time=4.621e-04, forward_time=0.277, loss_att=229.085, acc=0.795, loss=229.085, backward_time=0.392, grad_norm=108.212, clip=100.000, loss_scale=1.000, optim_step_time=0.136, optim0_lr0=6.545e-04, train_time=3.509 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:04:14,540 (trainer:732) INFO: 15epoch:train:461-552batch: iter_time=5.256e-04, forward_time=0.276, loss_att=228.230, acc=0.792, loss=228.230, backward_time=0.388, grad_norm=93.524, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=6.568e-04, train_time=3.450 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:05:34,102 (trainer:732) INFO: 15epoch:train:553-644batch: iter_time=5.467e-04, forward_time=0.276, loss_att=233.765, acc=0.796, loss=233.765, backward_time=0.390, grad_norm=98.862, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=6.591e-04, train_time=3.458 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:06:55,246 (trainer:732) INFO: 15epoch:train:645-736batch: iter_time=6.168e-04, forward_time=0.282, loss_att=240.397, acc=0.795, loss=240.397, backward_time=0.398, grad_norm=101.405, clip=100.000, loss_scale=1.000, optim_step_time=0.135, optim0_lr0=6.614e-04, train_time=3.526 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:08:16,084 (trainer:732) INFO: 15epoch:train:737-828batch: iter_time=5.919e-04, forward_time=0.280, loss_att=241.128, acc=0.796, loss=241.128, backward_time=0.392, grad_norm=104.080, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=6.637e-04, train_time=3.515 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:09:36,055 (trainer:732) INFO: 15epoch:train:829-920batch: iter_time=6.409e-04, forward_time=0.276, loss_att=234.757, acc=0.797, loss=234.757, backward_time=0.390, grad_norm=101.970, clip=100.000, loss_scale=1.000, optim_step_time=0.129, optim0_lr0=6.660e-04, train_time=3.476 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:10:55,618 (trainer:732) INFO: 15epoch:train:921-1012batch: iter_time=9.029e-04, forward_time=0.275, loss_att=231.904, acc=0.793, loss=231.904, backward_time=0.390, grad_norm=91.765, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=6.683e-04, train_time=3.458 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:12:15,298 (trainer:732) INFO: 15epoch:train:1013-1104batch: iter_time=6.084e-04, forward_time=0.274, loss_att=225.482, acc=0.795, loss=225.482, backward_time=0.389, grad_norm=95.185, clip=100.000, loss_scale=1.000, optim_step_time=0.127, optim0_lr0=6.706e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:13:35,802 (trainer:732) INFO: 15epoch:train:1105-1196batch: iter_time=5.160e-04, forward_time=0.279, loss_att=237.503, acc=0.792, loss=237.503, backward_time=0.398, grad_norm=106.073, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=6.729e-04, train_time=3.499 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:14:55,856 (trainer:732) INFO: 15epoch:train:1197-1288batch: iter_time=5.182e-04, forward_time=0.277, loss_att=235.912, acc=0.793, loss=235.912, backward_time=0.391, grad_norm=107.872, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=6.752e-04, train_time=3.480 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:16:16,719 (trainer:732) INFO: 15epoch:train:1289-1380batch: iter_time=6.616e-04, forward_time=0.280, loss_att=226.274, acc=0.792, loss=226.274, backward_time=0.388, grad_norm=105.744, clip=100.000, loss_scale=1.000, optim_step_time=0.130, optim0_lr0=6.775e-04, train_time=3.515 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:17:37,210 (trainer:732) INFO: 15epoch:train:1381-1472batch: iter_time=5.737e-04, forward_time=0.279, loss_att=236.788, acc=0.796, loss=236.788, backward_time=0.393, grad_norm=101.265, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=6.798e-04, train_time=3.499 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:18:58,325 (trainer:732) INFO: 15epoch:train:1473-1564batch: iter_time=6.156e-04, forward_time=0.278, loss_att=240.573, acc=0.794, loss=240.573, backward_time=0.396, grad_norm=107.943, clip=100.000, loss_scale=1.000, optim_step_time=0.127, optim0_lr0=6.821e-04, train_time=3.526 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:20:19,700 (trainer:732) INFO: 15epoch:train:1565-1656batch: iter_time=4.620e-04, forward_time=0.282, loss_att=237.580, acc=0.796, loss=237.580, backward_time=0.390, grad_norm=100.791, clip=100.000, loss_scale=1.000, optim_step_time=0.136, optim0_lr0=6.844e-04, train_time=3.537 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:21:39,577 (trainer:732) INFO: 15epoch:train:1657-1748batch: iter_time=4.300e-04, forward_time=0.273, loss_att=238.528, acc=0.798, loss=238.528, backward_time=0.394, grad_norm=100.471, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=6.867e-04, train_time=3.472 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:22:59,969 (trainer:732) INFO: 15epoch:train:1749-1840batch: iter_time=5.127e-04, forward_time=0.277, loss_att=237.559, acc=0.800, loss=237.559, backward_time=0.398, grad_norm=111.796, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=6.890e-04, train_time=3.494 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:35:21,535 (trainer:338) INFO: 15epoch results: [train] iter_time=0.001, forward_time=0.277, loss_att=234.184, acc=0.795, loss=234.184, backward_time=0.392, grad_norm=101.274, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=6.671e-04, train_time=4.131, time=31 minutes and 46.5 seconds, total_count=27645, gpu_max_cached_mem_GB=29.846, [valid] loss_att=254.581, acc=0.779, cer=0.280, wer=0.590, loss=254.581, time=5 minutes and 10.83 seconds, total_count=285, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 4.83 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:35:31,623 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:35:31,648 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/5epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:35:31,649 (trainer:272) INFO: 16/60epoch started. Estimated time to finish: 1 day, 7 hours and 50 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:41:49,193 (trainer:732) INFO: 16epoch:train:1-92batch: iter_time=0.009, forward_time=0.283, loss_att=225.311, acc=0.801, loss=225.311, backward_time=0.390, grad_norm=100.156, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=6.913e-04, train_time=16.413 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:43:09,032 (trainer:732) INFO: 16epoch:train:93-184batch: iter_time=5.037e-04, forward_time=0.278, loss_att=227.458, acc=0.800, loss=227.458, backward_time=0.389, grad_norm=100.964, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=6.936e-04, train_time=3.471 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:44:29,442 (trainer:732) INFO: 16epoch:train:185-276batch: iter_time=4.647e-04, forward_time=0.277, loss_att=227.536, acc=0.800, loss=227.536, backward_time=0.391, grad_norm=98.828, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=6.959e-04, train_time=3.495 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:45:49,841 (trainer:732) INFO: 16epoch:train:277-368batch: iter_time=6.418e-04, forward_time=0.278, loss_att=229.443, acc=0.802, loss=229.443, backward_time=0.394, grad_norm=111.310, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=6.982e-04, train_time=3.495 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:47:09,953 (trainer:732) INFO: 16epoch:train:369-460batch: iter_time=5.035e-04, forward_time=0.275, loss_att=226.034, acc=0.802, loss=226.034, backward_time=0.386, grad_norm=94.429, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=7.005e-04, train_time=3.482 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:48:29,246 (trainer:732) INFO: 16epoch:train:461-552batch: iter_time=5.173e-04, forward_time=0.274, loss_att=225.186, acc=0.797, loss=225.186, backward_time=0.390, grad_norm=94.985, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=7.028e-04, train_time=3.447 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:49:48,273 (trainer:732) INFO: 16epoch:train:553-644batch: iter_time=4.564e-04, forward_time=0.269, loss_att=217.519, acc=0.798, loss=217.519, backward_time=0.385, grad_norm=92.646, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=7.051e-04, train_time=3.435 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:51:10,543 (trainer:732) INFO: 16epoch:train:645-736batch: iter_time=5.657e-04, forward_time=0.287, loss_att=236.136, acc=0.804, loss=236.136, backward_time=0.396, grad_norm=97.270, clip=100.000, loss_scale=1.000, optim_step_time=0.134, optim0_lr0=7.074e-04, train_time=3.576 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:52:29,854 (trainer:732) INFO: 16epoch:train:737-828batch: iter_time=5.558e-04, forward_time=0.269, loss_att=225.728, acc=0.798, loss=225.728, backward_time=0.387, grad_norm=97.771, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=7.097e-04, train_time=3.448 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:53:47,903 (trainer:732) INFO: 16epoch:train:829-920batch: iter_time=4.845e-04, forward_time=0.265, loss_att=227.601, acc=0.800, loss=227.601, backward_time=0.388, grad_norm=103.588, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=7.120e-04, train_time=3.393 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:55:05,969 (trainer:732) INFO: 16epoch:train:921-1012batch: iter_time=4.467e-04, forward_time=0.264, loss_att=229.063, acc=0.803, loss=229.063, backward_time=0.390, grad_norm=100.122, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=7.143e-04, train_time=3.393 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:56:23,271 (trainer:732) INFO: 16epoch:train:1013-1104batch: iter_time=4.476e-04, forward_time=0.264, loss_att=229.241, acc=0.797, loss=229.241, backward_time=0.389, grad_norm=90.039, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=7.166e-04, train_time=3.360 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:57:40,754 (trainer:732) INFO: 16epoch:train:1105-1196batch: iter_time=4.137e-04, forward_time=0.263, loss_att=229.995, acc=0.798, loss=229.995, backward_time=0.388, grad_norm=103.443, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=7.189e-04, train_time=3.368 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 07:58:58,888 (trainer:732) INFO: 16epoch:train:1197-1288batch: iter_time=3.986e-04, forward_time=0.265, loss_att=233.321, acc=0.800, loss=233.321, backward_time=0.391, grad_norm=94.968, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=7.212e-04, train_time=3.396 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:00:16,784 (trainer:732) INFO: 16epoch:train:1289-1380batch: iter_time=4.044e-04, forward_time=0.265, loss_att=234.184, acc=0.801, loss=234.184, backward_time=0.392, grad_norm=106.429, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=7.235e-04, train_time=3.386 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:01:34,099 (trainer:732) INFO: 16epoch:train:1381-1472batch: iter_time=4.390e-04, forward_time=0.263, loss_att=230.030, acc=0.800, loss=230.030, backward_time=0.387, grad_norm=94.933, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=7.258e-04, train_time=3.360 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:02:51,967 (trainer:732) INFO: 16epoch:train:1473-1564batch: iter_time=4.605e-04, forward_time=0.267, loss_att=225.854, acc=0.798, loss=225.854, backward_time=0.388, grad_norm=104.593, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=7.281e-04, train_time=3.385 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:04:09,731 (trainer:732) INFO: 16epoch:train:1565-1656batch: iter_time=4.193e-04, forward_time=0.264, loss_att=225.455, acc=0.795, loss=225.455, backward_time=0.387, grad_norm=104.800, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=7.304e-04, train_time=3.380 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:05:27,987 (trainer:732) INFO: 16epoch:train:1657-1748batch: iter_time=4.550e-04, forward_time=0.266, loss_att=237.030, acc=0.798, loss=237.030, backward_time=0.394, grad_norm=97.856, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=7.327e-04, train_time=3.402 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:06:45,262 (trainer:732) INFO: 16epoch:train:1749-1840batch: iter_time=3.719e-04, forward_time=0.262, loss_att=227.220, acc=0.798, loss=227.220, backward_time=0.386, grad_norm=105.950, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=7.350e-04, train_time=3.359 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:19:14,421 (trainer:338) INFO: 16epoch results: [train] iter_time=9.004e-04, forward_time=0.270, loss_att=228.374, acc=0.800, loss=228.374, backward_time=0.389, grad_norm=99.754, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=7.131e-04, train_time=4.072, time=31 minutes and 20.05 seconds, total_count=29488, gpu_max_cached_mem_GB=29.846, [valid] loss_att=250.663, acc=0.782, cer=0.275, wer=0.582, loss=250.663, time=5 minutes and 18.76 seconds, total_count=304, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 3.95 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:19:24,380 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:19:24,401 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/6epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:19:24,402 (trainer:272) INFO: 17/60epoch started. Estimated time to finish: 1 day, 7 hours and 11 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:25:40,596 (trainer:732) INFO: 17epoch:train:1-92batch: iter_time=0.011, forward_time=0.277, loss_att=226.170, acc=0.807, loss=226.170, backward_time=0.389, grad_norm=103.355, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=7.373e-04, train_time=16.355 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:26:59,312 (trainer:732) INFO: 17epoch:train:93-184batch: iter_time=5.526e-04, forward_time=0.270, loss_att=212.958, acc=0.798, loss=212.958, backward_time=0.383, grad_norm=101.294, clip=100.000, loss_scale=1.000, optim_step_time=0.132, optim0_lr0=7.396e-04, train_time=3.422 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:28:19,496 (trainer:732) INFO: 17epoch:train:185-276batch: iter_time=5.963e-04, forward_time=0.277, loss_att=231.833, acc=0.807, loss=231.833, backward_time=0.393, grad_norm=111.618, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=7.419e-04, train_time=3.486 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:29:39,348 (trainer:732) INFO: 17epoch:train:277-368batch: iter_time=5.380e-04, forward_time=0.276, loss_att=216.167, acc=0.801, loss=216.167, backward_time=0.389, grad_norm=94.700, clip=100.000, loss_scale=1.000, optim_step_time=0.129, optim0_lr0=7.442e-04, train_time=3.471 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:31:00,894 (trainer:732) INFO: 17epoch:train:369-460batch: iter_time=8.209e-04, forward_time=0.279, loss_att=226.981, acc=0.806, loss=226.981, backward_time=0.392, grad_norm=106.308, clip=100.000, loss_scale=1.000, optim_step_time=0.132, optim0_lr0=7.465e-04, train_time=3.545 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:32:21,621 (trainer:732) INFO: 17epoch:train:461-552batch: iter_time=7.077e-04, forward_time=0.280, loss_att=220.888, acc=0.808, loss=220.888, backward_time=0.395, grad_norm=97.513, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=7.488e-04, train_time=3.509 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:33:42,327 (trainer:732) INFO: 17epoch:train:553-644batch: iter_time=5.059e-04, forward_time=0.279, loss_att=232.193, acc=0.804, loss=232.193, backward_time=0.395, grad_norm=104.079, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=7.511e-04, train_time=3.508 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:35:04,795 (trainer:732) INFO: 17epoch:train:645-736batch: iter_time=5.457e-04, forward_time=0.281, loss_att=230.080, acc=0.805, loss=230.080, backward_time=0.395, grad_norm=113.059, clip=100.000, loss_scale=1.000, optim_step_time=0.164, optim0_lr0=7.534e-04, train_time=3.585 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:36:24,936 (trainer:732) INFO: 17epoch:train:737-828batch: iter_time=6.607e-04, forward_time=0.276, loss_att=212.287, acc=0.804, loss=212.287, backward_time=0.392, grad_norm=101.878, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=7.557e-04, train_time=3.484 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:37:44,827 (trainer:732) INFO: 17epoch:train:829-920batch: iter_time=5.178e-04, forward_time=0.273, loss_att=227.301, acc=0.808, loss=227.301, backward_time=0.394, grad_norm=97.846, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=7.580e-04, train_time=3.473 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:39:05,173 (trainer:732) INFO: 17epoch:train:921-1012batch: iter_time=5.267e-04, forward_time=0.274, loss_att=219.864, acc=0.801, loss=219.864, backward_time=0.392, grad_norm=96.503, clip=100.000, loss_scale=1.000, optim_step_time=0.143, optim0_lr0=7.603e-04, train_time=3.493 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:40:25,272 (trainer:732) INFO: 17epoch:train:1013-1104batch: iter_time=4.865e-04, forward_time=0.276, loss_att=223.384, acc=0.809, loss=223.384, backward_time=0.391, grad_norm=101.033, clip=100.000, loss_scale=1.000, optim_step_time=0.126, optim0_lr0=7.626e-04, train_time=3.482 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:41:44,693 (trainer:732) INFO: 17epoch:train:1105-1196batch: iter_time=6.475e-04, forward_time=0.275, loss_att=221.078, acc=0.798, loss=221.078, backward_time=0.391, grad_norm=98.079, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=7.649e-04, train_time=3.452 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:43:04,733 (trainer:732) INFO: 17epoch:train:1197-1288batch: iter_time=7.289e-04, forward_time=0.273, loss_att=224.555, acc=0.804, loss=224.555, backward_time=0.390, grad_norm=93.536, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=7.672e-04, train_time=3.479 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:44:24,441 (trainer:732) INFO: 17epoch:train:1289-1380batch: iter_time=5.937e-04, forward_time=0.276, loss_att=219.253, acc=0.802, loss=219.253, backward_time=0.389, grad_norm=106.818, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=7.695e-04, train_time=3.465 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:45:44,101 (trainer:732) INFO: 17epoch:train:1381-1472batch: iter_time=5.925e-04, forward_time=0.276, loss_att=227.818, acc=0.808, loss=227.818, backward_time=0.390, grad_norm=102.086, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=7.718e-04, train_time=3.463 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:47:03,802 (trainer:732) INFO: 17epoch:train:1473-1564batch: iter_time=5.583e-04, forward_time=0.272, loss_att=226.199, acc=0.800, loss=226.199, backward_time=0.391, grad_norm=100.703, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=7.741e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:48:23,888 (trainer:732) INFO: 17epoch:train:1565-1656batch: iter_time=4.916e-04, forward_time=0.275, loss_att=219.634, acc=0.802, loss=219.634, backward_time=0.393, grad_norm=93.468, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=7.764e-04, train_time=3.481 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:49:42,981 (trainer:732) INFO: 17epoch:train:1657-1748batch: iter_time=4.597e-04, forward_time=0.274, loss_att=223.741, acc=0.805, loss=223.741, backward_time=0.388, grad_norm=97.833, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=7.787e-04, train_time=3.438 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 08:51:01,856 (trainer:732) INFO: 17epoch:train:1749-1840batch: iter_time=4.465e-04, forward_time=0.270, loss_att=221.984, acc=0.806, loss=221.984, backward_time=0.387, grad_norm=91.336, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=7.810e-04, train_time=3.429 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:03:00,512 (trainer:338) INFO: 17epoch results: [train] iter_time=0.001, forward_time=0.275, loss_att=223.049, acc=0.804, loss=223.049, backward_time=0.391, grad_norm=100.652, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=7.591e-04, train_time=4.124, time=31 minutes and 43.34 seconds, total_count=31331, gpu_max_cached_mem_GB=29.846, [valid] loss_att=251.777, acc=0.782, cer=0.273, wer=0.584, loss=251.777, time=5 minutes and 11.96 seconds, total_count=323, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 40.8 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:03:08,932 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:03:08,942 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/7epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:03:08,943 (trainer:272) INFO: 18/60epoch started. Estimated time to finish: 1 day, 6 hours and 32 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:08:01,035 (trainer:732) INFO: 18epoch:train:1-92batch: iter_time=0.021, forward_time=0.272, loss_att=220.921, acc=0.808, loss=220.921, backward_time=0.388, grad_norm=109.311, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=7.833e-04, train_time=12.698 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:09:17,962 (trainer:732) INFO: 18epoch:train:93-184batch: iter_time=3.952e-04, forward_time=0.262, loss_att=215.350, acc=0.815, loss=215.350, backward_time=0.386, grad_norm=102.387, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=7.856e-04, train_time=3.344 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:10:36,269 (trainer:732) INFO: 18epoch:train:185-276batch: iter_time=4.039e-04, forward_time=0.266, loss_att=218.939, acc=0.813, loss=218.939, backward_time=0.394, grad_norm=98.313, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=7.879e-04, train_time=3.404 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:11:53,476 (trainer:732) INFO: 18epoch:train:277-368batch: iter_time=5.159e-04, forward_time=0.263, loss_att=215.305, acc=0.807, loss=215.305, backward_time=0.386, grad_norm=91.071, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=7.902e-04, train_time=3.356 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:13:10,866 (trainer:732) INFO: 18epoch:train:369-460batch: iter_time=4.236e-04, forward_time=0.263, loss_att=218.585, acc=0.809, loss=218.585, backward_time=0.389, grad_norm=95.860, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=7.925e-04, train_time=3.364 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:14:28,447 (trainer:732) INFO: 18epoch:train:461-552batch: iter_time=4.526e-04, forward_time=0.264, loss_att=224.535, acc=0.806, loss=224.535, backward_time=0.390, grad_norm=100.860, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=7.948e-04, train_time=3.372 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:15:46,717 (trainer:732) INFO: 18epoch:train:553-644batch: iter_time=3.843e-04, forward_time=0.264, loss_att=212.435, acc=0.810, loss=212.435, backward_time=0.392, grad_norm=103.012, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=7.971e-04, train_time=3.402 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:17:05,053 (trainer:732) INFO: 18epoch:train:645-736batch: iter_time=4.054e-04, forward_time=0.264, loss_att=222.652, acc=0.806, loss=222.652, backward_time=0.393, grad_norm=102.675, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=7.994e-04, train_time=3.405 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:18:21,555 (trainer:732) INFO: 18epoch:train:737-828batch: iter_time=4.212e-04, forward_time=0.259, loss_att=208.268, acc=0.803, loss=208.268, backward_time=0.382, grad_norm=89.680, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=8.017e-04, train_time=3.325 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:19:38,649 (trainer:732) INFO: 18epoch:train:829-920batch: iter_time=4.693e-04, forward_time=0.261, loss_att=213.929, acc=0.806, loss=213.929, backward_time=0.387, grad_norm=96.084, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=8.040e-04, train_time=3.351 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:20:56,198 (trainer:732) INFO: 18epoch:train:921-1012batch: iter_time=4.501e-04, forward_time=0.264, loss_att=223.386, acc=0.807, loss=223.386, backward_time=0.391, grad_norm=94.523, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=8.063e-04, train_time=3.371 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:22:13,829 (trainer:732) INFO: 18epoch:train:1013-1104batch: iter_time=4.494e-04, forward_time=0.263, loss_att=228.415, acc=0.809, loss=228.415, backward_time=0.389, grad_norm=98.810, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=8.086e-04, train_time=3.374 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:23:30,979 (trainer:732) INFO: 18epoch:train:1105-1196batch: iter_time=3.561e-04, forward_time=0.261, loss_att=222.237, acc=0.806, loss=222.237, backward_time=0.387, grad_norm=93.494, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=8.109e-04, train_time=3.354 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:24:48,232 (trainer:732) INFO: 18epoch:train:1197-1288batch: iter_time=4.031e-04, forward_time=0.263, loss_att=221.027, acc=0.808, loss=221.027, backward_time=0.387, grad_norm=103.147, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=8.132e-04, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:26:07,923 (trainer:732) INFO: 18epoch:train:1289-1380batch: iter_time=4.653e-04, forward_time=0.275, loss_att=226.313, acc=0.809, loss=226.313, backward_time=0.393, grad_norm=95.355, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=8.155e-04, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:27:27,025 (trainer:732) INFO: 18epoch:train:1381-1472batch: iter_time=5.057e-04, forward_time=0.275, loss_att=216.783, acc=0.804, loss=216.783, backward_time=0.388, grad_norm=97.261, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=8.178e-04, train_time=3.439 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:28:47,093 (trainer:732) INFO: 18epoch:train:1473-1564batch: iter_time=5.923e-04, forward_time=0.277, loss_att=220.425, acc=0.807, loss=220.425, backward_time=0.391, grad_norm=100.089, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=8.201e-04, train_time=3.480 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:30:08,266 (trainer:732) INFO: 18epoch:train:1565-1656batch: iter_time=5.171e-04, forward_time=0.278, loss_att=221.748, acc=0.807, loss=221.748, backward_time=0.395, grad_norm=94.281, clip=100.000, loss_scale=1.000, optim_step_time=0.138, optim0_lr0=8.224e-04, train_time=3.529 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:31:29,122 (trainer:732) INFO: 18epoch:train:1657-1748batch: iter_time=5.135e-04, forward_time=0.277, loss_att=211.160, acc=0.809, loss=211.160, backward_time=0.392, grad_norm=86.488, clip=100.000, loss_scale=1.000, optim_step_time=0.129, optim0_lr0=8.247e-04, train_time=3.515 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:32:49,034 (trainer:732) INFO: 18epoch:train:1749-1840batch: iter_time=3.857e-04, forward_time=0.274, loss_att=212.246, acc=0.808, loss=212.246, backward_time=0.387, grad_norm=98.198, clip=100.000, loss_scale=1.000, optim_step_time=0.145, optim0_lr0=8.270e-04, train_time=3.474 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:45:19,997 (trainer:338) INFO: 18epoch results: [train] iter_time=0.001, forward_time=0.267, loss_att=218.613, acc=0.808, loss=218.613, backward_time=0.389, grad_norm=97.545, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=8.051e-04, train_time=3.869, time=29 minutes and 47.25 seconds, total_count=33174, gpu_max_cached_mem_GB=29.846, [valid] loss_att=247.059, acc=0.786, cer=0.268, wer=0.583, loss=247.059, time=5 minutes and 16.49 seconds, total_count=342, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 7.31 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:45:29,638 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:45:29,653 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/8epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:45:29,654 (trainer:272) INFO: 19/60epoch started. Estimated time to finish: 1 day, 5 hours and 48 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:51:23,784 (trainer:732) INFO: 19epoch:train:1-92batch: iter_time=0.012, forward_time=0.304, loss_att=210.785, acc=0.813, loss=210.785, backward_time=0.399, grad_norm=95.196, clip=100.000, loss_scale=1.000, optim_step_time=0.182, optim0_lr0=8.293e-04, train_time=15.394 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:52:44,474 (trainer:732) INFO: 19epoch:train:93-184batch: iter_time=4.727e-04, forward_time=0.277, loss_att=210.970, acc=0.810, loss=210.970, backward_time=0.387, grad_norm=101.543, clip=100.000, loss_scale=1.000, optim_step_time=0.151, optim0_lr0=8.316e-04, train_time=3.507 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:54:05,386 (trainer:732) INFO: 19epoch:train:185-276batch: iter_time=5.813e-04, forward_time=0.282, loss_att=210.073, acc=0.808, loss=210.073, backward_time=0.396, grad_norm=102.479, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=8.339e-04, train_time=3.518 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:55:25,734 (trainer:732) INFO: 19epoch:train:277-368batch: iter_time=7.922e-04, forward_time=0.276, loss_att=214.899, acc=0.813, loss=214.899, backward_time=0.389, grad_norm=102.224, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=8.362e-04, train_time=3.493 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:56:47,360 (trainer:732) INFO: 19epoch:train:369-460batch: iter_time=5.533e-04, forward_time=0.281, loss_att=218.716, acc=0.812, loss=218.716, backward_time=0.396, grad_norm=97.952, clip=100.000, loss_scale=1.000, optim_step_time=0.132, optim0_lr0=8.385e-04, train_time=3.548 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:58:08,791 (trainer:732) INFO: 19epoch:train:461-552batch: iter_time=5.717e-04, forward_time=0.276, loss_att=207.338, acc=0.814, loss=207.338, backward_time=0.390, grad_norm=100.259, clip=100.000, loss_scale=1.000, optim_step_time=0.139, optim0_lr0=8.408e-04, train_time=3.540 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 09:59:29,581 (trainer:732) INFO: 19epoch:train:553-644batch: iter_time=6.808e-04, forward_time=0.277, loss_att=212.954, acc=0.811, loss=212.954, backward_time=0.390, grad_norm=87.265, clip=100.000, loss_scale=1.000, optim_step_time=0.134, optim0_lr0=8.431e-04, train_time=3.511 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:00:49,953 (trainer:732) INFO: 19epoch:train:645-736batch: iter_time=5.453e-04, forward_time=0.279, loss_att=214.305, acc=0.813, loss=214.305, backward_time=0.391, grad_norm=102.510, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=8.454e-04, train_time=3.494 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:02:10,645 (trainer:732) INFO: 19epoch:train:737-828batch: iter_time=4.777e-04, forward_time=0.279, loss_att=209.693, acc=0.808, loss=209.693, backward_time=0.389, grad_norm=88.901, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=8.477e-04, train_time=3.507 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:03:31,979 (trainer:732) INFO: 19epoch:train:829-920batch: iter_time=4.849e-04, forward_time=0.279, loss_att=224.196, acc=0.814, loss=224.196, backward_time=0.391, grad_norm=97.101, clip=100.000, loss_scale=1.000, optim_step_time=0.148, optim0_lr0=8.500e-04, train_time=3.535 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:04:52,826 (trainer:732) INFO: 19epoch:train:921-1012batch: iter_time=5.123e-04, forward_time=0.276, loss_att=208.798, acc=0.811, loss=208.798, backward_time=0.394, grad_norm=109.837, clip=100.000, loss_scale=1.000, optim_step_time=0.131, optim0_lr0=8.523e-04, train_time=3.514 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:06:12,540 (trainer:732) INFO: 19epoch:train:1013-1104batch: iter_time=5.012e-04, forward_time=0.275, loss_att=210.779, acc=0.809, loss=210.779, backward_time=0.387, grad_norm=91.458, clip=100.000, loss_scale=1.000, optim_step_time=0.129, optim0_lr0=8.546e-04, train_time=3.465 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:07:35,456 (trainer:732) INFO: 19epoch:train:1105-1196batch: iter_time=4.909e-04, forward_time=0.279, loss_att=216.483, acc=0.812, loss=216.483, backward_time=0.399, grad_norm=92.918, clip=100.000, loss_scale=1.000, optim_step_time=0.128, optim0_lr0=8.569e-04, train_time=3.604 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:08:57,286 (trainer:732) INFO: 19epoch:train:1197-1288batch: iter_time=4.905e-04, forward_time=0.284, loss_att=221.398, acc=0.814, loss=221.398, backward_time=0.398, grad_norm=98.934, clip=100.000, loss_scale=1.000, optim_step_time=0.136, optim0_lr0=8.592e-04, train_time=3.557 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:10:18,777 (trainer:732) INFO: 19epoch:train:1289-1380batch: iter_time=5.242e-04, forward_time=0.279, loss_att=226.313, acc=0.811, loss=226.313, backward_time=0.393, grad_norm=99.874, clip=100.000, loss_scale=1.000, optim_step_time=0.149, optim0_lr0=8.615e-04, train_time=3.542 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:11:38,942 (trainer:732) INFO: 19epoch:train:1381-1472batch: iter_time=4.659e-04, forward_time=0.277, loss_att=206.462, acc=0.811, loss=206.462, backward_time=0.389, grad_norm=107.213, clip=100.000, loss_scale=1.000, optim_step_time=0.128, optim0_lr0=8.638e-04, train_time=3.485 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:12:58,819 (trainer:732) INFO: 19epoch:train:1473-1564batch: iter_time=4.465e-04, forward_time=0.276, loss_att=217.579, acc=0.813, loss=217.579, backward_time=0.391, grad_norm=94.482, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=8.661e-04, train_time=3.472 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:14:19,202 (trainer:732) INFO: 19epoch:train:1565-1656batch: iter_time=5.554e-04, forward_time=0.277, loss_att=210.843, acc=0.808, loss=210.843, backward_time=0.390, grad_norm=91.643, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=8.684e-04, train_time=3.494 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:15:40,592 (trainer:732) INFO: 19epoch:train:1657-1748batch: iter_time=6.205e-04, forward_time=0.280, loss_att=222.013, acc=0.814, loss=222.013, backward_time=0.397, grad_norm=104.930, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=8.707e-04, train_time=3.538 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:17:01,192 (trainer:732) INFO: 19epoch:train:1749-1840batch: iter_time=3.937e-04, forward_time=0.278, loss_att=212.366, acc=0.814, loss=212.366, backward_time=0.392, grad_norm=97.141, clip=100.000, loss_scale=1.000, optim_step_time=0.135, optim0_lr0=8.730e-04, train_time=3.504 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:29:30,961 (trainer:338) INFO: 19epoch results: [train] iter_time=0.001, forward_time=0.280, loss_att=214.163, acc=0.812, loss=214.163, backward_time=0.392, grad_norm=98.193, clip=100.000, loss_scale=1.000, optim_step_time=0.133, optim0_lr0=8.512e-04, train_time=4.111, time=31 minutes and 38.1 seconds, total_count=35017, gpu_max_cached_mem_GB=29.846, [valid] loss_att=244.085, acc=0.787, cer=0.266, wer=0.577, loss=244.085, time=5 minutes and 13.34 seconds, total_count=361, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 9.86 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:29:41,293 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:29:41,325 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/9epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:29:41,326 (trainer:272) INFO: 20/60epoch started. Estimated time to finish: 1 day, 5 hours and 9 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:36:04,415 (trainer:732) INFO: 20epoch:train:1-92batch: iter_time=0.012, forward_time=0.287, loss_att=208.772, acc=0.818, loss=208.772, backward_time=0.394, grad_norm=95.360, clip=100.000, loss_scale=1.000, optim_step_time=0.128, optim0_lr0=8.753e-04, train_time=16.654 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:37:26,878 (trainer:732) INFO: 20epoch:train:93-184batch: iter_time=4.932e-04, forward_time=0.283, loss_att=207.931, acc=0.814, loss=207.931, backward_time=0.393, grad_norm=107.291, clip=100.000, loss_scale=1.000, optim_step_time=0.147, optim0_lr0=8.776e-04, train_time=3.585 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:38:45,511 (trainer:732) INFO: 20epoch:train:185-276batch: iter_time=5.008e-04, forward_time=0.269, loss_att=204.491, acc=0.812, loss=204.491, backward_time=0.386, grad_norm=94.889, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=8.799e-04, train_time=3.418 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:40:06,664 (trainer:732) INFO: 20epoch:train:277-368batch: iter_time=5.615e-04, forward_time=0.278, loss_att=208.394, acc=0.818, loss=208.394, backward_time=0.394, grad_norm=99.265, clip=100.000, loss_scale=1.000, optim_step_time=0.132, optim0_lr0=8.822e-04, train_time=3.528 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:41:24,845 (trainer:732) INFO: 20epoch:train:369-460batch: iter_time=4.396e-04, forward_time=0.264, loss_att=207.363, acc=0.816, loss=207.363, backward_time=0.388, grad_norm=95.563, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=8.845e-04, train_time=3.398 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:42:42,798 (trainer:732) INFO: 20epoch:train:461-552batch: iter_time=4.763e-04, forward_time=0.264, loss_att=213.403, acc=0.819, loss=213.403, backward_time=0.390, grad_norm=92.321, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=8.868e-04, train_time=3.389 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:43:59,982 (trainer:732) INFO: 20epoch:train:553-644batch: iter_time=4.187e-04, forward_time=0.262, loss_att=209.489, acc=0.815, loss=209.489, backward_time=0.387, grad_norm=99.340, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=8.891e-04, train_time=3.355 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:45:19,078 (trainer:732) INFO: 20epoch:train:645-736batch: iter_time=5.864e-04, forward_time=0.270, loss_att=208.153, acc=0.818, loss=208.153, backward_time=0.391, grad_norm=98.147, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=8.914e-04, train_time=3.438 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:46:39,394 (trainer:732) INFO: 20epoch:train:737-828batch: iter_time=6.538e-04, forward_time=0.276, loss_att=213.531, acc=0.817, loss=213.531, backward_time=0.396, grad_norm=90.239, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=8.937e-04, train_time=3.491 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:48:00,247 (trainer:732) INFO: 20epoch:train:829-920batch: iter_time=5.791e-04, forward_time=0.282, loss_att=205.973, acc=0.816, loss=205.973, backward_time=0.390, grad_norm=94.964, clip=100.000, loss_scale=1.000, optim_step_time=0.127, optim0_lr0=8.960e-04, train_time=3.515 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:49:20,862 (trainer:732) INFO: 20epoch:train:921-1012batch: iter_time=5.350e-04, forward_time=0.275, loss_att=213.419, acc=0.814, loss=213.419, backward_time=0.392, grad_norm=94.145, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=8.983e-04, train_time=3.504 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:50:41,393 (trainer:732) INFO: 20epoch:train:1013-1104batch: iter_time=5.224e-04, forward_time=0.277, loss_att=217.205, acc=0.815, loss=217.205, backward_time=0.393, grad_norm=94.459, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=9.006e-04, train_time=3.501 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:52:02,196 (trainer:732) INFO: 20epoch:train:1105-1196batch: iter_time=4.330e-04, forward_time=0.277, loss_att=207.584, acc=0.814, loss=207.584, backward_time=0.390, grad_norm=96.288, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=9.029e-04, train_time=3.511 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:53:23,027 (trainer:732) INFO: 20epoch:train:1197-1288batch: iter_time=5.679e-04, forward_time=0.276, loss_att=205.587, acc=0.820, loss=205.587, backward_time=0.393, grad_norm=96.942, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=9.052e-04, train_time=3.515 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:54:43,615 (trainer:732) INFO: 20epoch:train:1289-1380batch: iter_time=5.728e-04, forward_time=0.276, loss_att=207.841, acc=0.812, loss=207.841, backward_time=0.397, grad_norm=103.323, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=9.075e-04, train_time=3.503 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:56:04,286 (trainer:732) INFO: 20epoch:train:1381-1472batch: iter_time=5.810e-04, forward_time=0.279, loss_att=212.607, acc=0.815, loss=212.607, backward_time=0.392, grad_norm=107.257, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=9.098e-04, train_time=3.507 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:57:25,198 (trainer:732) INFO: 20epoch:train:1473-1564batch: iter_time=5.281e-04, forward_time=0.278, loss_att=205.004, acc=0.815, loss=205.004, backward_time=0.389, grad_norm=99.671, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=9.121e-04, train_time=3.517 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 10:58:46,701 (trainer:732) INFO: 20epoch:train:1565-1656batch: iter_time=6.981e-04, forward_time=0.280, loss_att=219.574, acc=0.819, loss=219.574, backward_time=0.394, grad_norm=98.768, clip=100.000, loss_scale=1.000, optim_step_time=0.133, optim0_lr0=9.144e-04, train_time=3.543 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:00:07,665 (trainer:732) INFO: 20epoch:train:1657-1748batch: iter_time=4.974e-04, forward_time=0.276, loss_att=210.895, acc=0.813, loss=210.895, backward_time=0.393, grad_norm=104.785, clip=100.000, loss_scale=1.000, optim_step_time=0.129, optim0_lr0=9.167e-04, train_time=3.517 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:01:26,263 (trainer:732) INFO: 20epoch:train:1749-1840batch: iter_time=4.002e-04, forward_time=0.270, loss_att=205.263, acc=0.813, loss=205.263, backward_time=0.388, grad_norm=95.605, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=9.190e-04, train_time=3.419 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:13:58,972 (trainer:338) INFO: 20epoch results: [train] iter_time=0.001, forward_time=0.275, loss_att=209.567, acc=0.816, loss=209.567, backward_time=0.392, grad_norm=97.931, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=8.971e-04, train_time=4.140, time=31 minutes and 51.95 seconds, total_count=36860, gpu_max_cached_mem_GB=29.846, [valid] loss_att=245.009, acc=0.788, cer=0.266, wer=0.580, loss=245.009, time=5 minutes and 14.95 seconds, total_count=380, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 10.75 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:14:08,741 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:14:08,759 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/10epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:14:08,760 (trainer:272) INFO: 21/60epoch started. Estimated time to finish: 1 day, 4 hours and 30 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:19:43,008 (trainer:732) INFO: 21epoch:train:1-92batch: iter_time=0.017, forward_time=0.271, loss_att=204.441, acc=0.820, loss=204.441, backward_time=0.388, grad_norm=96.064, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=9.213e-04, train_time=14.531 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:21:00,750 (trainer:732) INFO: 21epoch:train:93-184batch: iter_time=4.769e-04, forward_time=0.265, loss_att=201.974, acc=0.819, loss=201.974, backward_time=0.389, grad_norm=99.095, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=9.236e-04, train_time=3.379 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:22:18,421 (trainer:732) INFO: 21epoch:train:185-276batch: iter_time=4.539e-04, forward_time=0.265, loss_att=199.840, acc=0.820, loss=199.840, backward_time=0.388, grad_norm=89.551, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=9.259e-04, train_time=3.376 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:23:35,829 (trainer:732) INFO: 21epoch:train:277-368batch: iter_time=4.014e-04, forward_time=0.262, loss_att=202.373, acc=0.823, loss=202.373, backward_time=0.387, grad_norm=94.040, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=9.282e-04, train_time=3.365 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:24:53,629 (trainer:732) INFO: 21epoch:train:369-460batch: iter_time=4.453e-04, forward_time=0.264, loss_att=206.338, acc=0.822, loss=206.338, backward_time=0.388, grad_norm=94.926, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=9.305e-04, train_time=3.382 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:26:10,911 (trainer:732) INFO: 21epoch:train:461-552batch: iter_time=3.773e-04, forward_time=0.263, loss_att=207.541, acc=0.818, loss=207.541, backward_time=0.387, grad_norm=91.790, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=9.328e-04, train_time=3.359 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:27:28,416 (trainer:732) INFO: 21epoch:train:553-644batch: iter_time=3.983e-04, forward_time=0.264, loss_att=210.051, acc=0.818, loss=210.051, backward_time=0.389, grad_norm=96.414, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=9.351e-04, train_time=3.369 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:28:46,865 (trainer:732) INFO: 21epoch:train:645-736batch: iter_time=4.585e-04, forward_time=0.265, loss_att=204.462, acc=0.816, loss=204.462, backward_time=0.389, grad_norm=98.124, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=9.374e-04, train_time=3.410 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:30:05,325 (trainer:732) INFO: 21epoch:train:737-828batch: iter_time=4.250e-04, forward_time=0.266, loss_att=212.266, acc=0.820, loss=212.266, backward_time=0.392, grad_norm=98.766, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=9.397e-04, train_time=3.411 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:31:22,241 (trainer:732) INFO: 21epoch:train:829-920batch: iter_time=4.375e-04, forward_time=0.263, loss_att=199.323, acc=0.815, loss=199.323, backward_time=0.385, grad_norm=85.218, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=9.420e-04, train_time=3.343 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:32:39,267 (trainer:732) INFO: 21epoch:train:921-1012batch: iter_time=4.159e-04, forward_time=0.262, loss_att=209.220, acc=0.820, loss=209.220, backward_time=0.386, grad_norm=91.766, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=9.443e-04, train_time=3.348 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:33:56,680 (trainer:732) INFO: 21epoch:train:1013-1104batch: iter_time=3.948e-04, forward_time=0.261, loss_att=206.956, acc=0.818, loss=206.956, backward_time=0.386, grad_norm=107.288, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=9.466e-04, train_time=3.365 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:35:15,005 (trainer:732) INFO: 21epoch:train:1105-1196batch: iter_time=4.264e-04, forward_time=0.267, loss_att=210.830, acc=0.822, loss=210.830, backward_time=0.393, grad_norm=96.350, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=9.489e-04, train_time=3.405 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:36:33,100 (trainer:732) INFO: 21epoch:train:1197-1288batch: iter_time=4.664e-04, forward_time=0.266, loss_att=212.338, acc=0.817, loss=212.338, backward_time=0.393, grad_norm=91.480, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=9.512e-04, train_time=3.395 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:37:50,943 (trainer:732) INFO: 21epoch:train:1289-1380batch: iter_time=4.607e-04, forward_time=0.265, loss_att=208.268, acc=0.816, loss=208.268, backward_time=0.390, grad_norm=96.910, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=9.535e-04, train_time=3.384 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:39:08,494 (trainer:732) INFO: 21epoch:train:1381-1472batch: iter_time=4.642e-04, forward_time=0.263, loss_att=205.099, acc=0.817, loss=205.099, backward_time=0.387, grad_norm=93.451, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=9.558e-04, train_time=3.371 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:40:24,721 (trainer:732) INFO: 21epoch:train:1473-1564batch: iter_time=4.134e-04, forward_time=0.259, loss_att=201.203, acc=0.816, loss=201.203, backward_time=0.381, grad_norm=93.846, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=9.581e-04, train_time=3.313 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:41:42,833 (trainer:732) INFO: 21epoch:train:1565-1656batch: iter_time=4.825e-04, forward_time=0.264, loss_att=210.220, acc=0.821, loss=210.220, backward_time=0.392, grad_norm=105.348, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=9.604e-04, train_time=3.395 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:43:00,574 (trainer:732) INFO: 21epoch:train:1657-1748batch: iter_time=4.185e-04, forward_time=0.264, loss_att=208.347, acc=0.817, loss=208.347, backward_time=0.392, grad_norm=96.570, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=9.627e-04, train_time=3.379 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:44:17,795 (trainer:732) INFO: 21epoch:train:1749-1840batch: iter_time=3.174e-04, forward_time=0.262, loss_att=198.190, acc=0.819, loss=198.190, backward_time=0.388, grad_norm=96.768, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=9.650e-04, train_time=3.357 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:55:59,072 (trainer:338) INFO: 21epoch results: [train] iter_time=0.001, forward_time=0.264, loss_att=205.885, acc=0.819, loss=205.885, backward_time=0.388, grad_norm=95.688, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=9.431e-04, train_time=3.932, time=30 minutes and 15.42 seconds, total_count=38703, gpu_max_cached_mem_GB=29.846, [valid] loss_att=242.324, acc=0.790, cer=0.261, wer=0.577, loss=242.324, time=5 minutes and 7.16 seconds, total_count=399, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 27.73 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:56:07,185 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:56:07,195 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/11epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 11:56:07,196 (trainer:272) INFO: 22/60epoch started. Estimated time to finish: 1 day, 3 hours and 46 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:01:26,577 (trainer:732) INFO: 22epoch:train:1-92batch: iter_time=0.011, forward_time=0.281, loss_att=197.671, acc=0.821, loss=197.671, backward_time=0.396, grad_norm=92.677, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=9.673e-04, train_time=13.885 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:02:47,576 (trainer:732) INFO: 22epoch:train:93-184batch: iter_time=7.305e-04, forward_time=0.283, loss_att=201.468, acc=0.826, loss=201.468, backward_time=0.398, grad_norm=96.247, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=9.696e-04, train_time=3.521 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:04:07,034 (trainer:732) INFO: 22epoch:train:185-276batch: iter_time=5.053e-04, forward_time=0.273, loss_att=205.211, acc=0.820, loss=205.211, backward_time=0.390, grad_norm=104.763, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=9.719e-04, train_time=3.454 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:05:28,373 (trainer:732) INFO: 22epoch:train:277-368batch: iter_time=5.453e-04, forward_time=0.279, loss_att=202.554, acc=0.824, loss=202.554, backward_time=0.397, grad_norm=99.086, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=9.742e-04, train_time=3.536 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:06:48,590 (trainer:732) INFO: 22epoch:train:369-460batch: iter_time=5.461e-04, forward_time=0.278, loss_att=205.369, acc=0.825, loss=205.369, backward_time=0.396, grad_norm=97.596, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=9.765e-04, train_time=3.487 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:08:10,964 (trainer:732) INFO: 22epoch:train:461-552batch: iter_time=4.930e-04, forward_time=0.287, loss_att=203.591, acc=0.826, loss=203.591, backward_time=0.393, grad_norm=99.950, clip=100.000, loss_scale=1.000, optim_step_time=0.130, optim0_lr0=9.788e-04, train_time=3.581 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:09:30,093 (trainer:732) INFO: 22epoch:train:553-644batch: iter_time=4.462e-04, forward_time=0.272, loss_att=199.670, acc=0.823, loss=199.670, backward_time=0.387, grad_norm=95.578, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=9.811e-04, train_time=3.440 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:10:49,656 (trainer:732) INFO: 22epoch:train:645-736batch: iter_time=5.001e-04, forward_time=0.273, loss_att=202.525, acc=0.823, loss=202.525, backward_time=0.393, grad_norm=96.066, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=9.834e-04, train_time=3.458 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:12:09,331 (trainer:732) INFO: 22epoch:train:737-828batch: iter_time=5.192e-04, forward_time=0.273, loss_att=197.188, acc=0.818, loss=197.188, backward_time=0.390, grad_norm=94.512, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=9.857e-04, train_time=3.463 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:13:28,553 (trainer:732) INFO: 22epoch:train:829-920batch: iter_time=4.722e-04, forward_time=0.273, loss_att=202.904, acc=0.821, loss=202.904, backward_time=0.391, grad_norm=98.333, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=9.880e-04, train_time=3.444 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:14:46,923 (trainer:732) INFO: 22epoch:train:921-1012batch: iter_time=5.737e-04, forward_time=0.266, loss_att=205.503, acc=0.823, loss=205.503, backward_time=0.390, grad_norm=98.567, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=9.903e-04, train_time=3.407 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:16:04,160 (trainer:732) INFO: 22epoch:train:1013-1104batch: iter_time=4.672e-04, forward_time=0.265, loss_att=196.944, acc=0.820, loss=196.944, backward_time=0.386, grad_norm=90.852, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=9.926e-04, train_time=3.357 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:17:20,798 (trainer:732) INFO: 22epoch:train:1105-1196batch: iter_time=3.808e-04, forward_time=0.261, loss_att=202.025, acc=0.820, loss=202.025, backward_time=0.384, grad_norm=89.459, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=9.949e-04, train_time=3.331 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:18:38,555 (trainer:732) INFO: 22epoch:train:1197-1288batch: iter_time=3.906e-04, forward_time=0.264, loss_att=204.305, acc=0.822, loss=204.305, backward_time=0.390, grad_norm=92.796, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=9.972e-04, train_time=3.380 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:19:56,213 (trainer:732) INFO: 22epoch:train:1289-1380batch: iter_time=4.279e-04, forward_time=0.262, loss_att=203.699, acc=0.820, loss=203.699, backward_time=0.389, grad_norm=111.372, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=9.995e-04, train_time=3.376 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:21:13,480 (trainer:732) INFO: 22epoch:train:1381-1472batch: iter_time=4.256e-04, forward_time=0.263, loss_att=198.266, acc=0.823, loss=198.266, backward_time=0.387, grad_norm=99.999, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.359 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:22:31,644 (trainer:732) INFO: 22epoch:train:1473-1564batch: iter_time=4.141e-04, forward_time=0.265, loss_att=209.572, acc=0.823, loss=209.572, backward_time=0.392, grad_norm=95.286, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.398 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:23:49,061 (trainer:732) INFO: 22epoch:train:1565-1656batch: iter_time=4.134e-04, forward_time=0.262, loss_att=202.246, acc=0.819, loss=202.246, backward_time=0.388, grad_norm=102.793, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.001, train_time=3.365 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:25:06,695 (trainer:732) INFO: 22epoch:train:1657-1748batch: iter_time=4.360e-04, forward_time=0.263, loss_att=196.504, acc=0.823, loss=196.504, backward_time=0.390, grad_norm=91.917, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.001, train_time=3.375 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:26:23,290 (trainer:732) INFO: 22epoch:train:1749-1840batch: iter_time=3.725e-04, forward_time=0.260, loss_att=205.362, acc=0.818, loss=205.362, backward_time=0.384, grad_norm=96.017, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.001, train_time=3.329 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:37:41,422 (trainer:338) INFO: 22epoch results: [train] iter_time=9.846e-04, forward_time=0.270, loss_att=202.111, acc=0.822, loss=202.111, backward_time=0.391, grad_norm=97.193, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=9.892e-04, train_time=3.947, time=30 minutes and 22.14 seconds, total_count=40546, gpu_max_cached_mem_GB=29.846, [valid] loss_att=240.725, acc=0.793, cer=0.262, wer=0.566, loss=240.725, time=4 minutes and 56.19 seconds, total_count=418, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 15.89 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:37:49,632 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:37:49,641 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/12epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:37:49,642 (trainer:272) INFO: 23/60epoch started. Estimated time to finish: 1 day, 3 hours and 1 minute +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:42:28,086 (trainer:732) INFO: 23epoch:train:1-92batch: iter_time=0.020, forward_time=0.271, loss_att=199.227, acc=0.828, loss=199.227, backward_time=0.392, grad_norm=114.941, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.001, train_time=12.105 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:43:45,394 (trainer:732) INFO: 23epoch:train:93-184batch: iter_time=4.627e-04, forward_time=0.264, loss_att=201.282, acc=0.826, loss=201.282, backward_time=0.387, grad_norm=95.781, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.360 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:45:02,997 (trainer:732) INFO: 23epoch:train:185-276batch: iter_time=4.543e-04, forward_time=0.263, loss_att=196.964, acc=0.828, loss=196.964, backward_time=0.389, grad_norm=95.578, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.373 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:46:20,152 (trainer:732) INFO: 23epoch:train:277-368batch: iter_time=4.283e-04, forward_time=0.262, loss_att=191.820, acc=0.826, loss=191.820, backward_time=0.386, grad_norm=88.045, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.001, train_time=3.354 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:47:38,006 (trainer:732) INFO: 23epoch:train:369-460batch: iter_time=4.695e-04, forward_time=0.264, loss_att=192.114, acc=0.827, loss=192.114, backward_time=0.390, grad_norm=86.991, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.384 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:48:55,961 (trainer:732) INFO: 23epoch:train:461-552batch: iter_time=4.035e-04, forward_time=0.265, loss_att=205.695, acc=0.826, loss=205.695, backward_time=0.393, grad_norm=100.122, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.001, train_time=3.389 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:50:13,563 (trainer:732) INFO: 23epoch:train:553-644batch: iter_time=3.832e-04, forward_time=0.263, loss_att=202.056, acc=0.826, loss=202.056, backward_time=0.389, grad_norm=89.698, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.001, train_time=3.373 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:51:30,909 (trainer:732) INFO: 23epoch:train:645-736batch: iter_time=4.593e-04, forward_time=0.263, loss_att=200.345, acc=0.823, loss=200.345, backward_time=0.390, grad_norm=105.328, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.362 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:52:47,208 (trainer:732) INFO: 23epoch:train:737-828batch: iter_time=4.204e-04, forward_time=0.258, loss_att=192.000, acc=0.821, loss=192.000, backward_time=0.381, grad_norm=83.362, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.317 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:54:05,204 (trainer:732) INFO: 23epoch:train:829-920batch: iter_time=4.207e-04, forward_time=0.265, loss_att=206.095, acc=0.824, loss=206.095, backward_time=0.391, grad_norm=93.949, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.390 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:55:22,552 (trainer:732) INFO: 23epoch:train:921-1012batch: iter_time=4.461e-04, forward_time=0.261, loss_att=193.038, acc=0.825, loss=193.038, backward_time=0.388, grad_norm=100.188, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.362 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:56:39,864 (trainer:732) INFO: 23epoch:train:1013-1104batch: iter_time=4.226e-04, forward_time=0.264, loss_att=204.148, acc=0.822, loss=204.148, backward_time=0.389, grad_norm=97.658, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.361 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:57:57,474 (trainer:732) INFO: 23epoch:train:1105-1196batch: iter_time=4.077e-04, forward_time=0.262, loss_att=201.549, acc=0.825, loss=201.549, backward_time=0.388, grad_norm=91.803, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.001, train_time=3.374 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 12:59:14,173 (trainer:732) INFO: 23epoch:train:1197-1288batch: iter_time=4.072e-04, forward_time=0.260, loss_att=192.191, acc=0.824, loss=192.191, backward_time=0.385, grad_norm=88.967, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.001, train_time=3.334 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:00:31,822 (trainer:732) INFO: 23epoch:train:1289-1380batch: iter_time=4.140e-04, forward_time=0.263, loss_att=197.867, acc=0.822, loss=197.867, backward_time=0.390, grad_norm=99.692, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.375 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:01:49,225 (trainer:732) INFO: 23epoch:train:1381-1472batch: iter_time=3.878e-04, forward_time=0.263, loss_att=197.462, acc=0.827, loss=197.462, backward_time=0.387, grad_norm=97.635, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.365 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:03:07,110 (trainer:732) INFO: 23epoch:train:1473-1564batch: iter_time=4.035e-04, forward_time=0.265, loss_att=202.598, acc=0.826, loss=202.598, backward_time=0.390, grad_norm=97.735, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.386 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:04:23,823 (trainer:732) INFO: 23epoch:train:1565-1656batch: iter_time=4.143e-04, forward_time=0.262, loss_att=192.733, acc=0.822, loss=192.733, backward_time=0.385, grad_norm=98.499, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.335 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:05:41,267 (trainer:732) INFO: 23epoch:train:1657-1748batch: iter_time=3.984e-04, forward_time=0.264, loss_att=201.766, acc=0.824, loss=201.766, backward_time=0.391, grad_norm=100.699, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.366 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:06:58,521 (trainer:732) INFO: 23epoch:train:1749-1840batch: iter_time=3.400e-04, forward_time=0.262, loss_att=199.134, acc=0.826, loss=199.134, backward_time=0.387, grad_norm=98.267, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:19:08,365 (trainer:338) INFO: 23epoch results: [train] iter_time=0.001, forward_time=0.263, loss_att=198.372, acc=0.825, loss=198.372, backward_time=0.388, grad_norm=96.247, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.801, time=29 minutes and 15.15 seconds, total_count=42389, gpu_max_cached_mem_GB=29.846, [valid] loss_att=237.598, acc=0.795, cer=0.255, wer=0.566, loss=237.598, time=5 minutes and 28.26 seconds, total_count=437, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 35.3 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:19:18,174 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:19:18,193 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/13epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:19:18,193 (trainer:272) INFO: 24/60epoch started. Estimated time to finish: 1 day, 2 hours and 17 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:25:10,127 (trainer:732) INFO: 24epoch:train:1-92batch: iter_time=0.013, forward_time=0.281, loss_att=197.935, acc=0.831, loss=197.935, backward_time=0.391, grad_norm=110.466, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.001, train_time=15.300 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:26:31,100 (trainer:732) INFO: 24epoch:train:93-184batch: iter_time=5.048e-04, forward_time=0.281, loss_att=189.842, acc=0.830, loss=189.842, backward_time=0.389, grad_norm=89.366, clip=100.000, loss_scale=1.000, optim_step_time=0.128, optim0_lr0=0.001, train_time=3.520 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:27:50,125 (trainer:732) INFO: 24epoch:train:185-276batch: iter_time=5.576e-04, forward_time=0.271, loss_att=184.313, acc=0.823, loss=184.313, backward_time=0.387, grad_norm=87.995, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=0.001, train_time=3.435 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:29:08,967 (trainer:732) INFO: 24epoch:train:277-368batch: iter_time=6.328e-04, forward_time=0.272, loss_att=189.503, acc=0.826, loss=189.503, backward_time=0.386, grad_norm=99.805, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=0.001, train_time=3.427 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:30:28,536 (trainer:732) INFO: 24epoch:train:369-460batch: iter_time=6.415e-04, forward_time=0.273, loss_att=193.953, acc=0.829, loss=193.953, backward_time=0.390, grad_norm=104.303, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.459 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:31:48,371 (trainer:732) INFO: 24epoch:train:461-552batch: iter_time=5.034e-04, forward_time=0.276, loss_att=191.661, acc=0.826, loss=191.661, backward_time=0.388, grad_norm=84.284, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.001, train_time=3.470 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:33:07,757 (trainer:732) INFO: 24epoch:train:553-644batch: iter_time=4.573e-04, forward_time=0.272, loss_att=197.008, acc=0.829, loss=197.008, backward_time=0.392, grad_norm=91.934, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.001, train_time=3.451 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:34:27,608 (trainer:732) INFO: 24epoch:train:645-736batch: iter_time=4.597e-04, forward_time=0.276, loss_att=189.553, acc=0.832, loss=189.553, backward_time=0.392, grad_norm=100.464, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=0.001, train_time=3.471 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:35:46,694 (trainer:732) INFO: 24epoch:train:737-828batch: iter_time=5.242e-04, forward_time=0.273, loss_att=195.994, acc=0.829, loss=195.994, backward_time=0.389, grad_norm=90.700, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.001, train_time=3.438 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:37:06,208 (trainer:732) INFO: 24epoch:train:829-920batch: iter_time=4.272e-04, forward_time=0.275, loss_att=200.002, acc=0.827, loss=200.002, backward_time=0.391, grad_norm=98.881, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.001, train_time=3.456 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:38:25,516 (trainer:732) INFO: 24epoch:train:921-1012batch: iter_time=5.580e-04, forward_time=0.272, loss_att=190.907, acc=0.829, loss=190.907, backward_time=0.392, grad_norm=93.843, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.001, train_time=3.447 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:39:45,170 (trainer:732) INFO: 24epoch:train:1013-1104batch: iter_time=6.136e-04, forward_time=0.276, loss_att=195.010, acc=0.832, loss=195.010, backward_time=0.389, grad_norm=95.390, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=0.001, train_time=3.462 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:41:05,148 (trainer:732) INFO: 24epoch:train:1105-1196batch: iter_time=4.355e-04, forward_time=0.275, loss_att=194.912, acc=0.828, loss=194.912, backward_time=0.393, grad_norm=93.754, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.001, train_time=3.477 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:42:26,737 (trainer:732) INFO: 24epoch:train:1197-1288batch: iter_time=6.394e-04, forward_time=0.282, loss_att=199.570, acc=0.831, loss=199.570, backward_time=0.397, grad_norm=96.135, clip=100.000, loss_scale=1.000, optim_step_time=0.131, optim0_lr0=0.001, train_time=3.547 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:43:45,762 (trainer:732) INFO: 24epoch:train:1289-1380batch: iter_time=6.774e-04, forward_time=0.270, loss_att=193.577, acc=0.826, loss=193.577, backward_time=0.388, grad_norm=89.291, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.001, train_time=3.435 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:45:07,585 (trainer:732) INFO: 24epoch:train:1381-1472batch: iter_time=4.834e-04, forward_time=0.287, loss_att=205.679, acc=0.829, loss=205.679, backward_time=0.396, grad_norm=106.328, clip=100.000, loss_scale=1.000, optim_step_time=0.144, optim0_lr0=0.001, train_time=3.557 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:46:26,554 (trainer:732) INFO: 24epoch:train:1473-1564batch: iter_time=4.650e-04, forward_time=0.268, loss_att=194.098, acc=0.825, loss=194.098, backward_time=0.390, grad_norm=91.852, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.001, train_time=3.433 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:47:46,030 (trainer:732) INFO: 24epoch:train:1565-1656batch: iter_time=5.500e-04, forward_time=0.272, loss_att=195.712, acc=0.830, loss=195.712, backward_time=0.393, grad_norm=100.657, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.001, train_time=3.455 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:49:06,288 (trainer:732) INFO: 24epoch:train:1657-1748batch: iter_time=5.061e-04, forward_time=0.276, loss_att=199.233, acc=0.824, loss=199.233, backward_time=0.394, grad_norm=93.073, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.001, train_time=3.488 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 13:50:25,893 (trainer:732) INFO: 24epoch:train:1749-1840batch: iter_time=4.056e-04, forward_time=0.273, loss_att=201.289, acc=0.827, loss=201.289, backward_time=0.392, grad_norm=99.379, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.001, train_time=3.461 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:02:41,833 (trainer:338) INFO: 24epoch results: [train] iter_time=0.001, forward_time=0.275, loss_att=194.897, acc=0.828, loss=194.897, backward_time=0.391, grad_norm=95.895, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.001, train_time=4.059, time=31 minutes and 14.56 seconds, total_count=44232, gpu_max_cached_mem_GB=29.846, [valid] loss_att=234.011, acc=0.798, cer=0.255, wer=0.561, loss=234.011, time=5 minutes and 10.1 seconds, total_count=456, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 58.97 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:02:51,654 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:02:51,679 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/14epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:02:51,680 (trainer:272) INFO: 25/60epoch started. Estimated time to finish: 1 day, 1 hour and 36 minutes +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:08:52,073 (trainer:732) INFO: 25epoch:train:1-92batch: iter_time=0.023, forward_time=0.281, loss_att=187.755, acc=0.830, loss=187.755, backward_time=0.390, grad_norm=90.733, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.001, train_time=15.668 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:10:11,731 (trainer:732) INFO: 25epoch:train:93-184batch: iter_time=4.814e-04, forward_time=0.274, loss_att=188.777, acc=0.828, loss=188.777, backward_time=0.394, grad_norm=91.364, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=0.001, train_time=3.463 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:11:31,089 (trainer:732) INFO: 25epoch:train:185-276batch: iter_time=5.240e-04, forward_time=0.272, loss_att=188.537, acc=0.830, loss=188.537, backward_time=0.389, grad_norm=90.348, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.001, train_time=3.450 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:12:50,932 (trainer:732) INFO: 25epoch:train:277-368batch: iter_time=4.705e-04, forward_time=0.271, loss_att=194.205, acc=0.830, loss=194.205, backward_time=0.390, grad_norm=94.511, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.001, train_time=3.471 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:14:11,009 (trainer:732) INFO: 25epoch:train:369-460batch: iter_time=5.233e-04, forward_time=0.276, loss_att=189.271, acc=0.833, loss=189.271, backward_time=0.390, grad_norm=101.590, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=0.001, train_time=3.481 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:15:30,894 (trainer:732) INFO: 25epoch:train:461-552batch: iter_time=5.248e-04, forward_time=0.275, loss_att=191.391, acc=0.832, loss=191.391, backward_time=0.390, grad_norm=89.824, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.001, train_time=3.473 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:16:50,167 (trainer:732) INFO: 25epoch:train:553-644batch: iter_time=4.426e-04, forward_time=0.271, loss_att=189.637, acc=0.830, loss=189.637, backward_time=0.388, grad_norm=83.828, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=0.001, train_time=3.446 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:18:09,855 (trainer:732) INFO: 25epoch:train:645-736batch: iter_time=4.609e-04, forward_time=0.275, loss_att=193.384, acc=0.830, loss=193.384, backward_time=0.390, grad_norm=102.264, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.001, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:19:29,399 (trainer:732) INFO: 25epoch:train:737-828batch: iter_time=5.319e-04, forward_time=0.272, loss_att=188.474, acc=0.831, loss=188.474, backward_time=0.394, grad_norm=90.755, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=0.001, train_time=3.458 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:20:49,261 (trainer:732) INFO: 25epoch:train:829-920batch: iter_time=5.303e-04, forward_time=0.276, loss_att=195.072, acc=0.831, loss=195.072, backward_time=0.388, grad_norm=101.503, clip=100.000, loss_scale=1.000, optim_step_time=0.128, optim0_lr0=0.001, train_time=3.472 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:22:08,954 (trainer:732) INFO: 25epoch:train:921-1012batch: iter_time=6.086e-04, forward_time=0.274, loss_att=190.791, acc=0.834, loss=190.791, backward_time=0.392, grad_norm=89.314, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.001, train_time=3.464 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:23:30,099 (trainer:732) INFO: 25epoch:train:1013-1104batch: iter_time=4.882e-04, forward_time=0.285, loss_att=191.257, acc=0.831, loss=191.257, backward_time=0.395, grad_norm=91.127, clip=100.000, loss_scale=1.000, optim_step_time=0.136, optim0_lr0=0.001, train_time=3.527 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:24:49,704 (trainer:732) INFO: 25epoch:train:1105-1196batch: iter_time=4.483e-04, forward_time=0.274, loss_att=191.061, acc=0.830, loss=191.061, backward_time=0.391, grad_norm=95.396, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=0.001, train_time=3.460 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:26:10,322 (trainer:732) INFO: 25epoch:train:1197-1288batch: iter_time=4.315e-04, forward_time=0.275, loss_att=198.974, acc=0.831, loss=198.974, backward_time=0.392, grad_norm=97.968, clip=100.000, loss_scale=1.000, optim_step_time=0.124, optim0_lr0=0.001, train_time=3.504 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:27:30,339 (trainer:732) INFO: 25epoch:train:1289-1380batch: iter_time=4.434e-04, forward_time=0.274, loss_att=188.985, acc=0.829, loss=188.985, backward_time=0.388, grad_norm=96.303, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=0.001, train_time=3.478 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:28:50,283 (trainer:732) INFO: 25epoch:train:1381-1472batch: iter_time=5.182e-04, forward_time=0.277, loss_att=194.061, acc=0.830, loss=194.061, backward_time=0.394, grad_norm=93.158, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=0.001, train_time=3.476 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:30:10,979 (trainer:732) INFO: 25epoch:train:1473-1564batch: iter_time=4.609e-04, forward_time=0.276, loss_att=192.791, acc=0.837, loss=192.791, backward_time=0.395, grad_norm=93.387, clip=100.000, loss_scale=1.000, optim_step_time=0.129, optim0_lr0=0.001, train_time=3.508 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:31:30,605 (trainer:732) INFO: 25epoch:train:1565-1656batch: iter_time=4.707e-04, forward_time=0.274, loss_att=187.442, acc=0.829, loss=187.442, backward_time=0.391, grad_norm=97.542, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=0.001, train_time=3.461 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:32:50,590 (trainer:732) INFO: 25epoch:train:1657-1748batch: iter_time=4.497e-04, forward_time=0.276, loss_att=191.975, acc=0.831, loss=191.975, backward_time=0.392, grad_norm=91.443, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.001, train_time=3.477 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:34:10,179 (trainer:732) INFO: 25epoch:train:1749-1840batch: iter_time=3.804e-04, forward_time=0.274, loss_att=196.902, acc=0.829, loss=196.902, backward_time=0.391, grad_norm=92.891, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.001, train_time=3.460 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:45:45,867 (trainer:338) INFO: 25epoch results: [train] iter_time=0.002, forward_time=0.275, loss_att=191.507, acc=0.831, loss=191.507, backward_time=0.391, grad_norm=93.762, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=0.001, train_time=4.083, time=31 minutes and 25.91 seconds, total_count=46075, gpu_max_cached_mem_GB=29.846, [valid] loss_att=235.311, acc=0.798, cer=0.251, wer=0.564, loss=235.311, time=5 minutes and 2.68 seconds, total_count=475, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 25.58 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:45:53,649 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:45:53,659 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/15epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:45:53,660 (trainer:272) INFO: 26/60epoch started. Estimated time to finish: 1 day, 53 minutes and 56.83 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:49:53,625 (trainer:732) INFO: 26epoch:train:1-92batch: iter_time=0.015, forward_time=0.259, loss_att=185.544, acc=0.831, loss=185.544, backward_time=0.382, grad_norm=92.172, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=10.432 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:51:10,432 (trainer:732) INFO: 26epoch:train:93-184batch: iter_time=3.836e-04, forward_time=0.259, loss_att=183.754, acc=0.835, loss=183.754, backward_time=0.387, grad_norm=92.453, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.339 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:52:27,150 (trainer:732) INFO: 26epoch:train:185-276batch: iter_time=3.861e-04, forward_time=0.261, loss_att=185.639, acc=0.836, loss=185.639, backward_time=0.387, grad_norm=99.860, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=0.001, train_time=3.335 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:53:43,681 (trainer:732) INFO: 26epoch:train:277-368batch: iter_time=3.937e-04, forward_time=0.259, loss_att=180.484, acc=0.833, loss=180.484, backward_time=0.385, grad_norm=93.580, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=0.001, train_time=3.327 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:55:00,057 (trainer:732) INFO: 26epoch:train:369-460batch: iter_time=3.729e-04, forward_time=0.259, loss_att=187.899, acc=0.838, loss=187.899, backward_time=0.385, grad_norm=88.807, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.320 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:56:16,275 (trainer:732) INFO: 26epoch:train:461-552batch: iter_time=3.577e-04, forward_time=0.259, loss_att=182.536, acc=0.834, loss=182.536, backward_time=0.385, grad_norm=89.816, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.001, train_time=3.313 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:57:33,704 (trainer:732) INFO: 26epoch:train:553-644batch: iter_time=3.558e-04, forward_time=0.262, loss_att=190.472, acc=0.840, loss=190.472, backward_time=0.391, grad_norm=95.031, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.366 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 14:58:50,519 (trainer:732) INFO: 26epoch:train:645-736batch: iter_time=3.421e-04, forward_time=0.260, loss_att=193.818, acc=0.834, loss=193.818, backward_time=0.387, grad_norm=97.683, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=0.001, train_time=3.339 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:00:06,946 (trainer:732) INFO: 26epoch:train:737-828batch: iter_time=3.412e-04, forward_time=0.259, loss_att=189.782, acc=0.833, loss=189.782, backward_time=0.386, grad_norm=89.700, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.322 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:01:23,285 (trainer:732) INFO: 26epoch:train:829-920batch: iter_time=5.184e-04, forward_time=0.259, loss_att=182.353, acc=0.832, loss=182.353, backward_time=0.384, grad_norm=95.184, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.001, train_time=3.318 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:02:39,278 (trainer:732) INFO: 26epoch:train:921-1012batch: iter_time=3.767e-04, forward_time=0.258, loss_att=188.000, acc=0.834, loss=188.000, backward_time=0.383, grad_norm=90.828, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.001, train_time=3.303 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:03:55,627 (trainer:732) INFO: 26epoch:train:1013-1104batch: iter_time=3.644e-04, forward_time=0.258, loss_att=188.422, acc=0.831, loss=188.422, backward_time=0.385, grad_norm=90.174, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.001, train_time=3.319 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:05:12,698 (trainer:732) INFO: 26epoch:train:1105-1196batch: iter_time=3.558e-04, forward_time=0.261, loss_att=184.958, acc=0.835, loss=184.958, backward_time=0.387, grad_norm=100.071, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.001, train_time=3.350 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:06:30,025 (trainer:732) INFO: 26epoch:train:1197-1288batch: iter_time=4.009e-04, forward_time=0.261, loss_att=195.220, acc=0.837, loss=195.220, backward_time=0.389, grad_norm=93.935, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=0.001, train_time=3.361 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:07:46,776 (trainer:732) INFO: 26epoch:train:1289-1380batch: iter_time=3.935e-04, forward_time=0.260, loss_att=193.461, acc=0.835, loss=193.461, backward_time=0.386, grad_norm=93.613, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.001, train_time=3.336 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:09:02,990 (trainer:732) INFO: 26epoch:train:1381-1472batch: iter_time=4.134e-04, forward_time=0.258, loss_att=183.757, acc=0.835, loss=183.757, backward_time=0.384, grad_norm=102.392, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=0.001, train_time=3.313 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:10:20,600 (trainer:732) INFO: 26epoch:train:1473-1564batch: iter_time=4.193e-04, forward_time=0.263, loss_att=193.854, acc=0.838, loss=193.854, backward_time=0.391, grad_norm=94.248, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.374 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:11:37,843 (trainer:732) INFO: 26epoch:train:1565-1656batch: iter_time=4.102e-04, forward_time=0.261, loss_att=195.733, acc=0.832, loss=195.733, backward_time=0.388, grad_norm=86.177, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.001, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:12:55,468 (trainer:732) INFO: 26epoch:train:1657-1748batch: iter_time=4.186e-04, forward_time=0.262, loss_att=185.129, acc=0.835, loss=185.129, backward_time=0.389, grad_norm=98.057, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.001, train_time=3.374 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:14:11,549 (trainer:732) INFO: 26epoch:train:1749-1840batch: iter_time=3.140e-04, forward_time=0.258, loss_att=186.230, acc=0.828, loss=186.230, backward_time=0.383, grad_norm=91.361, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.001, train_time=3.307 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:25:13,227 (trainer:338) INFO: 26epoch results: [train] iter_time=0.001, forward_time=0.260, loss_att=187.723, acc=0.834, loss=187.723, backward_time=0.386, grad_norm=93.757, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.690, time=28 minutes and 23.6 seconds, total_count=47918, gpu_max_cached_mem_GB=29.846, [valid] loss_att=229.553, acc=0.801, cer=0.246, wer=0.558, loss=229.553, time=4 minutes and 58.54 seconds, total_count=494, gpu_max_cached_mem_GB=29.846, [att_plot] time=5 minutes and 57.42 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:25:21,382 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:25:21,397 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/16epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:25:21,398 (trainer:272) INFO: 27/60epoch started. Estimated time to finish: 1 day, 7 minutes and 2.98 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:29:37,091 (trainer:732) INFO: 27epoch:train:1-92batch: iter_time=0.007, forward_time=0.271, loss_att=188.804, acc=0.843, loss=188.804, backward_time=0.393, grad_norm=99.328, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=11.115 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:30:55,246 (trainer:732) INFO: 27epoch:train:93-184batch: iter_time=4.189e-04, forward_time=0.266, loss_att=187.272, acc=0.840, loss=187.272, backward_time=0.393, grad_norm=88.261, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.001, train_time=3.397 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:32:12,129 (trainer:732) INFO: 27epoch:train:185-276batch: iter_time=4.169e-04, forward_time=0.261, loss_att=184.177, acc=0.838, loss=184.177, backward_time=0.386, grad_norm=89.303, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.342 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:33:28,645 (trainer:732) INFO: 27epoch:train:277-368batch: iter_time=4.000e-04, forward_time=0.260, loss_att=183.405, acc=0.838, loss=183.405, backward_time=0.385, grad_norm=91.389, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.001, train_time=3.326 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:34:45,344 (trainer:732) INFO: 27epoch:train:369-460batch: iter_time=4.423e-04, forward_time=0.260, loss_att=182.643, acc=0.837, loss=182.643, backward_time=0.387, grad_norm=93.508, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.001, train_time=3.334 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:36:02,270 (trainer:732) INFO: 27epoch:train:461-552batch: iter_time=4.818e-04, forward_time=0.259, loss_att=184.206, acc=0.836, loss=184.206, backward_time=0.385, grad_norm=92.090, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.344 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:37:18,404 (trainer:732) INFO: 27epoch:train:553-644batch: iter_time=3.867e-04, forward_time=0.260, loss_att=180.906, acc=0.835, loss=180.906, backward_time=0.383, grad_norm=95.181, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.309 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:38:35,819 (trainer:732) INFO: 27epoch:train:645-736batch: iter_time=3.780e-04, forward_time=0.263, loss_att=182.458, acc=0.837, loss=182.458, backward_time=0.390, grad_norm=97.380, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.001, train_time=3.365 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:39:52,835 (trainer:732) INFO: 27epoch:train:737-828batch: iter_time=4.221e-04, forward_time=0.263, loss_att=181.567, acc=0.834, loss=181.567, backward_time=0.389, grad_norm=105.386, clip=100.000, loss_scale=1.000, optim_step_time=0.097, optim0_lr0=0.001, train_time=3.348 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:41:09,139 (trainer:732) INFO: 27epoch:train:829-920batch: iter_time=4.209e-04, forward_time=0.258, loss_att=179.928, acc=0.838, loss=179.928, backward_time=0.383, grad_norm=81.512, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.317 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:42:26,107 (trainer:732) INFO: 27epoch:train:921-1012batch: iter_time=3.732e-04, forward_time=0.261, loss_att=190.078, acc=0.835, loss=190.078, backward_time=0.388, grad_norm=92.457, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.346 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:43:43,052 (trainer:732) INFO: 27epoch:train:1013-1104batch: iter_time=3.913e-04, forward_time=0.261, loss_att=190.020, acc=0.835, loss=190.020, backward_time=0.387, grad_norm=101.919, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.001, train_time=3.345 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:45:00,306 (trainer:732) INFO: 27epoch:train:1105-1196batch: iter_time=4.327e-04, forward_time=0.263, loss_att=185.146, acc=0.837, loss=185.146, backward_time=0.386, grad_norm=94.438, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.001, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:46:18,083 (trainer:732) INFO: 27epoch:train:1197-1288batch: iter_time=4.997e-04, forward_time=0.265, loss_att=175.777, acc=0.833, loss=175.777, backward_time=0.385, grad_norm=85.875, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.001, train_time=3.381 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:47:35,710 (trainer:732) INFO: 27epoch:train:1289-1380batch: iter_time=4.076e-04, forward_time=0.265, loss_att=181.821, acc=0.838, loss=181.821, backward_time=0.386, grad_norm=92.326, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.374 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:48:54,788 (trainer:732) INFO: 27epoch:train:1381-1472batch: iter_time=4.207e-04, forward_time=0.270, loss_att=191.549, acc=0.840, loss=191.549, backward_time=0.394, grad_norm=99.624, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.437 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:50:13,659 (trainer:732) INFO: 27epoch:train:1473-1564batch: iter_time=3.890e-04, forward_time=0.272, loss_att=184.479, acc=0.834, loss=184.479, backward_time=0.389, grad_norm=94.210, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.001, train_time=3.429 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:51:32,778 (trainer:732) INFO: 27epoch:train:1565-1656batch: iter_time=4.968e-04, forward_time=0.271, loss_att=185.660, acc=0.840, loss=185.660, backward_time=0.391, grad_norm=93.962, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.001, train_time=3.439 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:52:51,762 (trainer:732) INFO: 27epoch:train:1657-1748batch: iter_time=4.684e-04, forward_time=0.270, loss_att=186.082, acc=0.839, loss=186.082, backward_time=0.390, grad_norm=89.935, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.001, train_time=3.433 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 15:54:09,913 (trainer:732) INFO: 27epoch:train:1749-1840batch: iter_time=3.795e-04, forward_time=0.265, loss_att=178.268, acc=0.839, loss=178.268, backward_time=0.385, grad_norm=92.682, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=0.001, train_time=3.397 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:06:04,111 (trainer:338) INFO: 27epoch results: [train] iter_time=7.354e-04, forward_time=0.264, loss_att=184.063, acc=0.837, loss=184.063, backward_time=0.388, grad_norm=93.538, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.757, time=28 minutes and 54.71 seconds, total_count=49761, gpu_max_cached_mem_GB=29.846, [valid] loss_att=230.913, acc=0.803, cer=0.250, wer=0.553, loss=230.913, time=5 minutes and 20.07 seconds, total_count=513, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 27.92 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:06:11,959 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:06:11,970 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/17epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:06:11,970 (trainer:272) INFO: 28/60epoch started. Estimated time to finish: 23 hours, 22 minutes and 23.42 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:10:43,313 (trainer:732) INFO: 28epoch:train:1-92batch: iter_time=0.011, forward_time=0.270, loss_att=184.776, acc=0.843, loss=184.776, backward_time=0.393, grad_norm=95.093, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.001, train_time=11.796 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:12:00,108 (trainer:732) INFO: 28epoch:train:93-184batch: iter_time=4.449e-04, forward_time=0.261, loss_att=175.594, acc=0.840, loss=175.594, backward_time=0.385, grad_norm=86.827, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.338 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:13:17,635 (trainer:732) INFO: 28epoch:train:185-276batch: iter_time=5.321e-04, forward_time=0.263, loss_att=178.299, acc=0.840, loss=178.299, backward_time=0.388, grad_norm=94.819, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.370 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:14:34,410 (trainer:732) INFO: 28epoch:train:277-368batch: iter_time=4.416e-04, forward_time=0.261, loss_att=178.231, acc=0.842, loss=178.231, backward_time=0.385, grad_norm=86.637, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.337 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:15:51,760 (trainer:732) INFO: 28epoch:train:369-460batch: iter_time=4.548e-04, forward_time=0.263, loss_att=183.452, acc=0.843, loss=183.452, backward_time=0.390, grad_norm=93.153, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.001, train_time=3.362 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:17:08,277 (trainer:732) INFO: 28epoch:train:461-552batch: iter_time=4.346e-04, forward_time=0.261, loss_att=179.251, acc=0.839, loss=179.251, backward_time=0.384, grad_norm=99.943, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.001, train_time=3.326 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:18:24,805 (trainer:732) INFO: 28epoch:train:553-644batch: iter_time=3.807e-04, forward_time=0.258, loss_att=174.484, acc=0.843, loss=174.484, backward_time=0.384, grad_norm=91.772, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.327 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:19:41,536 (trainer:732) INFO: 28epoch:train:645-736batch: iter_time=3.857e-04, forward_time=0.261, loss_att=175.773, acc=0.841, loss=175.773, backward_time=0.386, grad_norm=90.549, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.001, train_time=3.335 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:20:58,577 (trainer:732) INFO: 28epoch:train:737-828batch: iter_time=4.137e-04, forward_time=0.262, loss_att=181.452, acc=0.837, loss=181.452, backward_time=0.388, grad_norm=94.646, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.349 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:22:16,556 (trainer:732) INFO: 28epoch:train:829-920batch: iter_time=4.259e-04, forward_time=0.265, loss_att=187.937, acc=0.843, loss=187.937, backward_time=0.392, grad_norm=89.253, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.390 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:23:33,640 (trainer:732) INFO: 28epoch:train:921-1012batch: iter_time=4.244e-04, forward_time=0.263, loss_att=185.434, acc=0.842, loss=185.434, backward_time=0.391, grad_norm=93.847, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.001, train_time=3.351 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:24:50,339 (trainer:732) INFO: 28epoch:train:1013-1104batch: iter_time=4.021e-04, forward_time=0.259, loss_att=179.185, acc=0.840, loss=179.185, backward_time=0.384, grad_norm=95.385, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.334 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:26:07,707 (trainer:732) INFO: 28epoch:train:1105-1196batch: iter_time=3.580e-04, forward_time=0.262, loss_att=184.357, acc=0.845, loss=184.357, backward_time=0.391, grad_norm=92.348, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.363 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:27:24,706 (trainer:732) INFO: 28epoch:train:1197-1288batch: iter_time=3.727e-04, forward_time=0.261, loss_att=179.880, acc=0.840, loss=179.880, backward_time=0.387, grad_norm=92.024, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.001, train_time=3.347 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:28:41,945 (trainer:732) INFO: 28epoch:train:1289-1380batch: iter_time=3.738e-04, forward_time=0.260, loss_att=177.055, acc=0.841, loss=177.055, backward_time=0.388, grad_norm=86.152, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:29:59,837 (trainer:732) INFO: 28epoch:train:1381-1472batch: iter_time=3.800e-04, forward_time=0.263, loss_att=183.881, acc=0.841, loss=183.881, backward_time=0.391, grad_norm=95.993, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.386 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:31:17,788 (trainer:732) INFO: 28epoch:train:1473-1564batch: iter_time=5.762e-04, forward_time=0.267, loss_att=173.555, acc=0.840, loss=173.555, backward_time=0.385, grad_norm=86.393, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.001, train_time=3.388 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:32:35,443 (trainer:732) INFO: 28epoch:train:1565-1656batch: iter_time=4.120e-04, forward_time=0.263, loss_att=185.656, acc=0.840, loss=185.656, backward_time=0.386, grad_norm=88.157, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.001, train_time=3.376 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:33:53,232 (trainer:732) INFO: 28epoch:train:1657-1748batch: iter_time=4.768e-04, forward_time=0.264, loss_att=175.678, acc=0.842, loss=175.678, backward_time=0.387, grad_norm=104.955, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.382 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:35:10,743 (trainer:732) INFO: 28epoch:train:1749-1840batch: iter_time=4.989e-04, forward_time=0.263, loss_att=177.003, acc=0.838, loss=177.003, backward_time=0.385, grad_norm=99.135, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.001, train_time=3.369 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:47:43,548 (trainer:338) INFO: 28epoch results: [train] iter_time=9.361e-04, forward_time=0.263, loss_att=179.863, acc=0.841, loss=179.863, backward_time=0.387, grad_norm=92.854, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.779, time=29 minutes and 5.85 seconds, total_count=51604, gpu_max_cached_mem_GB=29.846, [valid] loss_att=224.341, acc=0.808, cer=0.242, wer=0.549, loss=224.341, time=5 minutes and 6.05 seconds, total_count=532, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 19.67 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:47:55,988 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:47:56,006 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/18epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:47:56,007 (trainer:272) INFO: 29/60epoch started. Estimated time to finish: 22 hours, 39 minutes and 1.32 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:53:14,955 (trainer:732) INFO: 29epoch:train:1-92batch: iter_time=0.029, forward_time=0.277, loss_att=170.755, acc=0.844, loss=170.755, backward_time=0.389, grad_norm=90.456, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=13.866 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:54:32,931 (trainer:732) INFO: 29epoch:train:93-184batch: iter_time=5.057e-04, forward_time=0.264, loss_att=176.661, acc=0.847, loss=176.661, backward_time=0.388, grad_norm=91.741, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.001, train_time=3.390 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:55:51,346 (trainer:732) INFO: 29epoch:train:185-276batch: iter_time=4.862e-04, forward_time=0.266, loss_att=173.953, acc=0.847, loss=173.953, backward_time=0.389, grad_norm=102.572, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.001, train_time=3.409 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:57:09,137 (trainer:732) INFO: 29epoch:train:277-368batch: iter_time=4.905e-04, forward_time=0.266, loss_att=178.868, acc=0.844, loss=178.868, backward_time=0.387, grad_norm=98.063, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.382 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:58:27,170 (trainer:732) INFO: 29epoch:train:369-460batch: iter_time=4.448e-04, forward_time=0.265, loss_att=168.369, acc=0.850, loss=168.369, backward_time=0.387, grad_norm=97.789, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.001, train_time=3.392 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 16:59:45,623 (trainer:732) INFO: 29epoch:train:461-552batch: iter_time=4.664e-04, forward_time=0.268, loss_att=179.841, acc=0.840, loss=179.841, backward_time=0.388, grad_norm=92.918, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.001, train_time=3.410 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:01:03,385 (trainer:732) INFO: 29epoch:train:553-644batch: iter_time=5.294e-04, forward_time=0.266, loss_att=173.469, acc=0.841, loss=173.469, backward_time=0.386, grad_norm=82.258, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.001, train_time=3.380 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:02:21,538 (trainer:732) INFO: 29epoch:train:645-736batch: iter_time=4.111e-04, forward_time=0.265, loss_att=173.549, acc=0.846, loss=173.549, backward_time=0.388, grad_norm=86.830, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.001, train_time=3.397 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:03:39,314 (trainer:732) INFO: 29epoch:train:737-828batch: iter_time=4.331e-04, forward_time=0.266, loss_att=173.847, acc=0.845, loss=173.847, backward_time=0.386, grad_norm=98.472, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.381 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:04:56,936 (trainer:732) INFO: 29epoch:train:829-920batch: iter_time=4.779e-04, forward_time=0.265, loss_att=175.168, acc=0.843, loss=175.168, backward_time=0.386, grad_norm=97.731, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.374 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:06:15,110 (trainer:732) INFO: 29epoch:train:921-1012batch: iter_time=4.408e-04, forward_time=0.269, loss_att=173.284, acc=0.843, loss=173.284, backward_time=0.388, grad_norm=97.762, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.398 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:07:33,675 (trainer:732) INFO: 29epoch:train:1013-1104batch: iter_time=5.095e-04, forward_time=0.269, loss_att=174.171, acc=0.847, loss=174.171, backward_time=0.385, grad_norm=89.483, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.415 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:08:51,257 (trainer:732) INFO: 29epoch:train:1105-1196batch: iter_time=4.065e-04, forward_time=0.265, loss_att=174.150, acc=0.840, loss=174.150, backward_time=0.383, grad_norm=89.943, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.001, train_time=3.372 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:10:10,354 (trainer:732) INFO: 29epoch:train:1197-1288batch: iter_time=4.619e-04, forward_time=0.269, loss_att=182.509, acc=0.847, loss=182.509, backward_time=0.393, grad_norm=100.974, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.001, train_time=3.438 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:11:28,375 (trainer:732) INFO: 29epoch:train:1289-1380batch: iter_time=4.250e-04, forward_time=0.267, loss_att=174.624, acc=0.847, loss=174.624, backward_time=0.389, grad_norm=91.882, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:12:46,923 (trainer:732) INFO: 29epoch:train:1381-1472batch: iter_time=4.399e-04, forward_time=0.267, loss_att=172.139, acc=0.844, loss=172.139, backward_time=0.386, grad_norm=97.412, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.001, train_time=3.414 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:14:06,221 (trainer:732) INFO: 29epoch:train:1473-1564batch: iter_time=4.782e-04, forward_time=0.273, loss_att=178.573, acc=0.845, loss=178.573, backward_time=0.393, grad_norm=96.265, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.447 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:15:24,539 (trainer:732) INFO: 29epoch:train:1565-1656batch: iter_time=5.383e-04, forward_time=0.268, loss_att=176.084, acc=0.848, loss=176.084, backward_time=0.389, grad_norm=95.275, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.001, train_time=3.404 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:16:42,424 (trainer:732) INFO: 29epoch:train:1657-1748batch: iter_time=4.336e-04, forward_time=0.265, loss_att=175.549, acc=0.845, loss=175.549, backward_time=0.389, grad_norm=94.692, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.386 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:18:00,725 (trainer:732) INFO: 29epoch:train:1749-1840batch: iter_time=3.130e-04, forward_time=0.268, loss_att=182.575, acc=0.843, loss=182.575, backward_time=0.391, grad_norm=98.899, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.001, train_time=3.404 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:29:47,829 (trainer:338) INFO: 29epoch results: [train] iter_time=0.002, forward_time=0.267, loss_att=175.375, acc=0.845, loss=175.375, backward_time=0.388, grad_norm=94.571, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.922, time=30 minutes and 11.52 seconds, total_count=53447, gpu_max_cached_mem_GB=29.846, [valid] loss_att=219.170, acc=0.813, cer=0.236, wer=0.544, loss=219.170, time=5 minutes and 1.09 seconds, total_count=551, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 39.21 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:29:55,450 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:29:55,471 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/19epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:29:55,471 (trainer:272) INFO: 30/60epoch started. Estimated time to finish: 21 hours, 56 minutes and 2.47 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:35:02,997 (trainer:732) INFO: 30epoch:train:1-92batch: iter_time=0.020, forward_time=0.269, loss_att=164.886, acc=0.846, loss=164.886, backward_time=0.384, grad_norm=90.567, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=13.369 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:36:20,487 (trainer:732) INFO: 30epoch:train:93-184batch: iter_time=4.964e-04, forward_time=0.265, loss_att=164.301, acc=0.849, loss=164.301, backward_time=0.384, grad_norm=87.120, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.368 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:37:39,130 (trainer:732) INFO: 30epoch:train:185-276batch: iter_time=4.329e-04, forward_time=0.269, loss_att=174.868, acc=0.851, loss=174.868, backward_time=0.391, grad_norm=97.241, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.419 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:38:58,105 (trainer:732) INFO: 30epoch:train:277-368batch: iter_time=5.580e-04, forward_time=0.270, loss_att=167.247, acc=0.853, loss=167.247, backward_time=0.391, grad_norm=90.903, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.001, train_time=3.433 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:40:17,526 (trainer:732) INFO: 30epoch:train:369-460batch: iter_time=5.961e-04, forward_time=0.271, loss_att=174.416, acc=0.851, loss=174.416, backward_time=0.393, grad_norm=94.797, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.001, train_time=3.452 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:41:35,815 (trainer:732) INFO: 30epoch:train:461-552batch: iter_time=4.366e-04, forward_time=0.268, loss_att=173.913, acc=0.849, loss=173.913, backward_time=0.389, grad_norm=90.407, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.403 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:42:54,556 (trainer:732) INFO: 30epoch:train:553-644batch: iter_time=4.395e-04, forward_time=0.269, loss_att=175.551, acc=0.848, loss=175.551, backward_time=0.390, grad_norm=105.172, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.001, train_time=3.423 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:44:13,415 (trainer:732) INFO: 30epoch:train:645-736batch: iter_time=4.979e-04, forward_time=0.269, loss_att=169.814, acc=0.847, loss=169.814, backward_time=0.389, grad_norm=111.223, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.001, train_time=3.428 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:45:32,140 (trainer:732) INFO: 30epoch:train:737-828batch: iter_time=4.576e-04, forward_time=0.269, loss_att=173.057, acc=0.846, loss=173.057, backward_time=0.388, grad_norm=83.970, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.001, train_time=3.422 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:46:50,465 (trainer:732) INFO: 30epoch:train:829-920batch: iter_time=4.760e-04, forward_time=0.268, loss_att=169.961, acc=0.845, loss=169.961, backward_time=0.386, grad_norm=90.418, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=0.001, train_time=3.405 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:48:08,599 (trainer:732) INFO: 30epoch:train:921-1012batch: iter_time=5.195e-04, forward_time=0.266, loss_att=166.303, acc=0.851, loss=166.303, backward_time=0.389, grad_norm=89.005, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.001, train_time=3.396 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:49:27,463 (trainer:732) INFO: 30epoch:train:1013-1104batch: iter_time=5.987e-04, forward_time=0.270, loss_att=167.103, acc=0.849, loss=167.103, backward_time=0.389, grad_norm=103.225, clip=100.000, loss_scale=1.000, optim_step_time=0.123, optim0_lr0=0.001, train_time=3.426 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:50:45,722 (trainer:732) INFO: 30epoch:train:1105-1196batch: iter_time=4.862e-04, forward_time=0.268, loss_att=169.874, acc=0.852, loss=169.874, backward_time=0.386, grad_norm=87.367, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.402 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:52:04,313 (trainer:732) INFO: 30epoch:train:1197-1288batch: iter_time=5.050e-04, forward_time=0.268, loss_att=178.036, acc=0.850, loss=178.036, backward_time=0.391, grad_norm=101.308, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.416 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:53:22,979 (trainer:732) INFO: 30epoch:train:1289-1380batch: iter_time=4.392e-04, forward_time=0.270, loss_att=173.059, acc=0.847, loss=173.059, backward_time=0.390, grad_norm=92.475, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.420 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:54:40,675 (trainer:732) INFO: 30epoch:train:1381-1472batch: iter_time=5.111e-04, forward_time=0.267, loss_att=169.560, acc=0.841, loss=169.560, backward_time=0.384, grad_norm=96.715, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.001, train_time=3.377 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:55:59,117 (trainer:732) INFO: 30epoch:train:1473-1564batch: iter_time=4.900e-04, forward_time=0.268, loss_att=176.494, acc=0.842, loss=176.494, backward_time=0.389, grad_norm=96.073, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.001, train_time=3.410 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:57:18,155 (trainer:732) INFO: 30epoch:train:1565-1656batch: iter_time=4.304e-04, forward_time=0.270, loss_att=179.615, acc=0.849, loss=179.615, backward_time=0.393, grad_norm=109.593, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.001, train_time=3.436 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:58:36,202 (trainer:732) INFO: 30epoch:train:1657-1748batch: iter_time=4.213e-04, forward_time=0.265, loss_att=170.948, acc=0.849, loss=170.948, backward_time=0.389, grad_norm=101.559, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.393 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 17:59:54,371 (trainer:732) INFO: 30epoch:train:1749-1840batch: iter_time=3.612e-04, forward_time=0.265, loss_att=172.981, acc=0.848, loss=172.981, backward_time=0.388, grad_norm=87.386, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.001, train_time=3.398 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:11:44,031 (trainer:338) INFO: 30epoch results: [train] iter_time=0.001, forward_time=0.268, loss_att=171.445, acc=0.848, loss=171.445, backward_time=0.389, grad_norm=95.326, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.001, train_time=3.910, time=30 minutes and 6.21 seconds, total_count=55290, gpu_max_cached_mem_GB=29.846, [valid] loss_att=217.368, acc=0.814, cer=0.232, wer=0.540, loss=217.368, time=5 minutes and 5.48 seconds, total_count=570, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 36.85 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:11:52,752 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:11:52,763 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/20epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:11:52,764 (trainer:272) INFO: 31/60epoch started. Estimated time to finish: 21 hours, 13 minutes and 5.41 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:17:01,142 (trainer:732) INFO: 31epoch:train:1-92batch: iter_time=0.017, forward_time=0.277, loss_att=168.393, acc=0.853, loss=168.393, backward_time=0.393, grad_norm=113.187, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=13.406 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:18:20,926 (trainer:732) INFO: 31epoch:train:93-184batch: iter_time=4.683e-04, forward_time=0.268, loss_att=166.399, acc=0.853, loss=166.399, backward_time=0.390, grad_norm=91.025, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.001, train_time=3.468 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:19:39,269 (trainer:732) INFO: 31epoch:train:185-276batch: iter_time=5.390e-04, forward_time=0.267, loss_att=161.770, acc=0.856, loss=161.770, backward_time=0.386, grad_norm=94.692, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.405 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:20:56,828 (trainer:732) INFO: 31epoch:train:277-368batch: iter_time=5.147e-04, forward_time=0.265, loss_att=170.136, acc=0.844, loss=170.136, backward_time=0.386, grad_norm=84.934, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.001, train_time=3.371 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:22:14,692 (trainer:732) INFO: 31epoch:train:369-460batch: iter_time=4.476e-04, forward_time=0.265, loss_att=163.975, acc=0.852, loss=163.975, backward_time=0.386, grad_norm=84.404, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.001, train_time=3.385 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:23:33,028 (trainer:732) INFO: 31epoch:train:461-552batch: iter_time=5.253e-04, forward_time=0.269, loss_att=172.100, acc=0.850, loss=172.100, backward_time=0.390, grad_norm=88.669, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.405 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:24:52,366 (trainer:732) INFO: 31epoch:train:553-644batch: iter_time=4.661e-04, forward_time=0.270, loss_att=176.000, acc=0.855, loss=176.000, backward_time=0.392, grad_norm=92.107, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.001, train_time=3.449 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:26:10,809 (trainer:732) INFO: 31epoch:train:645-736batch: iter_time=4.520e-04, forward_time=0.267, loss_att=171.213, acc=0.848, loss=171.213, backward_time=0.389, grad_norm=92.035, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.001, train_time=3.408 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:27:29,493 (trainer:732) INFO: 31epoch:train:737-828batch: iter_time=4.765e-04, forward_time=0.268, loss_att=171.307, acc=0.850, loss=171.307, backward_time=0.390, grad_norm=89.443, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.001, train_time=3.420 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:28:48,596 (trainer:732) INFO: 31epoch:train:829-920batch: iter_time=5.358e-04, forward_time=0.270, loss_att=166.304, acc=0.857, loss=166.304, backward_time=0.393, grad_norm=93.222, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.001, train_time=3.439 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:30:05,633 (trainer:732) INFO: 31epoch:train:921-1012batch: iter_time=4.768e-04, forward_time=0.262, loss_att=163.513, acc=0.847, loss=163.513, backward_time=0.383, grad_norm=99.846, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.001, train_time=3.349 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:31:23,362 (trainer:732) INFO: 31epoch:train:1013-1104batch: iter_time=4.305e-04, forward_time=0.263, loss_att=165.249, acc=0.852, loss=165.249, backward_time=0.389, grad_norm=91.126, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=3.379 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:32:41,135 (trainer:732) INFO: 31epoch:train:1105-1196batch: iter_time=3.781e-04, forward_time=0.263, loss_att=169.776, acc=0.852, loss=169.776, backward_time=0.390, grad_norm=93.926, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.381 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:33:58,384 (trainer:732) INFO: 31epoch:train:1197-1288batch: iter_time=3.947e-04, forward_time=0.262, loss_att=167.366, acc=0.852, loss=167.366, backward_time=0.388, grad_norm=94.875, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:35:14,731 (trainer:732) INFO: 31epoch:train:1289-1380batch: iter_time=4.067e-04, forward_time=0.258, loss_att=167.534, acc=0.850, loss=167.534, backward_time=0.385, grad_norm=99.207, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.001, train_time=3.319 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:36:31,708 (trainer:732) INFO: 31epoch:train:1381-1472batch: iter_time=3.910e-04, forward_time=0.261, loss_att=162.903, acc=0.854, loss=162.903, backward_time=0.387, grad_norm=87.535, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.001, train_time=3.346 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:37:48,745 (trainer:732) INFO: 31epoch:train:1473-1564batch: iter_time=3.730e-04, forward_time=0.261, loss_att=171.902, acc=0.853, loss=171.902, backward_time=0.388, grad_norm=91.364, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.001, train_time=3.349 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:39:05,191 (trainer:732) INFO: 31epoch:train:1565-1656batch: iter_time=3.678e-04, forward_time=0.259, loss_att=165.709, acc=0.854, loss=165.709, backward_time=0.384, grad_norm=94.772, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.323 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:40:22,943 (trainer:732) INFO: 31epoch:train:1657-1748batch: iter_time=3.974e-04, forward_time=0.264, loss_att=172.469, acc=0.854, loss=172.469, backward_time=0.391, grad_norm=101.088, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.001, train_time=3.380 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:41:38,659 (trainer:732) INFO: 31epoch:train:1749-1840batch: iter_time=3.249e-04, forward_time=0.256, loss_att=156.633, acc=0.848, loss=156.633, backward_time=0.380, grad_norm=88.498, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.001, train_time=3.291 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:52:28,096 (trainer:338) INFO: 31epoch results: [train] iter_time=0.001, forward_time=0.265, loss_att=167.420, acc=0.852, loss=167.420, backward_time=0.388, grad_norm=93.298, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.001, train_time=3.882, time=29 minutes and 51.85 seconds, total_count=57133, gpu_max_cached_mem_GB=29.846, [valid] loss_att=210.749, acc=0.819, cer=0.227, wer=0.534, loss=210.749, time=4 minutes and 36.33 seconds, total_count=589, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 7.15 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:52:35,325 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:52:35,336 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/21epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:52:35,337 (trainer:272) INFO: 32/60epoch started. Estimated time to finish: 20 hours, 29 minutes and 2.31 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:56:37,825 (trainer:732) INFO: 32epoch:train:1-92batch: iter_time=0.011, forward_time=0.266, loss_att=163.342, acc=0.859, loss=163.342, backward_time=0.390, grad_norm=97.105, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.001, train_time=10.542 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:57:54,453 (trainer:732) INFO: 32epoch:train:93-184batch: iter_time=4.113e-04, forward_time=0.259, loss_att=157.756, acc=0.854, loss=157.756, backward_time=0.384, grad_norm=93.519, clip=100.000, loss_scale=1.000, optim_step_time=0.097, optim0_lr0=0.001, train_time=3.331 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 18:59:11,038 (trainer:732) INFO: 32epoch:train:185-276batch: iter_time=4.022e-04, forward_time=0.260, loss_att=165.123, acc=0.856, loss=165.123, backward_time=0.387, grad_norm=85.387, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.329 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:00:27,920 (trainer:732) INFO: 32epoch:train:277-368batch: iter_time=3.852e-04, forward_time=0.260, loss_att=158.609, acc=0.858, loss=158.609, backward_time=0.387, grad_norm=91.528, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.001, train_time=3.342 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:01:43,901 (trainer:732) INFO: 32epoch:train:369-460batch: iter_time=3.919e-04, forward_time=0.258, loss_att=158.219, acc=0.853, loss=158.219, backward_time=0.383, grad_norm=86.448, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=0.001, train_time=3.303 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:03:01,212 (trainer:732) INFO: 32epoch:train:461-552batch: iter_time=3.679e-04, forward_time=0.263, loss_att=162.830, acc=0.862, loss=162.830, backward_time=0.391, grad_norm=95.072, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.001, train_time=3.361 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:04:18,301 (trainer:732) INFO: 32epoch:train:553-644batch: iter_time=3.751e-04, forward_time=0.261, loss_att=162.583, acc=0.858, loss=162.583, backward_time=0.388, grad_norm=93.666, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.001, train_time=3.351 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:05:35,416 (trainer:732) INFO: 32epoch:train:645-736batch: iter_time=4.765e-04, forward_time=0.262, loss_att=163.860, acc=0.856, loss=163.860, backward_time=0.389, grad_norm=91.783, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.001, train_time=3.352 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:06:52,013 (trainer:732) INFO: 32epoch:train:737-828batch: iter_time=3.857e-04, forward_time=0.259, loss_att=162.408, acc=0.853, loss=162.408, backward_time=0.386, grad_norm=95.705, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.330 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:08:08,783 (trainer:732) INFO: 32epoch:train:829-920batch: iter_time=3.743e-04, forward_time=0.260, loss_att=167.766, acc=0.857, loss=167.766, backward_time=0.389, grad_norm=96.098, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.001, train_time=3.337 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:09:24,298 (trainer:732) INFO: 32epoch:train:921-1012batch: iter_time=3.820e-04, forward_time=0.257, loss_att=163.099, acc=0.851, loss=163.099, backward_time=0.383, grad_norm=89.710, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=0.001, train_time=3.283 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:10:39,850 (trainer:732) INFO: 32epoch:train:1013-1104batch: iter_time=3.857e-04, forward_time=0.256, loss_att=164.068, acc=0.854, loss=164.068, backward_time=0.384, grad_norm=95.421, clip=100.000, loss_scale=1.000, optim_step_time=0.095, optim0_lr0=0.001, train_time=3.284 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:11:55,496 (trainer:732) INFO: 32epoch:train:1105-1196batch: iter_time=3.708e-04, forward_time=0.258, loss_att=165.472, acc=0.851, loss=165.472, backward_time=0.384, grad_norm=96.504, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=0.001, train_time=3.288 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:13:10,838 (trainer:732) INFO: 32epoch:train:1197-1288batch: iter_time=3.519e-04, forward_time=0.255, loss_att=159.393, acc=0.852, loss=159.393, backward_time=0.382, grad_norm=90.655, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=0.001, train_time=3.275 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:14:27,813 (trainer:732) INFO: 32epoch:train:1289-1380batch: iter_time=3.781e-04, forward_time=0.261, loss_att=167.463, acc=0.861, loss=167.463, backward_time=0.391, grad_norm=99.204, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.001, train_time=3.346 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:15:44,224 (trainer:732) INFO: 32epoch:train:1381-1472batch: iter_time=3.831e-04, forward_time=0.259, loss_att=157.983, acc=0.856, loss=157.983, backward_time=0.387, grad_norm=100.590, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.001, train_time=3.322 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:17:00,204 (trainer:732) INFO: 32epoch:train:1473-1564batch: iter_time=3.673e-04, forward_time=0.259, loss_att=166.601, acc=0.854, loss=166.601, backward_time=0.387, grad_norm=95.508, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=0.001, train_time=3.303 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:18:16,036 (trainer:732) INFO: 32epoch:train:1565-1656batch: iter_time=3.605e-04, forward_time=0.257, loss_att=168.601, acc=0.854, loss=168.601, backward_time=0.385, grad_norm=99.199, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.001, train_time=3.296 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:19:31,928 (trainer:732) INFO: 32epoch:train:1657-1748batch: iter_time=3.689e-04, forward_time=0.257, loss_att=158.948, acc=0.855, loss=158.948, backward_time=0.386, grad_norm=92.786, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=0.001, train_time=3.299 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:20:48,587 (trainer:732) INFO: 32epoch:train:1749-1840batch: iter_time=2.955e-04, forward_time=0.260, loss_att=163.832, acc=0.860, loss=163.832, backward_time=0.390, grad_norm=94.522, clip=100.000, loss_scale=1.000, optim_step_time=0.088, optim0_lr0=0.001, train_time=3.332 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:31:06,292 (trainer:338) INFO: 32epoch results: [train] iter_time=8.870e-04, forward_time=0.259, loss_att=162.833, acc=0.856, loss=162.833, backward_time=0.387, grad_norm=94.020, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.001, train_time=3.680, time=28 minutes and 18.08 seconds, total_count=58976, gpu_max_cached_mem_GB=29.846, [valid] loss_att=212.233, acc=0.819, cer=0.227, wer=0.531, loss=212.233, time=4 minutes and 27.94 seconds, total_count=608, gpu_max_cached_mem_GB=29.846, [att_plot] time=5 minutes and 44.93 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:31:13,090 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:31:13,101 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/22epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:31:13,102 (trainer:272) INFO: 33/60epoch started. Estimated time to finish: 19 hours, 43 minutes and 22.53 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:35:12,945 (trainer:732) INFO: 33epoch:train:1-92batch: iter_time=0.009, forward_time=0.264, loss_att=155.754, acc=0.860, loss=155.754, backward_time=0.387, grad_norm=84.774, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=10.427 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:36:29,613 (trainer:732) INFO: 33epoch:train:93-184batch: iter_time=3.734e-04, forward_time=0.260, loss_att=159.283, acc=0.859, loss=159.283, backward_time=0.388, grad_norm=96.163, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.001, train_time=3.333 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:37:46,040 (trainer:732) INFO: 33epoch:train:185-276batch: iter_time=3.583e-04, forward_time=0.259, loss_att=159.160, acc=0.859, loss=159.160, backward_time=0.386, grad_norm=85.507, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.001, train_time=3.322 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:39:03,789 (trainer:732) INFO: 33epoch:train:277-368batch: iter_time=4.627e-04, forward_time=0.265, loss_att=157.307, acc=0.862, loss=157.307, backward_time=0.390, grad_norm=98.830, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.001, train_time=3.380 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:40:20,775 (trainer:732) INFO: 33epoch:train:369-460batch: iter_time=4.032e-04, forward_time=0.261, loss_att=160.905, acc=0.857, loss=160.905, backward_time=0.383, grad_norm=87.573, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.347 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:41:38,948 (trainer:732) INFO: 33epoch:train:461-552batch: iter_time=4.293e-04, forward_time=0.266, loss_att=167.433, acc=0.858, loss=167.433, backward_time=0.391, grad_norm=94.321, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.001, train_time=3.398 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:42:56,333 (trainer:732) INFO: 33epoch:train:553-644batch: iter_time=4.714e-04, forward_time=0.262, loss_att=160.533, acc=0.856, loss=160.533, backward_time=0.387, grad_norm=93.802, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.001, train_time=3.364 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:44:13,377 (trainer:732) INFO: 33epoch:train:645-736batch: iter_time=5.340e-04, forward_time=0.263, loss_att=160.737, acc=0.854, loss=160.737, backward_time=0.386, grad_norm=92.406, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.001, train_time=3.349 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:45:30,786 (trainer:732) INFO: 33epoch:train:737-828batch: iter_time=4.498e-04, forward_time=0.263, loss_att=155.959, acc=0.860, loss=155.959, backward_time=0.386, grad_norm=91.596, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.001, train_time=3.365 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:46:46,968 (trainer:732) INFO: 33epoch:train:829-920batch: iter_time=4.142e-04, forward_time=0.258, loss_att=155.662, acc=0.859, loss=155.662, backward_time=0.380, grad_norm=82.050, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.001, train_time=3.312 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:48:04,710 (trainer:732) INFO: 33epoch:train:921-1012batch: iter_time=5.069e-04, forward_time=0.262, loss_att=153.126, acc=0.861, loss=153.126, backward_time=0.386, grad_norm=88.838, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.001, train_time=3.379 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:49:22,198 (trainer:732) INFO: 33epoch:train:1013-1104batch: iter_time=4.271e-04, forward_time=0.264, loss_att=162.016, acc=0.856, loss=162.016, backward_time=0.387, grad_norm=105.923, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.001, train_time=3.368 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:50:40,285 (trainer:732) INFO: 33epoch:train:1105-1196batch: iter_time=4.383e-04, forward_time=0.265, loss_att=165.958, acc=0.859, loss=165.958, backward_time=0.389, grad_norm=103.123, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.002, train_time=3.394 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:51:57,343 (trainer:732) INFO: 33epoch:train:1197-1288batch: iter_time=5.111e-04, forward_time=0.262, loss_att=154.427, acc=0.858, loss=154.427, backward_time=0.385, grad_norm=92.289, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.350 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:53:14,212 (trainer:732) INFO: 33epoch:train:1289-1380batch: iter_time=4.750e-04, forward_time=0.263, loss_att=154.941, acc=0.855, loss=154.941, backward_time=0.382, grad_norm=87.969, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.002, train_time=3.341 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:54:32,074 (trainer:732) INFO: 33epoch:train:1381-1472batch: iter_time=4.249e-04, forward_time=0.263, loss_att=160.649, acc=0.857, loss=160.649, backward_time=0.388, grad_norm=104.430, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.385 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:55:50,379 (trainer:732) INFO: 33epoch:train:1473-1564batch: iter_time=4.973e-04, forward_time=0.265, loss_att=158.317, acc=0.863, loss=158.317, backward_time=0.389, grad_norm=90.810, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.002, train_time=3.404 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:57:08,345 (trainer:732) INFO: 33epoch:train:1565-1656batch: iter_time=4.287e-04, forward_time=0.266, loss_att=158.552, acc=0.859, loss=158.552, backward_time=0.390, grad_norm=89.127, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.389 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:58:26,261 (trainer:732) INFO: 33epoch:train:1657-1748batch: iter_time=4.216e-04, forward_time=0.264, loss_att=163.870, acc=0.863, loss=163.870, backward_time=0.389, grad_norm=102.133, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.387 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 19:59:44,211 (trainer:732) INFO: 33epoch:train:1749-1840batch: iter_time=3.184e-04, forward_time=0.264, loss_att=158.595, acc=0.862, loss=158.595, backward_time=0.390, grad_norm=102.516, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.388 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:11:28,153 (trainer:338) INFO: 33epoch results: [train] iter_time=8.819e-04, forward_time=0.263, loss_att=159.086, acc=0.859, loss=159.086, backward_time=0.387, grad_norm=93.709, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.001, train_time=3.719, time=28 minutes and 39.01 seconds, total_count=60819, gpu_max_cached_mem_GB=29.846, [valid] loss_att=207.216, acc=0.824, cer=0.220, wer=0.519, loss=207.216, time=5 minutes and 11.69 seconds, total_count=627, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 24.34 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:11:36,140 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:11:36,156 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/23epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:11:36,157 (trainer:272) INFO: 34/60epoch started. Estimated time to finish: 18 hours, 59 minutes and 34.48 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:16:27,002 (trainer:732) INFO: 34epoch:train:1-92batch: iter_time=0.024, forward_time=0.267, loss_att=154.155, acc=0.866, loss=154.155, backward_time=0.391, grad_norm=96.722, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=12.644 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:17:44,578 (trainer:732) INFO: 34epoch:train:93-184batch: iter_time=4.172e-04, forward_time=0.263, loss_att=156.766, acc=0.864, loss=156.766, backward_time=0.388, grad_norm=106.716, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.372 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:19:02,131 (trainer:732) INFO: 34epoch:train:185-276batch: iter_time=4.566e-04, forward_time=0.263, loss_att=151.905, acc=0.867, loss=151.905, backward_time=0.387, grad_norm=83.329, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.371 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:20:20,123 (trainer:732) INFO: 34epoch:train:277-368batch: iter_time=4.235e-04, forward_time=0.265, loss_att=161.515, acc=0.864, loss=161.515, backward_time=0.389, grad_norm=102.392, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.390 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:21:37,489 (trainer:732) INFO: 34epoch:train:369-460batch: iter_time=4.229e-04, forward_time=0.263, loss_att=153.639, acc=0.863, loss=153.639, backward_time=0.386, grad_norm=102.030, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.002, train_time=3.363 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:22:55,543 (trainer:732) INFO: 34epoch:train:461-552batch: iter_time=5.024e-04, forward_time=0.267, loss_att=154.957, acc=0.867, loss=154.957, backward_time=0.390, grad_norm=94.408, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.393 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:24:12,524 (trainer:732) INFO: 34epoch:train:553-644batch: iter_time=4.485e-04, forward_time=0.261, loss_att=155.413, acc=0.861, loss=155.413, backward_time=0.384, grad_norm=96.696, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.346 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:25:29,890 (trainer:732) INFO: 34epoch:train:645-736batch: iter_time=4.070e-04, forward_time=0.262, loss_att=154.886, acc=0.863, loss=154.886, backward_time=0.385, grad_norm=101.769, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.363 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:26:48,313 (trainer:732) INFO: 34epoch:train:737-828batch: iter_time=4.313e-04, forward_time=0.268, loss_att=157.457, acc=0.864, loss=157.457, backward_time=0.392, grad_norm=91.976, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.409 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:28:06,249 (trainer:732) INFO: 34epoch:train:829-920batch: iter_time=4.068e-04, forward_time=0.264, loss_att=157.482, acc=0.863, loss=157.482, backward_time=0.388, grad_norm=91.154, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.388 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:29:24,090 (trainer:732) INFO: 34epoch:train:921-1012batch: iter_time=4.077e-04, forward_time=0.266, loss_att=153.620, acc=0.866, loss=153.620, backward_time=0.389, grad_norm=95.464, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.384 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:30:42,092 (trainer:732) INFO: 34epoch:train:1013-1104batch: iter_time=4.129e-04, forward_time=0.264, loss_att=155.564, acc=0.859, loss=155.564, backward_time=0.385, grad_norm=89.973, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:31:59,787 (trainer:732) INFO: 34epoch:train:1105-1196batch: iter_time=4.219e-04, forward_time=0.264, loss_att=153.281, acc=0.864, loss=153.281, backward_time=0.388, grad_norm=97.118, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.377 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:33:15,629 (trainer:732) INFO: 34epoch:train:1197-1288batch: iter_time=4.021e-04, forward_time=0.257, loss_att=142.368, acc=0.860, loss=142.368, backward_time=0.378, grad_norm=86.467, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.002, train_time=3.297 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:34:33,580 (trainer:732) INFO: 34epoch:train:1289-1380batch: iter_time=3.777e-04, forward_time=0.264, loss_att=153.756, acc=0.860, loss=153.756, backward_time=0.388, grad_norm=94.395, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.389 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:35:50,642 (trainer:732) INFO: 34epoch:train:1381-1472batch: iter_time=4.726e-04, forward_time=0.262, loss_att=151.318, acc=0.858, loss=151.318, backward_time=0.384, grad_norm=93.420, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.002, train_time=3.350 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:37:08,475 (trainer:732) INFO: 34epoch:train:1473-1564batch: iter_time=4.194e-04, forward_time=0.265, loss_att=153.089, acc=0.862, loss=153.089, backward_time=0.385, grad_norm=90.619, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.002, train_time=3.383 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:38:26,104 (trainer:732) INFO: 34epoch:train:1565-1656batch: iter_time=4.395e-04, forward_time=0.265, loss_att=154.018, acc=0.861, loss=154.018, backward_time=0.385, grad_norm=101.881, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.374 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:39:43,933 (trainer:732) INFO: 34epoch:train:1657-1748batch: iter_time=5.808e-04, forward_time=0.266, loss_att=152.978, acc=0.864, loss=152.978, backward_time=0.389, grad_norm=106.881, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.383 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:41:02,251 (trainer:732) INFO: 34epoch:train:1749-1840batch: iter_time=3.157e-04, forward_time=0.266, loss_att=159.876, acc=0.862, loss=159.876, backward_time=0.390, grad_norm=90.704, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.404 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:52:58,065 (trainer:338) INFO: 34epoch results: [train] iter_time=0.002, forward_time=0.264, loss_att=154.310, acc=0.863, loss=154.310, backward_time=0.387, grad_norm=95.706, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.839, time=29 minutes and 34.17 seconds, total_count=62662, gpu_max_cached_mem_GB=29.846, [valid] loss_att=208.578, acc=0.823, cer=0.234, wer=0.530, loss=208.578, time=5 minutes and 7.81 seconds, total_count=646, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 39.92 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:53:06,389 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:53:06,402 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/24epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:53:06,404 (trainer:272) INFO: 35/60epoch started. Estimated time to finish: 18 hours, 16 minutes and 49.86 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:58:22,575 (trainer:732) INFO: 35epoch:train:1-92batch: iter_time=0.014, forward_time=0.278, loss_att=148.401, acc=0.872, loss=148.401, backward_time=0.393, grad_norm=98.035, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.002, train_time=13.745 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 20:59:40,193 (trainer:732) INFO: 35epoch:train:93-184batch: iter_time=4.992e-04, forward_time=0.264, loss_att=146.481, acc=0.868, loss=146.481, backward_time=0.383, grad_norm=92.015, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.374 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:00:58,843 (trainer:732) INFO: 35epoch:train:185-276batch: iter_time=4.356e-04, forward_time=0.267, loss_att=147.323, acc=0.873, loss=147.323, backward_time=0.392, grad_norm=91.554, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.419 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:02:16,861 (trainer:732) INFO: 35epoch:train:277-368batch: iter_time=4.184e-04, forward_time=0.265, loss_att=144.594, acc=0.867, loss=144.594, backward_time=0.386, grad_norm=92.081, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.002, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:03:35,811 (trainer:732) INFO: 35epoch:train:369-460batch: iter_time=4.892e-04, forward_time=0.269, loss_att=153.552, acc=0.868, loss=153.552, backward_time=0.391, grad_norm=99.131, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=0.002, train_time=3.432 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:04:53,584 (trainer:732) INFO: 35epoch:train:461-552batch: iter_time=5.076e-04, forward_time=0.265, loss_att=144.891, acc=0.867, loss=144.891, backward_time=0.385, grad_norm=93.783, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.380 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:06:11,513 (trainer:732) INFO: 35epoch:train:553-644batch: iter_time=6.983e-04, forward_time=0.268, loss_att=147.241, acc=0.863, loss=147.241, backward_time=0.382, grad_norm=105.734, clip=100.000, loss_scale=1.000, optim_step_time=0.121, optim0_lr0=0.002, train_time=3.387 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:07:29,193 (trainer:732) INFO: 35epoch:train:645-736batch: iter_time=4.191e-04, forward_time=0.265, loss_att=146.762, acc=0.869, loss=146.762, backward_time=0.389, grad_norm=84.229, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.377 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:08:46,887 (trainer:732) INFO: 35epoch:train:737-828batch: iter_time=4.442e-04, forward_time=0.266, loss_att=153.915, acc=0.864, loss=153.915, backward_time=0.387, grad_norm=93.251, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.377 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:10:05,173 (trainer:732) INFO: 35epoch:train:829-920batch: iter_time=4.300e-04, forward_time=0.267, loss_att=156.561, acc=0.864, loss=156.561, backward_time=0.390, grad_norm=92.836, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.403 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:11:22,777 (trainer:732) INFO: 35epoch:train:921-1012batch: iter_time=4.468e-04, forward_time=0.266, loss_att=146.973, acc=0.863, loss=146.973, backward_time=0.388, grad_norm=94.462, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.373 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:12:41,254 (trainer:732) INFO: 35epoch:train:1013-1104batch: iter_time=4.956e-04, forward_time=0.266, loss_att=152.247, acc=0.868, loss=152.247, backward_time=0.391, grad_norm=99.525, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.411 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:13:58,940 (trainer:732) INFO: 35epoch:train:1105-1196batch: iter_time=5.095e-04, forward_time=0.265, loss_att=153.300, acc=0.866, loss=153.300, backward_time=0.386, grad_norm=99.530, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.377 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:15:16,871 (trainer:732) INFO: 35epoch:train:1197-1288batch: iter_time=4.348e-04, forward_time=0.264, loss_att=155.017, acc=0.867, loss=155.017, backward_time=0.390, grad_norm=94.792, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.388 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:16:34,668 (trainer:732) INFO: 35epoch:train:1289-1380batch: iter_time=5.359e-04, forward_time=0.266, loss_att=156.460, acc=0.867, loss=156.460, backward_time=0.389, grad_norm=97.649, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.382 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:17:52,559 (trainer:732) INFO: 35epoch:train:1381-1472batch: iter_time=5.182e-04, forward_time=0.265, loss_att=154.647, acc=0.865, loss=154.647, backward_time=0.389, grad_norm=90.542, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.386 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:19:09,675 (trainer:732) INFO: 35epoch:train:1473-1564batch: iter_time=5.352e-04, forward_time=0.262, loss_att=148.596, acc=0.860, loss=148.596, backward_time=0.383, grad_norm=94.000, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.352 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:20:27,411 (trainer:732) INFO: 35epoch:train:1565-1656batch: iter_time=4.466e-04, forward_time=0.266, loss_att=154.686, acc=0.866, loss=154.686, backward_time=0.389, grad_norm=95.903, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.379 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:21:43,885 (trainer:732) INFO: 35epoch:train:1657-1748batch: iter_time=3.620e-04, forward_time=0.259, loss_att=156.949, acc=0.860, loss=156.949, backward_time=0.382, grad_norm=95.508, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.324 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:23:00,472 (trainer:732) INFO: 35epoch:train:1749-1840batch: iter_time=3.221e-04, forward_time=0.259, loss_att=149.028, acc=0.864, loss=149.028, backward_time=0.382, grad_norm=84.129, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.329 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:34:30,070 (trainer:338) INFO: 35epoch results: [train] iter_time=0.001, forward_time=0.266, loss_att=150.802, acc=0.866, loss=150.802, backward_time=0.387, grad_norm=94.434, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.899, time=30 minutes and 0.53 seconds, total_count=64505, gpu_max_cached_mem_GB=29.846, [valid] loss_att=202.491, acc=0.829, cer=0.217, wer=0.512, loss=202.491, time=4 minutes and 57.09 seconds, total_count=665, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 26.04 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:34:38,527 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:34:38,540 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/25epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:34:38,541 (trainer:272) INFO: 36/60epoch started. Estimated time to finish: 17 hours, 34 minutes and 10.85 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:39:36,391 (trainer:732) INFO: 36epoch:train:1-92batch: iter_time=0.009, forward_time=0.272, loss_att=149.373, acc=0.872, loss=149.373, backward_time=0.387, grad_norm=95.540, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.002, train_time=12.948 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:40:55,234 (trainer:732) INFO: 36epoch:train:93-184batch: iter_time=5.716e-04, forward_time=0.270, loss_att=146.327, acc=0.877, loss=146.327, backward_time=0.392, grad_norm=98.005, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.002, train_time=3.427 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:42:13,067 (trainer:732) INFO: 36epoch:train:185-276batch: iter_time=5.364e-04, forward_time=0.265, loss_att=147.032, acc=0.873, loss=147.032, backward_time=0.387, grad_norm=85.303, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.383 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:43:30,503 (trainer:732) INFO: 36epoch:train:277-368batch: iter_time=4.393e-04, forward_time=0.264, loss_att=144.747, acc=0.868, loss=144.747, backward_time=0.385, grad_norm=86.543, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.366 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:44:48,187 (trainer:732) INFO: 36epoch:train:369-460batch: iter_time=5.669e-04, forward_time=0.265, loss_att=144.895, acc=0.870, loss=144.895, backward_time=0.387, grad_norm=94.480, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.377 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:46:05,618 (trainer:732) INFO: 36epoch:train:461-552batch: iter_time=4.492e-04, forward_time=0.265, loss_att=148.905, acc=0.865, loss=148.905, backward_time=0.386, grad_norm=105.685, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.366 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:47:23,181 (trainer:732) INFO: 36epoch:train:553-644batch: iter_time=4.228e-04, forward_time=0.266, loss_att=143.348, acc=0.868, loss=143.348, backward_time=0.387, grad_norm=104.992, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.372 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:48:41,369 (trainer:732) INFO: 36epoch:train:645-736batch: iter_time=5.303e-04, forward_time=0.265, loss_att=149.771, acc=0.872, loss=149.771, backward_time=0.391, grad_norm=93.374, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.399 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:49:58,902 (trainer:732) INFO: 36epoch:train:737-828batch: iter_time=4.308e-04, forward_time=0.262, loss_att=152.621, acc=0.862, loss=152.621, backward_time=0.383, grad_norm=111.766, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.370 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:51:17,315 (trainer:732) INFO: 36epoch:train:829-920batch: iter_time=4.241e-04, forward_time=0.266, loss_att=150.348, acc=0.868, loss=150.348, backward_time=0.389, grad_norm=109.042, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.409 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:52:35,421 (trainer:732) INFO: 36epoch:train:921-1012batch: iter_time=4.274e-04, forward_time=0.266, loss_att=148.668, acc=0.871, loss=148.668, backward_time=0.388, grad_norm=95.192, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.002, train_time=3.395 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:53:52,635 (trainer:732) INFO: 36epoch:train:1013-1104batch: iter_time=5.285e-04, forward_time=0.262, loss_att=145.154, acc=0.869, loss=145.154, backward_time=0.384, grad_norm=103.831, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.356 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:55:10,228 (trainer:732) INFO: 36epoch:train:1105-1196batch: iter_time=4.348e-04, forward_time=0.263, loss_att=149.438, acc=0.865, loss=149.438, backward_time=0.386, grad_norm=91.700, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.373 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:56:28,169 (trainer:732) INFO: 36epoch:train:1197-1288batch: iter_time=4.225e-04, forward_time=0.265, loss_att=146.601, acc=0.872, loss=146.601, backward_time=0.387, grad_norm=95.517, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.388 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:57:46,977 (trainer:732) INFO: 36epoch:train:1289-1380batch: iter_time=4.720e-04, forward_time=0.267, loss_att=150.293, acc=0.871, loss=150.293, backward_time=0.393, grad_norm=114.921, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.426 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 21:59:04,177 (trainer:732) INFO: 36epoch:train:1381-1472batch: iter_time=4.207e-04, forward_time=0.263, loss_att=143.253, acc=0.868, loss=143.253, backward_time=0.385, grad_norm=90.442, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.356 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:00:21,246 (trainer:732) INFO: 36epoch:train:1473-1564batch: iter_time=4.477e-04, forward_time=0.262, loss_att=148.279, acc=0.867, loss=148.279, backward_time=0.388, grad_norm=87.950, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.350 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:01:38,021 (trainer:732) INFO: 36epoch:train:1565-1656batch: iter_time=3.943e-04, forward_time=0.260, loss_att=147.322, acc=0.868, loss=147.322, backward_time=0.387, grad_norm=98.111, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.002, train_time=3.337 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:02:54,903 (trainer:732) INFO: 36epoch:train:1657-1748batch: iter_time=3.907e-04, forward_time=0.260, loss_att=148.384, acc=0.869, loss=148.384, backward_time=0.386, grad_norm=85.098, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.002, train_time=3.342 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:04:11,067 (trainer:732) INFO: 36epoch:train:1749-1840batch: iter_time=3.110e-04, forward_time=0.257, loss_att=139.733, acc=0.867, loss=139.733, backward_time=0.383, grad_norm=92.622, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=0.002, train_time=3.311 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:15:33,486 (trainer:338) INFO: 36epoch results: [train] iter_time=9.003e-04, forward_time=0.264, loss_att=147.187, acc=0.869, loss=147.187, backward_time=0.387, grad_norm=97.006, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.853, time=29 minutes and 38.76 seconds, total_count=66348, gpu_max_cached_mem_GB=29.846, [valid] loss_att=200.977, acc=0.830, cer=0.213, wer=0.508, loss=200.977, time=5 minutes and 24.87 seconds, total_count=684, gpu_max_cached_mem_GB=29.846, [att_plot] time=5 minutes and 51.31 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:15:40,717 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:15:40,727 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/26epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:15:40,728 (trainer:272) INFO: 37/60epoch started. Estimated time to finish: 16 hours, 51 minutes and 15.58 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:19:41,367 (trainer:732) INFO: 37epoch:train:1-92batch: iter_time=0.018, forward_time=0.260, loss_att=142.172, acc=0.878, loss=142.172, backward_time=0.386, grad_norm=99.812, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=0.002, train_time=10.461 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:20:57,096 (trainer:732) INFO: 37epoch:train:93-184batch: iter_time=3.322e-04, forward_time=0.258, loss_att=145.465, acc=0.870, loss=145.465, backward_time=0.385, grad_norm=92.570, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=0.002, train_time=3.292 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:22:13,487 (trainer:732) INFO: 37epoch:train:185-276batch: iter_time=3.248e-04, forward_time=0.259, loss_att=146.222, acc=0.872, loss=146.222, backward_time=0.388, grad_norm=94.811, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=0.002, train_time=3.321 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:23:28,717 (trainer:732) INFO: 37epoch:train:277-368batch: iter_time=3.172e-04, forward_time=0.255, loss_att=139.535, acc=0.867, loss=139.535, backward_time=0.381, grad_norm=94.188, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=0.002, train_time=3.270 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:24:43,971 (trainer:732) INFO: 37epoch:train:369-460batch: iter_time=3.204e-04, forward_time=0.255, loss_att=144.209, acc=0.871, loss=144.209, backward_time=0.382, grad_norm=96.413, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=0.002, train_time=3.271 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:25:59,525 (trainer:732) INFO: 37epoch:train:461-552batch: iter_time=3.920e-04, forward_time=0.256, loss_att=138.226, acc=0.874, loss=138.226, backward_time=0.383, grad_norm=93.101, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=0.002, train_time=3.284 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:27:15,652 (trainer:732) INFO: 37epoch:train:553-644batch: iter_time=3.413e-04, forward_time=0.258, loss_att=145.611, acc=0.873, loss=145.611, backward_time=0.385, grad_norm=97.603, clip=100.000, loss_scale=1.000, optim_step_time=0.087, optim0_lr0=0.002, train_time=3.309 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:28:32,516 (trainer:732) INFO: 37epoch:train:645-736batch: iter_time=3.722e-04, forward_time=0.260, loss_att=145.226, acc=0.878, loss=145.226, backward_time=0.390, grad_norm=90.555, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=0.002, train_time=3.341 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:29:48,363 (trainer:732) INFO: 37epoch:train:737-828batch: iter_time=3.662e-04, forward_time=0.256, loss_att=144.112, acc=0.870, loss=144.112, backward_time=0.384, grad_norm=95.266, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=0.002, train_time=3.297 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:31:04,822 (trainer:732) INFO: 37epoch:train:829-920batch: iter_time=3.665e-04, forward_time=0.259, loss_att=146.187, acc=0.874, loss=146.187, backward_time=0.389, grad_norm=89.469, clip=100.000, loss_scale=1.000, optim_step_time=0.088, optim0_lr0=0.002, train_time=3.324 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:32:20,847 (trainer:732) INFO: 37epoch:train:921-1012batch: iter_time=3.637e-04, forward_time=0.258, loss_att=147.376, acc=0.869, loss=147.376, backward_time=0.386, grad_norm=99.859, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=0.002, train_time=3.305 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:33:37,377 (trainer:732) INFO: 37epoch:train:1013-1104batch: iter_time=3.653e-04, forward_time=0.260, loss_att=145.129, acc=0.871, loss=145.129, backward_time=0.389, grad_norm=89.279, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=0.002, train_time=3.327 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:34:53,005 (trainer:732) INFO: 37epoch:train:1105-1196batch: iter_time=3.416e-04, forward_time=0.256, loss_att=143.126, acc=0.870, loss=143.126, backward_time=0.384, grad_norm=87.611, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=0.002, train_time=3.288 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:36:09,295 (trainer:732) INFO: 37epoch:train:1197-1288batch: iter_time=3.628e-04, forward_time=0.259, loss_att=146.488, acc=0.869, loss=146.488, backward_time=0.388, grad_norm=94.594, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=0.002, train_time=3.316 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:37:25,012 (trainer:732) INFO: 37epoch:train:1289-1380batch: iter_time=3.573e-04, forward_time=0.257, loss_att=143.834, acc=0.871, loss=143.834, backward_time=0.383, grad_norm=101.850, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=0.002, train_time=3.291 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:38:40,955 (trainer:732) INFO: 37epoch:train:1381-1472batch: iter_time=3.493e-04, forward_time=0.257, loss_att=141.243, acc=0.873, loss=141.243, backward_time=0.385, grad_norm=87.829, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.002, train_time=3.301 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:39:56,797 (trainer:732) INFO: 37epoch:train:1473-1564batch: iter_time=3.228e-04, forward_time=0.257, loss_att=141.816, acc=0.872, loss=141.816, backward_time=0.385, grad_norm=97.524, clip=100.000, loss_scale=1.000, optim_step_time=0.088, optim0_lr0=0.002, train_time=3.297 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:41:12,654 (trainer:732) INFO: 37epoch:train:1565-1656batch: iter_time=3.390e-04, forward_time=0.256, loss_att=142.013, acc=0.872, loss=142.013, backward_time=0.385, grad_norm=86.183, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=0.002, train_time=3.298 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:42:28,516 (trainer:732) INFO: 37epoch:train:1657-1748batch: iter_time=3.587e-04, forward_time=0.257, loss_att=149.930, acc=0.869, loss=149.930, backward_time=0.385, grad_norm=100.664, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=0.002, train_time=3.298 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:43:44,258 (trainer:732) INFO: 37epoch:train:1749-1840batch: iter_time=3.017e-04, forward_time=0.258, loss_att=139.841, acc=0.873, loss=139.841, backward_time=0.386, grad_norm=92.041, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=0.002, train_time=3.293 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:53:32,795 (trainer:338) INFO: 37epoch results: [train] iter_time=0.001, forward_time=0.258, loss_att=143.879, acc=0.872, loss=143.879, backward_time=0.385, grad_norm=94.061, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=0.002, train_time=3.659, time=28 minutes and 8.04 seconds, total_count=68191, gpu_max_cached_mem_GB=29.846, [valid] loss_att=198.396, acc=0.833, cer=0.210, wer=0.504, loss=198.396, time=4 minutes and 8.59 seconds, total_count=703, gpu_max_cached_mem_GB=29.846, [att_plot] time=5 minutes and 35.44 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:53:39,125 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:53:39,135 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/27epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:53:39,136 (trainer:272) INFO: 38/60epoch started. Estimated time to finish: 16 hours, 6 minutes and 32.19 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:57:08,822 (trainer:732) INFO: 38epoch:train:1-92batch: iter_time=0.010, forward_time=0.261, loss_att=140.144, acc=0.880, loss=140.144, backward_time=0.388, grad_norm=103.580, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.002, train_time=9.116 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:58:24,967 (trainer:732) INFO: 38epoch:train:93-184batch: iter_time=3.705e-04, forward_time=0.257, loss_att=132.520, acc=0.873, loss=132.520, backward_time=0.383, grad_norm=93.385, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.002, train_time=3.310 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 22:59:41,361 (trainer:732) INFO: 38epoch:train:185-276batch: iter_time=3.987e-04, forward_time=0.259, loss_att=139.409, acc=0.876, loss=139.409, backward_time=0.384, grad_norm=89.192, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.002, train_time=3.321 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:00:57,916 (trainer:732) INFO: 38epoch:train:277-368batch: iter_time=3.765e-04, forward_time=0.259, loss_att=136.405, acc=0.878, loss=136.405, backward_time=0.386, grad_norm=99.941, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=0.002, train_time=3.328 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:02:14,693 (trainer:732) INFO: 38epoch:train:369-460batch: iter_time=3.519e-04, forward_time=0.260, loss_att=140.677, acc=0.873, loss=140.677, backward_time=0.388, grad_norm=104.470, clip=100.000, loss_scale=1.000, optim_step_time=0.097, optim0_lr0=0.002, train_time=3.337 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:03:30,810 (trainer:732) INFO: 38epoch:train:461-552batch: iter_time=4.041e-04, forward_time=0.259, loss_att=136.016, acc=0.877, loss=136.016, backward_time=0.383, grad_norm=89.928, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.002, train_time=3.309 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:04:47,260 (trainer:732) INFO: 38epoch:train:553-644batch: iter_time=3.569e-04, forward_time=0.260, loss_att=142.753, acc=0.874, loss=142.753, backward_time=0.385, grad_norm=91.360, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.002, train_time=3.323 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:06:04,155 (trainer:732) INFO: 38epoch:train:645-736batch: iter_time=3.690e-04, forward_time=0.262, loss_att=142.911, acc=0.876, loss=142.911, backward_time=0.388, grad_norm=91.338, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.002, train_time=3.343 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:07:20,679 (trainer:732) INFO: 38epoch:train:737-828batch: iter_time=3.722e-04, forward_time=0.260, loss_att=139.595, acc=0.874, loss=139.595, backward_time=0.386, grad_norm=94.080, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.002, train_time=3.326 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:08:37,210 (trainer:732) INFO: 38epoch:train:829-920batch: iter_time=3.570e-04, forward_time=0.260, loss_att=139.788, acc=0.875, loss=139.788, backward_time=0.386, grad_norm=101.141, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.002, train_time=3.327 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:09:53,885 (trainer:732) INFO: 38epoch:train:921-1012batch: iter_time=3.950e-04, forward_time=0.260, loss_att=135.751, acc=0.879, loss=135.751, backward_time=0.385, grad_norm=92.833, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.333 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:11:10,810 (trainer:732) INFO: 38epoch:train:1013-1104batch: iter_time=4.175e-04, forward_time=0.260, loss_att=143.863, acc=0.875, loss=143.863, backward_time=0.386, grad_norm=96.064, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.002, train_time=3.344 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:12:28,550 (trainer:732) INFO: 38epoch:train:1105-1196batch: iter_time=4.442e-04, forward_time=0.265, loss_att=142.554, acc=0.877, loss=142.554, backward_time=0.386, grad_norm=98.702, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.379 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:13:45,953 (trainer:732) INFO: 38epoch:train:1197-1288batch: iter_time=4.140e-04, forward_time=0.264, loss_att=143.467, acc=0.872, loss=143.467, backward_time=0.386, grad_norm=89.222, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.365 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:15:03,465 (trainer:732) INFO: 38epoch:train:1289-1380batch: iter_time=4.129e-04, forward_time=0.264, loss_att=143.165, acc=0.874, loss=143.165, backward_time=0.387, grad_norm=103.952, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.369 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:16:20,514 (trainer:732) INFO: 38epoch:train:1381-1472batch: iter_time=4.196e-04, forward_time=0.262, loss_att=132.565, acc=0.874, loss=132.565, backward_time=0.385, grad_norm=97.741, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.002, train_time=3.349 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:17:38,167 (trainer:732) INFO: 38epoch:train:1473-1564batch: iter_time=3.993e-04, forward_time=0.265, loss_att=142.786, acc=0.874, loss=142.786, backward_time=0.385, grad_norm=85.315, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.376 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:18:55,496 (trainer:732) INFO: 38epoch:train:1565-1656batch: iter_time=5.922e-04, forward_time=0.263, loss_att=142.892, acc=0.870, loss=142.892, backward_time=0.384, grad_norm=93.607, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.361 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:20:13,496 (trainer:732) INFO: 38epoch:train:1657-1748batch: iter_time=3.916e-04, forward_time=0.267, loss_att=145.653, acc=0.875, loss=145.653, backward_time=0.390, grad_norm=108.435, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:21:31,498 (trainer:732) INFO: 38epoch:train:1749-1840batch: iter_time=3.812e-04, forward_time=0.266, loss_att=134.388, acc=0.882, loss=134.388, backward_time=0.389, grad_norm=102.348, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:33:09,086 (trainer:338) INFO: 38epoch results: [train] iter_time=8.714e-04, forward_time=0.262, loss_att=139.806, acc=0.875, loss=139.806, backward_time=0.386, grad_norm=96.332, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.002, train_time=3.635, time=27 minutes and 59.02 seconds, total_count=70034, gpu_max_cached_mem_GB=29.846, [valid] loss_att=199.660, acc=0.833, cer=0.212, wer=0.508, loss=199.660, time=5 minutes and 10.12 seconds, total_count=722, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 20.8 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:33:17,291 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:33:17,306 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/28epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:33:17,307 (trainer:272) INFO: 39/60epoch started. Estimated time to finish: 15 hours, 23 minutes and 7.87 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:38:11,359 (trainer:732) INFO: 39epoch:train:1-92batch: iter_time=0.029, forward_time=0.265, loss_att=134.242, acc=0.879, loss=134.242, backward_time=0.386, grad_norm=96.374, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=12.783 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:39:29,399 (trainer:732) INFO: 39epoch:train:93-184batch: iter_time=4.502e-04, forward_time=0.266, loss_att=139.866, acc=0.880, loss=139.866, backward_time=0.388, grad_norm=99.984, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.002, train_time=3.392 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:40:47,239 (trainer:732) INFO: 39epoch:train:185-276batch: iter_time=3.913e-04, forward_time=0.265, loss_att=135.945, acc=0.883, loss=135.945, backward_time=0.388, grad_norm=94.511, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.384 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:42:04,442 (trainer:732) INFO: 39epoch:train:277-368batch: iter_time=4.248e-04, forward_time=0.263, loss_att=132.565, acc=0.875, loss=132.565, backward_time=0.384, grad_norm=83.005, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.356 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:43:23,029 (trainer:732) INFO: 39epoch:train:369-460batch: iter_time=5.566e-04, forward_time=0.269, loss_att=138.420, acc=0.881, loss=138.420, backward_time=0.392, grad_norm=99.051, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.416 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:44:41,240 (trainer:732) INFO: 39epoch:train:461-552batch: iter_time=4.277e-04, forward_time=0.268, loss_att=141.261, acc=0.876, loss=141.261, backward_time=0.389, grad_norm=101.621, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.400 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:45:59,829 (trainer:732) INFO: 39epoch:train:553-644batch: iter_time=3.872e-04, forward_time=0.267, loss_att=137.608, acc=0.879, loss=137.608, backward_time=0.389, grad_norm=89.167, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.416 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:47:17,993 (trainer:732) INFO: 39epoch:train:645-736batch: iter_time=4.530e-04, forward_time=0.268, loss_att=140.343, acc=0.878, loss=140.343, backward_time=0.390, grad_norm=96.980, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.398 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:48:36,281 (trainer:732) INFO: 39epoch:train:737-828batch: iter_time=4.269e-04, forward_time=0.265, loss_att=138.621, acc=0.880, loss=138.621, backward_time=0.388, grad_norm=99.403, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.403 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:49:54,236 (trainer:732) INFO: 39epoch:train:829-920batch: iter_time=5.193e-04, forward_time=0.266, loss_att=135.842, acc=0.873, loss=135.842, backward_time=0.385, grad_norm=91.105, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.002, train_time=3.389 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:51:13,321 (trainer:732) INFO: 39epoch:train:921-1012batch: iter_time=8.175e-04, forward_time=0.270, loss_att=138.566, acc=0.879, loss=138.566, backward_time=0.393, grad_norm=95.208, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.438 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:52:30,969 (trainer:732) INFO: 39epoch:train:1013-1104batch: iter_time=4.383e-04, forward_time=0.265, loss_att=136.499, acc=0.875, loss=136.499, backward_time=0.388, grad_norm=91.100, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.375 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:53:48,497 (trainer:732) INFO: 39epoch:train:1105-1196batch: iter_time=4.106e-04, forward_time=0.265, loss_att=136.520, acc=0.874, loss=136.520, backward_time=0.385, grad_norm=95.753, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.370 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:55:06,317 (trainer:732) INFO: 39epoch:train:1197-1288batch: iter_time=4.639e-04, forward_time=0.265, loss_att=137.519, acc=0.878, loss=137.519, backward_time=0.386, grad_norm=101.952, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.383 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:56:23,584 (trainer:732) INFO: 39epoch:train:1289-1380batch: iter_time=4.815e-04, forward_time=0.264, loss_att=138.444, acc=0.872, loss=138.444, backward_time=0.383, grad_norm=135.641, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.359 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:57:41,749 (trainer:732) INFO: 39epoch:train:1381-1472batch: iter_time=4.656e-04, forward_time=0.267, loss_att=136.981, acc=0.880, loss=136.981, backward_time=0.390, grad_norm=101.896, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.398 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-17 23:59:00,075 (trainer:732) INFO: 39epoch:train:1473-1564batch: iter_time=3.963e-04, forward_time=0.266, loss_att=138.431, acc=0.882, loss=138.431, backward_time=0.391, grad_norm=94.381, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.405 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:00:17,413 (trainer:732) INFO: 39epoch:train:1565-1656batch: iter_time=4.147e-04, forward_time=0.261, loss_att=138.018, acc=0.875, loss=138.018, backward_time=0.388, grad_norm=102.661, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.362 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:01:34,795 (trainer:732) INFO: 39epoch:train:1657-1748batch: iter_time=4.819e-04, forward_time=0.263, loss_att=135.205, acc=0.876, loss=135.205, backward_time=0.385, grad_norm=85.447, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.364 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:02:52,090 (trainer:732) INFO: 39epoch:train:1749-1840batch: iter_time=4.314e-04, forward_time=0.263, loss_att=136.864, acc=0.876, loss=136.864, backward_time=0.389, grad_norm=92.866, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.360 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:15:16,175 (trainer:338) INFO: 39epoch results: [train] iter_time=0.002, forward_time=0.266, loss_att=137.380, acc=0.877, loss=137.380, backward_time=0.388, grad_norm=97.405, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.857, time=29 minutes and 42.02 seconds, total_count=71877, gpu_max_cached_mem_GB=29.846, [valid] loss_att=195.222, acc=0.835, cer=0.207, wer=0.504, loss=195.222, time=5 minutes and 14.48 seconds, total_count=741, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 2.36 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:15:24,068 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:15:24,092 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/29epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:15:24,093 (trainer:272) INFO: 40/60epoch started. Estimated time to finish: 14 hours, 41 minutes and 15.17 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:20:14,676 (trainer:732) INFO: 40epoch:train:1-92batch: iter_time=0.026, forward_time=0.269, loss_att=128.265, acc=0.883, loss=128.265, backward_time=0.384, grad_norm=89.100, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=12.632 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:21:31,921 (trainer:732) INFO: 40epoch:train:93-184batch: iter_time=4.487e-04, forward_time=0.263, loss_att=130.621, acc=0.883, loss=130.621, backward_time=0.386, grad_norm=97.129, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.002, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:22:49,602 (trainer:732) INFO: 40epoch:train:185-276batch: iter_time=4.226e-04, forward_time=0.265, loss_att=138.422, acc=0.883, loss=138.422, backward_time=0.387, grad_norm=94.117, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.377 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:24:07,603 (trainer:732) INFO: 40epoch:train:277-368batch: iter_time=4.338e-04, forward_time=0.265, loss_att=132.694, acc=0.880, loss=132.694, backward_time=0.387, grad_norm=93.275, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.391 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:25:25,343 (trainer:732) INFO: 40epoch:train:369-460batch: iter_time=4.815e-04, forward_time=0.264, loss_att=134.919, acc=0.883, loss=134.919, backward_time=0.388, grad_norm=99.502, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.379 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:26:43,916 (trainer:732) INFO: 40epoch:train:461-552batch: iter_time=4.585e-04, forward_time=0.266, loss_att=134.877, acc=0.886, loss=134.877, backward_time=0.392, grad_norm=96.727, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.415 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:28:02,520 (trainer:732) INFO: 40epoch:train:553-644batch: iter_time=4.097e-04, forward_time=0.266, loss_att=136.237, acc=0.881, loss=136.237, backward_time=0.389, grad_norm=90.721, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.417 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:29:20,880 (trainer:732) INFO: 40epoch:train:645-736batch: iter_time=5.270e-04, forward_time=0.265, loss_att=135.580, acc=0.882, loss=135.580, backward_time=0.391, grad_norm=86.216, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.406 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:30:37,897 (trainer:732) INFO: 40epoch:train:737-828batch: iter_time=4.479e-04, forward_time=0.262, loss_att=138.499, acc=0.877, loss=138.499, backward_time=0.386, grad_norm=103.081, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.348 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:31:54,862 (trainer:732) INFO: 40epoch:train:829-920batch: iter_time=4.053e-04, forward_time=0.261, loss_att=133.529, acc=0.880, loss=133.529, backward_time=0.387, grad_norm=101.070, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.002, train_time=3.346 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:33:12,332 (trainer:732) INFO: 40epoch:train:921-1012batch: iter_time=4.326e-04, forward_time=0.260, loss_att=133.382, acc=0.881, loss=133.382, backward_time=0.384, grad_norm=89.241, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.368 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:34:29,467 (trainer:732) INFO: 40epoch:train:1013-1104batch: iter_time=3.619e-04, forward_time=0.262, loss_att=139.064, acc=0.878, loss=139.064, backward_time=0.388, grad_norm=95.362, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.353 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:35:45,762 (trainer:732) INFO: 40epoch:train:1105-1196batch: iter_time=3.413e-04, forward_time=0.258, loss_att=135.214, acc=0.874, loss=135.214, backward_time=0.381, grad_norm=85.221, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.316 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:37:04,822 (trainer:732) INFO: 40epoch:train:1197-1288batch: iter_time=4.061e-04, forward_time=0.268, loss_att=136.592, acc=0.886, loss=136.592, backward_time=0.394, grad_norm=98.475, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.437 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:38:21,891 (trainer:732) INFO: 40epoch:train:1289-1380batch: iter_time=4.149e-04, forward_time=0.261, loss_att=139.156, acc=0.878, loss=139.156, backward_time=0.386, grad_norm=100.176, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.002, train_time=3.350 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:39:39,490 (trainer:732) INFO: 40epoch:train:1381-1472batch: iter_time=3.968e-04, forward_time=0.262, loss_att=136.053, acc=0.880, loss=136.053, backward_time=0.388, grad_norm=90.362, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.373 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:40:57,062 (trainer:732) INFO: 40epoch:train:1473-1564batch: iter_time=3.358e-04, forward_time=0.264, loss_att=134.457, acc=0.878, loss=134.457, backward_time=0.388, grad_norm=109.593, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.372 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:42:13,559 (trainer:732) INFO: 40epoch:train:1565-1656batch: iter_time=4.261e-04, forward_time=0.260, loss_att=124.650, acc=0.879, loss=124.650, backward_time=0.381, grad_norm=84.997, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.325 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:43:30,451 (trainer:732) INFO: 40epoch:train:1657-1748batch: iter_time=3.823e-04, forward_time=0.261, loss_att=133.392, acc=0.878, loss=133.392, backward_time=0.384, grad_norm=91.809, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.342 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:44:46,614 (trainer:732) INFO: 40epoch:train:1749-1840batch: iter_time=2.947e-04, forward_time=0.259, loss_att=133.102, acc=0.873, loss=133.102, backward_time=0.382, grad_norm=88.224, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.311 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:56:20,730 (trainer:338) INFO: 40epoch results: [train] iter_time=0.002, forward_time=0.263, loss_att=134.369, acc=0.880, loss=134.369, backward_time=0.387, grad_norm=94.220, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.831, time=29 minutes and 29.07 seconds, total_count=73720, gpu_max_cached_mem_GB=29.846, [valid] loss_att=195.364, acc=0.837, cer=0.205, wer=0.496, loss=195.364, time=5 minutes and 4.45 seconds, total_count=760, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 23.11 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:56:29,504 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:56:29,517 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/30epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 00:56:29,518 (trainer:272) INFO: 41/60epoch started. Estimated time to finish: 13 hours, 58 minutes and 51.08 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:01:17,567 (trainer:732) INFO: 41epoch:train:1-92batch: iter_time=0.013, forward_time=0.267, loss_att=128.557, acc=0.878, loss=128.557, backward_time=0.383, grad_norm=86.855, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=12.522 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:02:34,428 (trainer:732) INFO: 41epoch:train:93-184batch: iter_time=4.157e-04, forward_time=0.261, loss_att=125.426, acc=0.881, loss=125.426, backward_time=0.384, grad_norm=85.673, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.002, train_time=3.341 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:03:52,216 (trainer:732) INFO: 41epoch:train:185-276batch: iter_time=4.783e-04, forward_time=0.264, loss_att=131.600, acc=0.882, loss=131.600, backward_time=0.389, grad_norm=93.642, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.381 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:05:09,035 (trainer:732) INFO: 41epoch:train:277-368batch: iter_time=4.059e-04, forward_time=0.261, loss_att=126.951, acc=0.882, loss=126.951, backward_time=0.381, grad_norm=92.723, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.339 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:06:26,205 (trainer:732) INFO: 41epoch:train:369-460batch: iter_time=4.190e-04, forward_time=0.263, loss_att=128.786, acc=0.884, loss=128.786, backward_time=0.385, grad_norm=97.998, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.354 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:07:43,622 (trainer:732) INFO: 41epoch:train:461-552batch: iter_time=3.988e-04, forward_time=0.262, loss_att=130.851, acc=0.885, loss=130.851, backward_time=0.387, grad_norm=91.510, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.365 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:09:02,024 (trainer:732) INFO: 41epoch:train:553-644batch: iter_time=3.910e-04, forward_time=0.267, loss_att=130.047, acc=0.889, loss=130.047, backward_time=0.391, grad_norm=96.823, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.408 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:10:19,517 (trainer:732) INFO: 41epoch:train:645-736batch: iter_time=4.280e-04, forward_time=0.264, loss_att=131.231, acc=0.887, loss=131.231, backward_time=0.388, grad_norm=88.394, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.369 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:11:36,976 (trainer:732) INFO: 41epoch:train:737-828batch: iter_time=4.529e-04, forward_time=0.263, loss_att=133.067, acc=0.880, loss=133.067, backward_time=0.386, grad_norm=89.567, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.367 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:12:54,783 (trainer:732) INFO: 41epoch:train:829-920batch: iter_time=4.197e-04, forward_time=0.264, loss_att=136.879, acc=0.883, loss=136.879, backward_time=0.389, grad_norm=110.076, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.382 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:14:12,285 (trainer:732) INFO: 41epoch:train:921-1012batch: iter_time=4.290e-04, forward_time=0.263, loss_att=131.149, acc=0.886, loss=131.149, backward_time=0.388, grad_norm=98.039, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.002, train_time=3.369 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:15:29,249 (trainer:732) INFO: 41epoch:train:1013-1104batch: iter_time=3.784e-04, forward_time=0.261, loss_att=130.770, acc=0.883, loss=130.770, backward_time=0.384, grad_norm=83.501, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.346 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:16:47,307 (trainer:732) INFO: 41epoch:train:1105-1196batch: iter_time=3.960e-04, forward_time=0.263, loss_att=134.858, acc=0.882, loss=134.858, backward_time=0.387, grad_norm=116.280, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.002, train_time=3.393 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:18:03,881 (trainer:732) INFO: 41epoch:train:1197-1288batch: iter_time=3.958e-04, forward_time=0.259, loss_att=129.376, acc=0.879, loss=129.376, backward_time=0.384, grad_norm=85.249, clip=100.000, loss_scale=1.000, optim_step_time=0.106, optim0_lr0=0.002, train_time=3.329 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:19:21,580 (trainer:732) INFO: 41epoch:train:1289-1380batch: iter_time=4.063e-04, forward_time=0.266, loss_att=134.986, acc=0.884, loss=134.986, backward_time=0.389, grad_norm=92.776, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.378 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:20:39,669 (trainer:732) INFO: 41epoch:train:1381-1472batch: iter_time=4.758e-04, forward_time=0.265, loss_att=132.029, acc=0.885, loss=132.029, backward_time=0.387, grad_norm=99.662, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.394 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:21:58,622 (trainer:732) INFO: 41epoch:train:1473-1564batch: iter_time=5.995e-04, forward_time=0.269, loss_att=126.817, acc=0.884, loss=126.817, backward_time=0.391, grad_norm=107.391, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=0.002, train_time=3.432 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:23:17,112 (trainer:732) INFO: 41epoch:train:1565-1656batch: iter_time=4.160e-04, forward_time=0.268, loss_att=135.011, acc=0.884, loss=135.011, backward_time=0.391, grad_norm=97.750, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.412 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:24:35,173 (trainer:732) INFO: 41epoch:train:1657-1748batch: iter_time=4.543e-04, forward_time=0.265, loss_att=131.850, acc=0.881, loss=131.850, backward_time=0.387, grad_norm=97.644, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.002, train_time=3.393 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:25:53,208 (trainer:732) INFO: 41epoch:train:1749-1840batch: iter_time=3.559e-04, forward_time=0.266, loss_att=133.421, acc=0.879, loss=133.421, backward_time=0.389, grad_norm=97.588, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.392 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:37:36,686 (trainer:338) INFO: 41epoch results: [train] iter_time=0.001, forward_time=0.264, loss_att=131.129, acc=0.883, loss=131.129, backward_time=0.387, grad_norm=95.457, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.833, time=29 minutes and 30.64 seconds, total_count=75563, gpu_max_cached_mem_GB=29.846, [valid] loss_att=193.404, acc=0.840, cer=0.199, wer=0.492, loss=193.404, time=4 minutes and 55.66 seconds, total_count=779, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 40.86 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:37:45,663 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:37:45,692 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/31epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:37:45,693 (trainer:272) INFO: 42/60epoch started. Estimated time to finish: 13 hours, 16 minutes and 35.82 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:42:57,173 (trainer:732) INFO: 42epoch:train:1-92batch: iter_time=0.020, forward_time=0.267, loss_att=121.902, acc=0.887, loss=121.902, backward_time=0.385, grad_norm=92.095, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.002, train_time=13.539 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:44:14,423 (trainer:732) INFO: 42epoch:train:93-184batch: iter_time=4.001e-04, forward_time=0.262, loss_att=126.796, acc=0.886, loss=126.796, backward_time=0.388, grad_norm=100.360, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:45:31,897 (trainer:732) INFO: 42epoch:train:185-276batch: iter_time=3.824e-04, forward_time=0.263, loss_att=125.087, acc=0.887, loss=125.087, backward_time=0.388, grad_norm=91.239, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.368 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:46:49,777 (trainer:732) INFO: 42epoch:train:277-368batch: iter_time=4.288e-04, forward_time=0.263, loss_att=132.768, acc=0.885, loss=132.768, backward_time=0.389, grad_norm=94.066, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.002, train_time=3.385 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:48:07,099 (trainer:732) INFO: 42epoch:train:369-460batch: iter_time=4.205e-04, forward_time=0.263, loss_att=129.980, acc=0.884, loss=129.980, backward_time=0.388, grad_norm=90.704, clip=100.000, loss_scale=1.000, optim_step_time=0.108, optim0_lr0=0.002, train_time=3.361 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:49:23,400 (trainer:732) INFO: 42epoch:train:461-552batch: iter_time=4.305e-04, forward_time=0.258, loss_att=127.056, acc=0.881, loss=127.056, backward_time=0.382, grad_norm=93.569, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.317 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:50:41,350 (trainer:732) INFO: 42epoch:train:553-644batch: iter_time=5.065e-04, forward_time=0.266, loss_att=124.496, acc=0.889, loss=124.496, backward_time=0.390, grad_norm=90.162, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.388 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:51:58,366 (trainer:732) INFO: 42epoch:train:645-736batch: iter_time=3.838e-04, forward_time=0.261, loss_att=129.325, acc=0.886, loss=129.325, backward_time=0.386, grad_norm=101.542, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.002, train_time=3.348 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:53:15,644 (trainer:732) INFO: 42epoch:train:737-828batch: iter_time=3.896e-04, forward_time=0.262, loss_att=132.696, acc=0.884, loss=132.696, backward_time=0.389, grad_norm=93.392, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.002, train_time=3.359 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:54:32,998 (trainer:732) INFO: 42epoch:train:829-920batch: iter_time=3.823e-04, forward_time=0.262, loss_att=129.587, acc=0.887, loss=129.587, backward_time=0.389, grad_norm=94.731, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.363 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:55:51,824 (trainer:732) INFO: 42epoch:train:921-1012batch: iter_time=3.737e-04, forward_time=0.266, loss_att=128.296, acc=0.891, loss=128.296, backward_time=0.395, grad_norm=105.372, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.002, train_time=3.427 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:57:09,097 (trainer:732) INFO: 42epoch:train:1013-1104batch: iter_time=3.950e-04, forward_time=0.262, loss_att=122.196, acc=0.890, loss=122.196, backward_time=0.390, grad_norm=93.415, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=0.002, train_time=3.359 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:58:25,516 (trainer:732) INFO: 42epoch:train:1105-1196batch: iter_time=3.655e-04, forward_time=0.258, loss_att=129.314, acc=0.883, loss=129.314, backward_time=0.386, grad_norm=91.073, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.002, train_time=3.322 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 01:59:42,560 (trainer:732) INFO: 42epoch:train:1197-1288batch: iter_time=3.906e-04, forward_time=0.261, loss_att=130.769, acc=0.887, loss=130.769, backward_time=0.387, grad_norm=96.635, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.002, train_time=3.349 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:00:58,660 (trainer:732) INFO: 42epoch:train:1289-1380batch: iter_time=3.764e-04, forward_time=0.260, loss_att=131.844, acc=0.883, loss=131.844, backward_time=0.386, grad_norm=94.827, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.002, train_time=3.308 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:02:15,145 (trainer:732) INFO: 42epoch:train:1381-1472batch: iter_time=3.980e-04, forward_time=0.260, loss_att=122.501, acc=0.885, loss=122.501, backward_time=0.387, grad_norm=94.367, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=0.002, train_time=3.325 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:03:31,208 (trainer:732) INFO: 42epoch:train:1473-1564batch: iter_time=3.934e-04, forward_time=0.259, loss_att=125.876, acc=0.882, loss=125.876, backward_time=0.385, grad_norm=93.277, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.002, train_time=3.306 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:04:46,695 (trainer:732) INFO: 42epoch:train:1565-1656batch: iter_time=3.801e-04, forward_time=0.255, loss_att=130.443, acc=0.879, loss=130.443, backward_time=0.382, grad_norm=92.443, clip=100.000, loss_scale=1.000, optim_step_time=0.092, optim0_lr0=0.002, train_time=3.281 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:06:03,657 (trainer:732) INFO: 42epoch:train:1657-1748batch: iter_time=3.839e-04, forward_time=0.260, loss_att=127.036, acc=0.891, loss=127.036, backward_time=0.387, grad_norm=97.191, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=0.002, train_time=3.346 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:07:20,425 (trainer:732) INFO: 42epoch:train:1749-1840batch: iter_time=3.141e-04, forward_time=0.259, loss_att=129.664, acc=0.887, loss=129.664, backward_time=0.387, grad_norm=92.807, clip=100.000, loss_scale=1.000, optim_step_time=0.104, optim0_lr0=0.002, train_time=3.337 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:17:29,417 (trainer:338) INFO: 42epoch results: [train] iter_time=0.001, forward_time=0.261, loss_att=127.812, acc=0.886, loss=127.812, backward_time=0.387, grad_norm=94.663, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.002, train_time=3.857, time=29 minutes and 40.06 seconds, total_count=77406, gpu_max_cached_mem_GB=29.846, [valid] loss_att=189.765, acc=0.841, cer=0.198, wer=0.490, loss=189.765, time=4 minutes and 15.48 seconds, total_count=798, gpu_max_cached_mem_GB=29.846, [att_plot] time=5 minutes and 48.18 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:17:36,078 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:17:36,091 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/32epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:17:36,092 (trainer:272) INFO: 43/60epoch started. Estimated time to finish: 12 hours, 33 minutes and 46.6 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:21:20,905 (trainer:732) INFO: 43epoch:train:1-92batch: iter_time=0.012, forward_time=0.259, loss_att=119.420, acc=0.886, loss=119.420, backward_time=0.384, grad_norm=83.033, clip=100.000, loss_scale=1.000, optim_step_time=0.097, optim0_lr0=0.002, train_time=9.773 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:22:37,051 (trainer:732) INFO: 43epoch:train:93-184batch: iter_time=3.896e-04, forward_time=0.258, loss_att=124.497, acc=0.887, loss=124.497, backward_time=0.386, grad_norm=84.223, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.002, train_time=3.310 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:23:53,507 (trainer:732) INFO: 43epoch:train:185-276batch: iter_time=3.843e-04, forward_time=0.259, loss_att=123.148, acc=0.890, loss=123.148, backward_time=0.387, grad_norm=85.438, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=0.002, train_time=3.324 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:25:09,749 (trainer:732) INFO: 43epoch:train:277-368batch: iter_time=4.063e-04, forward_time=0.259, loss_att=128.235, acc=0.887, loss=128.235, backward_time=0.386, grad_norm=102.696, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=0.002, train_time=3.314 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:26:27,377 (trainer:732) INFO: 43epoch:train:369-460batch: iter_time=3.804e-04, forward_time=0.263, loss_att=123.218, acc=0.894, loss=123.218, backward_time=0.393, grad_norm=99.064, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=0.002, train_time=3.375 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:27:44,188 (trainer:732) INFO: 43epoch:train:461-552batch: iter_time=4.022e-04, forward_time=0.258, loss_att=125.461, acc=0.891, loss=125.461, backward_time=0.385, grad_norm=103.312, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.002, train_time=3.339 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:29:00,203 (trainer:732) INFO: 43epoch:train:553-644batch: iter_time=3.535e-04, forward_time=0.257, loss_att=122.565, acc=0.892, loss=122.565, backward_time=0.383, grad_norm=91.818, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.002, train_time=3.304 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:30:16,796 (trainer:732) INFO: 43epoch:train:645-736batch: iter_time=3.611e-04, forward_time=0.258, loss_att=125.306, acc=0.888, loss=125.306, backward_time=0.386, grad_norm=95.713, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=0.002, train_time=3.329 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:31:32,390 (trainer:732) INFO: 43epoch:train:737-828batch: iter_time=3.626e-04, forward_time=0.256, loss_att=125.813, acc=0.885, loss=125.813, backward_time=0.383, grad_norm=99.629, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.002, train_time=3.286 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:32:50,182 (trainer:732) INFO: 43epoch:train:829-920batch: iter_time=3.557e-04, forward_time=0.263, loss_att=130.261, acc=0.888, loss=130.261, backward_time=0.390, grad_norm=99.477, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.381 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:34:07,765 (trainer:732) INFO: 43epoch:train:921-1012batch: iter_time=3.886e-04, forward_time=0.263, loss_att=127.790, acc=0.887, loss=127.790, backward_time=0.389, grad_norm=100.147, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.373 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:35:25,273 (trainer:732) INFO: 43epoch:train:1013-1104batch: iter_time=3.835e-04, forward_time=0.262, loss_att=126.194, acc=0.892, loss=126.194, backward_time=0.388, grad_norm=88.101, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.369 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:36:42,778 (trainer:732) INFO: 43epoch:train:1105-1196batch: iter_time=3.593e-04, forward_time=0.263, loss_att=125.903, acc=0.888, loss=125.903, backward_time=0.390, grad_norm=98.326, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.369 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:37:59,859 (trainer:732) INFO: 43epoch:train:1197-1288batch: iter_time=3.886e-04, forward_time=0.260, loss_att=123.825, acc=0.882, loss=123.825, backward_time=0.387, grad_norm=89.244, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.351 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:39:17,403 (trainer:732) INFO: 43epoch:train:1289-1380batch: iter_time=3.676e-04, forward_time=0.262, loss_att=121.143, acc=0.890, loss=121.143, backward_time=0.389, grad_norm=98.258, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.371 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:40:35,264 (trainer:732) INFO: 43epoch:train:1381-1472batch: iter_time=3.616e-04, forward_time=0.263, loss_att=129.274, acc=0.887, loss=129.274, backward_time=0.389, grad_norm=96.954, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.385 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:41:52,517 (trainer:732) INFO: 43epoch:train:1473-1564batch: iter_time=3.706e-04, forward_time=0.261, loss_att=123.033, acc=0.884, loss=123.033, backward_time=0.388, grad_norm=90.593, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.358 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:43:10,187 (trainer:732) INFO: 43epoch:train:1565-1656batch: iter_time=3.948e-04, forward_time=0.264, loss_att=125.882, acc=0.889, loss=125.882, backward_time=0.390, grad_norm=88.606, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.376 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:44:27,579 (trainer:732) INFO: 43epoch:train:1657-1748batch: iter_time=3.564e-04, forward_time=0.262, loss_att=122.306, acc=0.887, loss=122.306, backward_time=0.388, grad_norm=92.059, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.364 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:45:45,397 (trainer:732) INFO: 43epoch:train:1749-1840batch: iter_time=3.124e-04, forward_time=0.264, loss_att=129.623, acc=0.890, loss=129.623, backward_time=0.391, grad_norm=104.698, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.383 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:57:29,225 (trainer:338) INFO: 43epoch results: [train] iter_time=9.587e-04, forward_time=0.261, loss_att=125.108, acc=0.888, loss=125.108, backward_time=0.388, grad_norm=94.569, clip=100.000, loss_scale=1.000, optim_step_time=0.103, optim0_lr0=0.002, train_time=3.672, time=28 minutes and 15.87 seconds, total_count=79249, gpu_max_cached_mem_GB=29.846, [valid] loss_att=192.264, acc=0.840, cer=0.200, wer=0.488, loss=192.264, time=4 minutes and 42.84 seconds, total_count=817, gpu_max_cached_mem_GB=29.846, [att_plot] time=6 minutes and 54.41 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:57:38,590 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:57:38,617 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/34epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 02:57:38,618 (trainer:272) INFO: 44/60epoch started. Estimated time to finish: 11 hours, 51 minutes and 10.5 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:03:44,089 (trainer:732) INFO: 44epoch:train:1-92batch: iter_time=0.019, forward_time=0.278, loss_att=119.683, acc=0.893, loss=119.683, backward_time=0.390, grad_norm=96.572, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=0.002, train_time=15.888 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:05:04,154 (trainer:732) INFO: 44epoch:train:93-184batch: iter_time=5.461e-04, forward_time=0.268, loss_att=115.507, acc=0.894, loss=115.507, backward_time=0.388, grad_norm=86.725, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=0.002, train_time=3.480 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:06:24,419 (trainer:732) INFO: 44epoch:train:185-276batch: iter_time=6.474e-04, forward_time=0.274, loss_att=124.298, acc=0.891, loss=124.298, backward_time=0.393, grad_norm=91.860, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.002, train_time=3.489 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:07:44,157 (trainer:732) INFO: 44epoch:train:277-368batch: iter_time=5.411e-04, forward_time=0.273, loss_att=123.303, acc=0.893, loss=123.303, backward_time=0.396, grad_norm=93.834, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.466 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:09:04,052 (trainer:732) INFO: 44epoch:train:369-460batch: iter_time=6.069e-04, forward_time=0.274, loss_att=122.823, acc=0.892, loss=122.823, backward_time=0.392, grad_norm=98.438, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=0.002, train_time=3.473 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:10:24,351 (trainer:732) INFO: 44epoch:train:461-552batch: iter_time=4.886e-04, forward_time=0.274, loss_att=124.232, acc=0.890, loss=124.232, backward_time=0.393, grad_norm=100.782, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.491 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:11:43,759 (trainer:732) INFO: 44epoch:train:553-644batch: iter_time=4.654e-04, forward_time=0.272, loss_att=120.773, acc=0.890, loss=120.773, backward_time=0.390, grad_norm=96.527, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.452 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:13:03,104 (trainer:732) INFO: 44epoch:train:645-736batch: iter_time=4.231e-04, forward_time=0.269, loss_att=120.413, acc=0.889, loss=120.413, backward_time=0.390, grad_norm=96.049, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.002, train_time=3.449 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:14:23,486 (trainer:732) INFO: 44epoch:train:737-828batch: iter_time=6.486e-04, forward_time=0.275, loss_att=126.564, acc=0.889, loss=126.564, backward_time=0.394, grad_norm=99.064, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=0.002, train_time=3.494 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:15:42,989 (trainer:732) INFO: 44epoch:train:829-920batch: iter_time=5.789e-04, forward_time=0.275, loss_att=120.546, acc=0.890, loss=120.546, backward_time=0.392, grad_norm=89.006, clip=100.000, loss_scale=1.000, optim_step_time=0.112, optim0_lr0=0.002, train_time=3.456 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:17:01,569 (trainer:732) INFO: 44epoch:train:921-1012batch: iter_time=4.743e-04, forward_time=0.271, loss_att=120.437, acc=0.886, loss=120.437, backward_time=0.386, grad_norm=96.496, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.002, train_time=3.416 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:18:21,214 (trainer:732) INFO: 44epoch:train:1013-1104batch: iter_time=5.365e-04, forward_time=0.272, loss_att=123.610, acc=0.890, loss=123.610, backward_time=0.391, grad_norm=89.932, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.462 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:19:42,083 (trainer:732) INFO: 44epoch:train:1105-1196batch: iter_time=4.830e-04, forward_time=0.277, loss_att=126.709, acc=0.893, loss=126.709, backward_time=0.394, grad_norm=91.711, clip=100.000, loss_scale=1.000, optim_step_time=0.138, optim0_lr0=0.002, train_time=3.515 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:21:01,989 (trainer:732) INFO: 44epoch:train:1197-1288batch: iter_time=4.519e-04, forward_time=0.275, loss_att=122.041, acc=0.892, loss=122.041, backward_time=0.398, grad_norm=99.912, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.473 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:22:20,579 (trainer:732) INFO: 44epoch:train:1289-1380batch: iter_time=5.010e-04, forward_time=0.269, loss_att=124.784, acc=0.883, loss=124.784, backward_time=0.388, grad_norm=102.800, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.002, train_time=3.416 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:23:40,209 (trainer:732) INFO: 44epoch:train:1381-1472batch: iter_time=5.872e-04, forward_time=0.275, loss_att=121.634, acc=0.893, loss=121.634, backward_time=0.391, grad_norm=93.306, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.002, train_time=3.461 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:24:59,721 (trainer:732) INFO: 44epoch:train:1473-1564batch: iter_time=4.836e-04, forward_time=0.270, loss_att=125.302, acc=0.890, loss=125.302, backward_time=0.388, grad_norm=87.875, clip=100.000, loss_scale=1.000, optim_step_time=0.116, optim0_lr0=0.002, train_time=3.456 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:26:18,636 (trainer:732) INFO: 44epoch:train:1565-1656batch: iter_time=4.445e-04, forward_time=0.272, loss_att=126.329, acc=0.891, loss=126.329, backward_time=0.390, grad_norm=99.664, clip=100.000, loss_scale=1.000, optim_step_time=0.111, optim0_lr0=0.002, train_time=3.430 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:27:38,473 (trainer:732) INFO: 44epoch:train:1657-1748batch: iter_time=6.475e-04, forward_time=0.271, loss_att=124.067, acc=0.889, loss=124.067, backward_time=0.391, grad_norm=86.358, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.002, train_time=3.470 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:28:57,475 (trainer:732) INFO: 44epoch:train:1749-1840batch: iter_time=4.013e-04, forward_time=0.270, loss_att=119.761, acc=0.889, loss=119.761, backward_time=0.391, grad_norm=89.987, clip=100.000, loss_scale=1.000, optim_step_time=0.119, optim0_lr0=0.002, train_time=3.434 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:41:24,581 (trainer:338) INFO: 44epoch results: [train] iter_time=0.001, forward_time=0.273, loss_att=122.603, acc=0.890, loss=122.603, backward_time=0.391, grad_norm=94.345, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.002, train_time=4.084, time=31 minutes and 25.35 seconds, total_count=81092, gpu_max_cached_mem_GB=29.846, [valid] loss_att=191.121, acc=0.841, cer=0.204, wer=0.493, loss=191.121, time=5 minutes and 7.79 seconds, total_count=836, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 12.82 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:41:34,006 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:41:34,041 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/33epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:41:34,041 (trainer:272) INFO: 45/60epoch started. Estimated time to finish: 11 hours, 10 minutes and 6.07 seconds +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:47:25,884 (trainer:732) INFO: 45epoch:train:1-92batch: iter_time=0.018, forward_time=0.286, loss_att=109.803, acc=0.894, loss=109.803, backward_time=0.397, grad_norm=94.359, clip=100.000, loss_scale=1.000, optim_step_time=0.154, optim0_lr0=0.002, train_time=15.295 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:48:45,774 (trainer:732) INFO: 45epoch:train:93-184batch: iter_time=5.182e-04, forward_time=0.274, loss_att=112.716, acc=0.896, loss=112.716, backward_time=0.391, grad_norm=86.649, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.473 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:50:05,805 (trainer:732) INFO: 45epoch:train:185-276batch: iter_time=4.854e-04, forward_time=0.270, loss_att=121.766, acc=0.890, loss=121.766, backward_time=0.393, grad_norm=88.680, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.002, train_time=3.479 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:51:23,802 (trainer:732) INFO: 45epoch:train:277-368batch: iter_time=4.252e-04, forward_time=0.265, loss_att=119.933, acc=0.892, loss=119.933, backward_time=0.387, grad_norm=95.396, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.002, train_time=3.390 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:52:43,578 (trainer:732) INFO: 45epoch:train:369-460batch: iter_time=4.361e-04, forward_time=0.274, loss_att=122.676, acc=0.895, loss=122.676, backward_time=0.392, grad_norm=100.652, clip=100.000, loss_scale=1.000, optim_step_time=0.105, optim0_lr0=0.002, train_time=3.468 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:54:03,758 (trainer:732) INFO: 45epoch:train:461-552batch: iter_time=5.214e-04, forward_time=0.270, loss_att=118.331, acc=0.898, loss=118.331, backward_time=0.390, grad_norm=89.319, clip=100.000, loss_scale=1.000, optim_step_time=0.125, optim0_lr0=0.002, train_time=3.485 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:55:22,338 (trainer:732) INFO: 45epoch:train:553-644batch: iter_time=6.290e-04, forward_time=0.268, loss_att=115.669, acc=0.890, loss=115.669, backward_time=0.384, grad_norm=80.047, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=0.002, train_time=3.416 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:56:41,077 (trainer:732) INFO: 45epoch:train:645-736batch: iter_time=5.620e-04, forward_time=0.268, loss_att=118.582, acc=0.896, loss=118.582, backward_time=0.389, grad_norm=100.232, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.423 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:57:59,451 (trainer:732) INFO: 45epoch:train:737-828batch: iter_time=4.630e-04, forward_time=0.269, loss_att=118.339, acc=0.896, loss=118.339, backward_time=0.389, grad_norm=93.899, clip=100.000, loss_scale=1.000, optim_step_time=0.109, optim0_lr0=0.002, train_time=3.407 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 03:59:18,606 (trainer:732) INFO: 45epoch:train:829-920batch: iter_time=5.171e-04, forward_time=0.271, loss_att=122.856, acc=0.895, loss=122.856, backward_time=0.390, grad_norm=80.606, clip=100.000, loss_scale=1.000, optim_step_time=0.107, optim0_lr0=0.002, train_time=3.441 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:00:38,516 (trainer:732) INFO: 45epoch:train:921-1012batch: iter_time=6.470e-04, forward_time=0.274, loss_att=122.738, acc=0.890, loss=122.738, backward_time=0.393, grad_norm=94.236, clip=100.000, loss_scale=1.000, optim_step_time=0.122, optim0_lr0=0.002, train_time=3.474 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:01:58,814 (trainer:732) INFO: 45epoch:train:1013-1104batch: iter_time=6.297e-04, forward_time=0.275, loss_att=119.964, acc=0.895, loss=119.964, backward_time=0.395, grad_norm=111.995, clip=100.000, loss_scale=1.000, optim_step_time=0.115, optim0_lr0=0.002, train_time=3.491 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:03:17,834 (trainer:732) INFO: 45epoch:train:1105-1196batch: iter_time=5.330e-04, forward_time=0.273, loss_att=115.555, acc=0.894, loss=115.555, backward_time=0.389, grad_norm=91.009, clip=100.000, loss_scale=1.000, optim_step_time=0.113, optim0_lr0=0.002, train_time=3.435 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:04:38,159 (trainer:732) INFO: 45epoch:train:1197-1288batch: iter_time=5.507e-04, forward_time=0.277, loss_att=121.079, acc=0.894, loss=121.079, backward_time=0.394, grad_norm=101.865, clip=100.000, loss_scale=1.000, optim_step_time=0.120, optim0_lr0=0.002, train_time=3.492 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:05:57,196 (trainer:732) INFO: 45epoch:train:1289-1380batch: iter_time=4.784e-04, forward_time=0.271, loss_att=121.271, acc=0.895, loss=121.271, backward_time=0.390, grad_norm=86.959, clip=100.000, loss_scale=1.000, optim_step_time=0.114, optim0_lr0=0.002, train_time=3.436 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:07:17,832 (trainer:732) INFO: 45epoch:train:1381-1472batch: iter_time=4.884e-04, forward_time=0.278, loss_att=123.265, acc=0.895, loss=123.265, backward_time=0.398, grad_norm=102.805, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.002, train_time=3.505 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:08:36,284 (trainer:732) INFO: 45epoch:train:1473-1564batch: iter_time=5.075e-04, forward_time=0.267, loss_att=117.607, acc=0.889, loss=117.607, backward_time=0.388, grad_norm=98.660, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.002, train_time=3.410 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:09:55,052 (trainer:732) INFO: 45epoch:train:1565-1656batch: iter_time=4.585e-04, forward_time=0.269, loss_att=118.102, acc=0.888, loss=118.102, backward_time=0.385, grad_norm=87.150, clip=100.000, loss_scale=1.000, optim_step_time=0.110, optim0_lr0=0.002, train_time=3.424 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:11:14,084 (trainer:732) INFO: 45epoch:train:1657-1748batch: iter_time=4.890e-04, forward_time=0.272, loss_att=119.133, acc=0.891, loss=119.133, backward_time=0.389, grad_norm=88.027, clip=100.000, loss_scale=1.000, optim_step_time=0.118, optim0_lr0=0.002, train_time=3.435 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:12:33,517 (trainer:732) INFO: 45epoch:train:1749-1840batch: iter_time=3.732e-04, forward_time=0.271, loss_att=123.519, acc=0.892, loss=123.519, backward_time=0.393, grad_norm=97.955, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.002, train_time=3.453 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:24:46,819 (trainer:338) INFO: 45epoch results: [train] iter_time=0.001, forward_time=0.272, loss_att=119.072, acc=0.893, loss=119.072, backward_time=0.391, grad_norm=93.525, clip=100.000, loss_scale=1.000, optim_step_time=0.117, optim0_lr0=0.002, train_time=4.042, time=31 minutes and 6.33 seconds, total_count=82935, gpu_max_cached_mem_GB=29.846, [valid] loss_att=187.471, acc=0.845, cer=0.195, wer=0.480, loss=187.471, time=5 minutes and 5.89 seconds, total_count=855, gpu_max_cached_mem_GB=29.846, [att_plot] time=7 minutes and 0.54 seconds, total_count=0, gpu_max_cached_mem_GB=29.846 +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:24:56,036 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:24:56,063 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/35epoch.pth +[de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:0/2] 2023-11-18 04:24:56,063 (trainer:272) INFO: 46/60epoch started. Estimated time to finish: 10 hours, 28 minutes and 42.9 seconds + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<56345> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<57023> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<62548> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<37319> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<39508> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<56627> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<16344> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<33778> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<44001> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 131) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 131) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<19171> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<19043> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 126) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 136) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<52631> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 136) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 136) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<52805> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 136) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 137) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 137) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 137) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 137) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 137) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101480:2103965 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 137) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<39528> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<49081> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<42527> +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 128) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-9-0208143539-7dbf569d4f-r7nrb:2101481:2103966 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 129) +Traceback (most recent call last): + File "/star-home/jinzengrui/lib/miniconda3/envs/dev39/lib/python3.9/runpy.py", line 197, in _run_module_as_main + return _run_code(code, main_globals, None, + File "/star-home/jinzengrui/lib/miniconda3/envs/dev39/lib/python3.9/runpy.py", line 87, in _run_code + exec(code, run_globals) + File "/star-home/jinzengrui/lib/miniconda3/envs/dev39/lib/python3.9/site-packages/espnet-202308-py3.9.egg/espnet2/bin/asr_train.py", line 23, in + main() + File "/star-home/jinzengrui/lib/miniconda3/envs/dev39/lib/python3.9/site-packages/espnet-202308-py3.9.egg/espnet2/bin/asr_train.py", line 19, in main + ASRTask.main(cmd=cmd) + File "/star-home/jinzengrui/lib/miniconda3/envs/dev39/lib/python3.9/site-packages/espnet-202308-py3.9.egg/espnet2/tasks/abs_task.py", line 1132, in main + while not ProcessContext(processes, error_queues).join(): + File "/star-home/jinzengrui/lib/miniconda3/envs/dev39/lib/python3.9/site-packages/torch/multiprocessing/spawn.py", line 140, in join + raise ProcessExitedException( +torch.multiprocessing.spawn.ProcessExitedException: process 0 terminated with signal SIGKILL +# Accounting: time=113579 threads=1 +# Ended (code 1) at Sat Nov 18 04:31:06 CST 2023, elapsed time 113579 seconds +/star-home/jinzengrui/lib/miniconda3/envs/dev39/lib/python3.9/multiprocessing/resource_tracker.py:216: UserWarning: resource_tracker: There appear to be 112 leaked semaphore objects to clean up at shutdown + warnings.warn('resource_tracker: There appear to be %d ' diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/valid.acc.best.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/valid.acc.best.pth new file mode 100644 index 0000000000000000000000000000000000000000..261caa86ba8036b31be7b22c4cb58f0bd435f10a --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new/valid.acc.best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b27ef31ab1e2e5974a590d9f464fc233cb8c99c2e55bf7708f403dd84b2add7b +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/51epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/51epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..2c732a7a31113525446052ca09da4d748be240e5 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/51epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8f1b493b1613f5da3d77832f43f9c7fe0f50d9f1f200669037d51bb6c3a9b8bd +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/52epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/52epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..a5327205c317974e5f81e128e9f51d1b58c7d56f --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/52epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7726bb1977608f1088f61ed42209b7a9bb1836944e59c6baf0fcd2c639cc658c +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/53epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/53epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..9db1358a4f5780cd5e4a0003a9c379747e46ff8c --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/53epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db4f175139f8a7a3f0165567e214db2edceef3982fd4e20898afe7ef6f5e3a8a +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/54epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/54epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..a8e9c7a9ad98a1d4e26ea6026e6403a9e412d82d --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/54epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9ff56cf1f853ba75bf1849975c7648bf947424186ca3958b03973977ef57e8cc +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/55epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/55epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..05e349db9042129d703d344aa4dd3e4343ac3e7d --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/55epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6466de0d2c33ccc74f6ac3dfe5a9dfedcce6534767381c4b367ab5984ed94cbb +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/56epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/56epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..100f96dd052053a65c7fcec8189fa4ee87105941 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/56epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:92f209ba881aa29c698f7272a5e2166646b374b167941865153a6bad1f2a4cff +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/57epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/57epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..739f16873f70073215e0f827a16336ca6f400031 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/57epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3bb9da49349d1027c17302dfdaa9dc307147e68c306723d447097f57d11cadbb +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/58epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/58epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..0cf1c9b487b01bc30c03d5c1c82f3be2b07863ac --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/58epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:262dc26dbdb3d0433518cb3bb919765f939209630860e0da68a6b4a8024606ca +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/59epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/59epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..e1fd095506889ff17beaad9dc196efd92106b832 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/59epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:93a466e2bdee6f717582e8f55b8cf7d2d189c6a38d7268196dcebd12b7b0ec2b +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/60epoch.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/60epoch.pth new file mode 100644 index 0000000000000000000000000000000000000000..07cffd1b1c091ad8875aa1b808929430842084db --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/60epoch.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc0710e1565fce6d3ce275190a1335a03d912f728a67066516df16d751f26514 +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/RESULTS.md b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/RESULTS.md new file mode 100644 index 0000000000000000000000000000000000000000..35e1c127b34392b247aea94319c157802927bb57 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/RESULTS.md @@ -0,0 +1,51 @@ + +# RESULTS +## Environments +- date: `Tue Mar 5 10:57:33 CST 2024` +- python version: `3.9.18 (main, Sep 11 2023, 13:41:44) [GCC 11.2.0]` +- espnet version: `espnet 202308` +- pytorch version: `pytorch 1.12.1+cu116` +- Git hash: `884659f9ee95374811015381c976fa3b4f6e01db` + - Commit date: `Thu Nov 23 00:23:29 2023 +0800` + +## exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr +### WER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +|decode_sot_asr_model_valid.acc.best/dev_2spk|3315|226216|58.5|26.0|15.5|12.5|54.0|100.0| +|decode_sot_asr_model_valid.acc.best/dev_2spk_kaldi_fmt|1606|135101|62.0|23.4|14.6|10.4|48.3|100.0| +|decode_sot_asr_model_valid.acc.best/dev_3spk|2059|209679|41.4|38.0|20.5|9.3|67.8|100.0| +|decode_sot_asr_model_valid.acc.best/dev_4spk|1467|200029|33.3|41.0|25.7|6.5|73.3|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_2spk|4570|301042|58.6|26.3|15.1|12.4|53.8|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_3spk|2072|212871|40.4|38.8|20.8|10.8|70.4|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_4spk|1326|185394|32.5|41.2|26.3|7.1|74.6|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_2spk|4663|336490|54.9|30.5|14.6|12.4|57.5|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_3spk|2453|266074|37.0|43.0|20.0|10.0|73.1|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_4spk|1795|259138|29.2|44.9|25.9|6.4|77.3|100.0| +|decode_sot_asr_model_valid.acc.best/test_clean_2spk_kaldi_fmt|2180|178761|62.5|24.0|13.5|10.1|47.6|100.0| +|decode_sot_asr_model_valid.acc.best/test_other_2spk_kaldi_fmt|2363|205496|58.1|28.9|13.1|10.1|52.1|100.0| +|decode_sot_asr_model_valid.acc.best/tt_mix_clean_reverb_max_16k|3000|3000|0.0|100.0|0.0|3115.7|3215.7|100.0| + +### CER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| +|decode_sot_asr_model_valid.acc.best/dev_2spk|3315|1230801|72.5|12.8|14.6|10.4|37.9|100.0| +|decode_sot_asr_model_valid.acc.best/dev_2spk_kaldi_fmt|1606|735694|75.0|11.2|13.8|8.2|33.2|100.0| +|decode_sot_asr_model_valid.acc.best/dev_3spk|2059|1140428|59.7|17.4|22.9|9.2|49.5|100.0| +|decode_sot_asr_model_valid.acc.best/dev_4spk|1467|1087409|52.4|18.0|29.6|7.4|54.9|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_2spk|4570|1550429|73.0|13.0|14.0|10.5|37.5|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_3spk|2072|1084475|59.1|18.6|22.3|10.8|51.7|100.0| +|decode_sot_asr_model_valid.acc.best/test-clean_4spk|1326|938467|52.2|18.9|28.9|8.2|56.0|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_2spk|4663|1742136|71.1|14.7|14.1|10.8|39.7|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_3spk|2453|1381987|57.5|19.5|23.0|10.7|53.2|100.0| +|decode_sot_asr_model_valid.acc.best/test-other_4spk|1795|1346646|50.2|19.8|30.1|7.8|57.6|100.0| +|decode_sot_asr_model_valid.acc.best/test_clean_2spk_kaldi_fmt|2180|921344|75.9|11.5|12.6|8.3|32.3|100.0| +|decode_sot_asr_model_valid.acc.best/test_other_2spk_kaldi_fmt|2363|1064868|73.7|13.3|13.0|8.8|35.1|100.0| +|decode_sot_asr_model_valid.acc.best/tt_mix_clean_reverb_max_16k|3000|143026|16.3|83.6|0.2|298.3|382.0|100.0| + +### TER + +|dataset|Snt|Wrd|Corr|Sub|Del|Ins|Err|S.Err| +|---|---|---|---|---|---|---|---|---| diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/checkpoint.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/checkpoint.pth new file mode 100644 index 0000000000000000000000000000000000000000..16ffe9408a6fcd1e27bd818f1dd24cc9f52c3815 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/checkpoint.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a89e7a0173da35e614c843035bef69d1f419c73a219c247975e38b71dd606062 +size 516972446 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/config.yaml b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..58519bed612b94928448342f7c6ca9258a51b3f1 --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/config.yaml @@ -0,0 +1,227 @@ +config: conf/tuning/train_sot_asr_conformer.yaml +print_config: false +log_level: INFO +drop_last_iter: false +dry_run: false +iterator_type: sequence +valid_iterator_type: null +output_dir: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr +ngpu: 1 +seed: 0 +num_workers: 16 +num_att_plot: 3 +dist_backend: nccl +dist_init_method: env:// +dist_world_size: 4 +dist_rank: 0 +local_rank: 0 +dist_master_addr: localhost +dist_master_port: 53697 +dist_launcher: null +multiprocessing_distributed: true +unused_parameters: false +sharded_ddp: false +cudnn_enabled: true +cudnn_benchmark: false +cudnn_deterministic: true +collect_stats: false +write_collected_feats: false +max_epoch: 60 +patience: null +val_scheduler_criterion: +- valid +- loss +early_stopping_criterion: +- valid +- loss +- min +best_model_criterion: +- - valid + - acc + - max +keep_nbest_models: 10 +nbest_averaging_interval: 0 +grad_clip: 5.0 +grad_clip_type: 2.0 +grad_noise: false +accum_grad: 4 +no_forward_run: false +resume: true +train_dtype: float32 +use_amp: false +log_interval: null +use_matplotlib: true +use_tensorboard: true +create_graph_in_tensorboard: false +use_wandb: false +wandb_project: null +wandb_id: null +wandb_entity: null +wandb_name: null +wandb_model_log_interval: -1 +detect_anomaly: false +pretrain_path: null +init_param: +- /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth +ignore_init_mismatch: false +freeze_param: [] +num_iters_per_epoch: null +batch_size: 20 +valid_batch_size: null +batch_bins: 10000000 +valid_batch_bins: null +train_shape_file: +- exp/asr_stats_raw_en_char/train/speech_shape +- exp/asr_stats_raw_en_char/train/text_shape.char +valid_shape_file: +- exp/asr_stats_raw_en_char/valid/speech_shape +- exp/asr_stats_raw_en_char/valid/text_shape.char +batch_type: numel +valid_batch_type: null +fold_length: +- 80000 +- 150 +sort_in_batch: descending +shuffle_within_batch: false +sort_batch: descending +multiple_iterator: false +chunk_length: 500 +chunk_shift_ratio: 0.5 +num_cache_chunks: 1024 +chunk_excluded_key_prefixes: [] +train_data_path_and_name_and_type: +- - dump/raw/small_w_whamr_sp/wav.scp + - speech + - kaldi_ark +- - dump/raw/small_w_whamr_sp/text + - text + - text +valid_data_path_and_name_and_type: +- - dump/raw/cv_mix_clean_reverb_max_16k/wav.scp + - speech + - kaldi_ark +- - dump/raw/cv_mix_clean_reverb_max_16k/text + - text + - text +allow_variable_data_keys: false +max_cache_size: 0.0 +max_cache_fd: 32 +valid_max_cache_size: null +exclude_weight_decay: false +exclude_weight_decay_conf: {} +optim: adam +optim_conf: + lr: 0.002 + weight_decay: 1.0e-06 +scheduler: warmuplr +scheduler_conf: + warmup_steps: 20000 +token_list: +- +- +- +- +- E +- T +- A +- O +- N +- I +- H +- S +- R +- D +- L +- U +- M +- C +- W +- F +- G +- Y +- P +- B +- V +- K +- '''' +- X +- J +- Q +- Z +- +init: null +input_size: null +ctc_conf: + dropout_rate: 0.0 + ctc_type: builtin + reduce: true + ignore_nan_grad: null + zero_infinity: true +joint_net_conf: null +use_preprocessor: true +token_type: char +bpemodel: null +non_linguistic_symbols: null +cleaner: null +g2p: null +speech_volume_normalize: null +rir_scp: null +rir_apply_prob: 1.0 +noise_scp: null +noise_apply_prob: 1.0 +noise_db_range: '13_15' +short_noise_thres: 0.5 +aux_ctc_tasks: [] +frontend: default +frontend_conf: + fs: 16k +specaug: null +specaug_conf: {} +normalize: global_mvn +normalize_conf: + stats_file: exp/asr_stats_raw_en_char/train/feats_stats.npz +model: espnet +model_conf: + ctc_weight: 0.0 + lsm_weight: 0.1 + length_normalized_loss: false +preencoder: null +preencoder_conf: {} +encoder: conformer +encoder_conf: + output_size: 256 + attention_heads: 4 + linear_units: 2048 + num_blocks: 12 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + attention_dropout_rate: 0.1 + input_layer: conv2d + normalize_before: true + macaron_style: true + rel_pos_type: latest + pos_enc_layer_type: rel_pos + selfattention_layer_type: rel_selfattn + activation_type: swish + use_cnn_module: true + cnn_module_kernel: 31 +postencoder: null +postencoder_conf: {} +decoder: transformer +decoder_conf: + attention_heads: 4 + linear_units: 2048 + num_blocks: 6 + dropout_rate: 0.1 + positional_dropout_rate: 0.1 + self_attention_dropout_rate: 0.1 + src_attention_dropout_rate: 0.1 +preprocessor: multi +preprocessor_conf: + speaker_change_symbol: + - +required: +- output_dir +- token_list +version: '202308' +distributed: true diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/acc.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/acc.png new file mode 100644 index 0000000000000000000000000000000000000000..6c384a2beecbede0a3f9bbfafcc54e1e9f60430b Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/acc.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/backward_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/backward_time.png new file mode 100644 index 0000000000000000000000000000000000000000..2fb02dbfcec2447d29523913a41958c5a72ae056 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/backward_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/cer.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/cer.png new file mode 100644 index 0000000000000000000000000000000000000000..e907a0f9ee09e655d1a2ffff8301da1692782fad Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/cer.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/clip.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/clip.png new file mode 100644 index 0000000000000000000000000000000000000000..8dcd44eeb8e61cefd50db5514c608663bad42973 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/clip.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/forward_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/forward_time.png new file mode 100644 index 0000000000000000000000000000000000000000..9a70df2f20ac59880fc3e42cae563fe514d783fd Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/forward_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/gpu_max_cached_mem_GB.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/gpu_max_cached_mem_GB.png new file mode 100644 index 0000000000000000000000000000000000000000..2518d7fbfb3f7b8a184430dd686ba4f564b24aa4 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/gpu_max_cached_mem_GB.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/grad_norm.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/grad_norm.png new file mode 100644 index 0000000000000000000000000000000000000000..ed9486b364200ab88a2ada4e22a8b260e8142ea1 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/grad_norm.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/iter_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/iter_time.png new file mode 100644 index 0000000000000000000000000000000000000000..b2e2b857de765cb53d80108e185ba6337a5cdd33 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/iter_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/loss.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/loss.png new file mode 100644 index 0000000000000000000000000000000000000000..83c226ebe64a4c5fc24bf362402770779cb388b5 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/loss.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/loss_att.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/loss_att.png new file mode 100644 index 0000000000000000000000000000000000000000..345ec6c25c75e1b37b54c75cf519a285ea3c7e4f Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/loss_att.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/loss_scale.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/loss_scale.png new file mode 100644 index 0000000000000000000000000000000000000000..460bfdb13d9e33389b68b962751b15221d3e531f Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/loss_scale.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/optim0_lr0.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/optim0_lr0.png new file mode 100644 index 0000000000000000000000000000000000000000..49339169aa0bb14c0ba08debeb91ff9801c52009 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/optim0_lr0.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/optim_step_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/optim_step_time.png new file mode 100644 index 0000000000000000000000000000000000000000..7cf9d9280ff8270437a5dc2cf0a904bde3d12037 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/optim_step_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/train_time.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/train_time.png new file mode 100644 index 0000000000000000000000000000000000000000..1e9b57c448023dba46d18e6b15216a79d08a8869 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/train_time.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/wer.png b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/wer.png new file mode 100644 index 0000000000000000000000000000000000000000..fb7a63a84f3b75f66dcce64835b27ee342c37354 Binary files /dev/null and b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/images/wer.png differ diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/latest.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/latest.pth new file mode 100644 index 0000000000000000000000000000000000000000..07cffd1b1c091ad8875aa1b808929430842084db --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/latest.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc0710e1565fce6d3ce275190a1335a03d912f728a67066516df16d751f26514 +size 172367337 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/run.sh b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/run.sh new file mode 100644 index 0000000000000000000000000000000000000000..4f3ab9818390368d5b6704209354d9f13104074d --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/run.sh @@ -0,0 +1 @@ +./asr.sh --lang en --audio_format flac.ark --stage 11 --stop_stage 15 --speed_perturb_factors '0.9 1.0 1.1' --feats_type raw --token_type char --sot_asr true --max_wav_duration 50 --speed_perturb_factors '' --feats_normalize global_mvn --use_lm false --pretrained_model /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth --asr_config conf/tuning/train_sot_asr_conformer.yaml --lm_config conf/tuning/train_lm_transformer.yaml --inference_config conf/tuning/decode_sot.yaml --train_set small_w_whamr --valid_set cv_mix_clean_reverb_max_16k --test_sets tt_mix_clean_reverb_max_16k --ngpu 4 --asr_tag train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr --lm_train_text data/local/other_text/text --bpe_train_text data/small_w_whamr/text --stage 11 "$@"; exit $? diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/train/events.out.tfevents.1708419253.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3131049.0 b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/train/events.out.tfevents.1708419253.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3131049.0 new file mode 100644 index 0000000000000000000000000000000000000000..3be5c604fc868b40854cba4fcf62692f14f58d3e --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/train/events.out.tfevents.1708419253.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3131049.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:212d02e36acee8be487e3e88ac5e843901520fdb93a2fba1f189550f1fcc697e +size 5263 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/train/events.out.tfevents.1708420165.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3146342.0 b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/train/events.out.tfevents.1708420165.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3146342.0 new file mode 100644 index 0000000000000000000000000000000000000000..f9a20cdfdf04a49521470a9e47930fc79adbf46a --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/train/events.out.tfevents.1708420165.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3146342.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:41e257bad088527aac71cbe7e15bd09ff5ba6487ff620c5f23251854ba7485e3 +size 834501589 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/valid/events.out.tfevents.1708419253.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3131049.1 b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/valid/events.out.tfevents.1708419253.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3131049.1 new file mode 100644 index 0000000000000000000000000000000000000000..502046e377bb343353e30fcdcfe784584648408f --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/valid/events.out.tfevents.1708419253.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3131049.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:add2c38e18320a5a23bd54a712f1b1fb986178f9de3a1b5692f66275699dc3f5 +size 88 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/valid/events.out.tfevents.1708420165.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3146342.1 b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/valid/events.out.tfevents.1708420165.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3146342.1 new file mode 100644 index 0000000000000000000000000000000000000000..8f52e013b3d5ed4775a93e32145bec7cffa09e7c --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/tensorboard/valid/events.out.tfevents.1708420165.de-74279-k2-train-3-0115153910-566d497669-rr6jb.3146342.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43d70bde83a46695ccee35c33bc37bd8b0cb052c6d5953fef776a73ca49a4660 +size 16978 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/train.log b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/train.log new file mode 100644 index 0000000000000000000000000000000000000000..de9b94225724b66f24d18fda40f74f33e2f6974b --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/train.log @@ -0,0 +1,3171 @@ +# python3 -m espnet2.bin.asr_train --use_preprocessor true --bpemodel none --token_type char --token_list data/en_token_list/char/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/cv_mix_clean_reverb_max_16k/wav.scp,speech,kaldi_ark --valid_shape_file exp/asr_stats_raw_en_char/valid/speech_shape --resume true --init_param /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth --ignore_init_mismatch false --fold_length 80000 --output_dir exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr --config conf/tuning/train_sot_asr_conformer.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/asr_stats_raw_en_char/train/feats_stats.npz --train_data_path_and_name_and_type dump/raw/small_w_whamr_sp/wav.scp,speech,kaldi_ark --train_shape_file exp/asr_stats_raw_en_char/train/speech_shape --fold_length 150 --train_data_path_and_name_and_type dump/raw/small_w_whamr_sp/text,text,text --train_shape_file exp/asr_stats_raw_en_char/train/text_shape.char --valid_data_path_and_name_and_type dump/raw/cv_mix_clean_reverb_max_16k/text,text,text --valid_shape_file exp/asr_stats_raw_en_char/valid/text_shape.char --ngpu 4 --multiprocessing_distributed True +# Started at Tue Feb 20 17:08:53 CST 2024 +# +/star-home/jinzengrui/lib/miniconda3/envs/espnet/bin/python3 /star-home/jinzengrui/lib/miniconda3/envs/espnet/lib/python3.9/site-packages/espnet-202308-py3.9.egg/espnet2/bin/asr_train.py --use_preprocessor true --bpemodel none --token_type char --token_list data/en_token_list/char/tokens.txt --non_linguistic_symbols none --cleaner none --g2p none --valid_data_path_and_name_and_type dump/raw/cv_mix_clean_reverb_max_16k/wav.scp,speech,kaldi_ark --valid_shape_file exp/asr_stats_raw_en_char/valid/speech_shape --resume true --init_param /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth --ignore_init_mismatch false --fold_length 80000 --output_dir exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr --config conf/tuning/train_sot_asr_conformer.yaml --frontend_conf fs=16k --normalize=global_mvn --normalize_conf stats_file=exp/asr_stats_raw_en_char/train/feats_stats.npz --train_data_path_and_name_and_type dump/raw/small_w_whamr_sp/wav.scp,speech,kaldi_ark --train_shape_file exp/asr_stats_raw_en_char/train/speech_shape --fold_length 150 --train_data_path_and_name_and_type dump/raw/small_w_whamr_sp/text,text,text --train_shape_file exp/asr_stats_raw_en_char/train/text_shape.char --valid_data_path_and_name_and_type dump/raw/cv_mix_clean_reverb_max_16k/text,text,text --valid_shape_file exp/asr_stats_raw_en_char/valid/text_shape.char --ngpu 4 --multiprocessing_distributed True +[W socket.cpp:558] [c10d] The client socket has failed to connect to [localhost]:53697 (errno: 99 - Cannot assign requested address). +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:18,239 (distributed_c10d:228) INFO: Added key: store_based_barrier_key:1 to store for rank: 0 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:18,239 (distributed_c10d:262) INFO: Rank 0: Completed store-based barrier for key:store_based_barrier_key:1 with 4 nodes. +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:18,263 (asr:490) INFO: Vocabulary size: 32 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:21,584 (abs_task:1229) INFO: pytorch.version=1.12.1+cu116, cuda.available=True, cudnn.version=8302, cudnn.benchmark=False, cudnn.deterministic=True +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:21,594 (abs_task:1230) INFO: Model structure: +ESPnetASRModel( + (frontend): DefaultFrontend( + (stft): Stft(n_fft=512, win_length=512, hop_length=128, center=True, normalized=False, onesided=True) + (frontend): Frontend() + (logmel): LogMel(sr=16000, n_fft=512, n_mels=80, fmin=0, fmax=8000.0, htk=False) + ) + (normalize): GlobalMVN(stats_file=exp/asr_stats_raw_en_char/train/feats_stats.npz, norm_means=True, norm_vars=True) + (encoder): ConformerEncoder( + (embed): Conv2dSubsampling( + (conv): Sequential( + (0): Conv2d(1, 256, kernel_size=(3, 3), stride=(2, 2)) + (1): ReLU() + (2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2)) + (3): ReLU() + ) + (out): Sequential( + (0): Linear(in_features=4864, out_features=256, bias=True) + (1): RelPositionalEncoding( + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + ) + (encoders): MultiSequential( + (0): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (1): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (2): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (3): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (4): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (5): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (6): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (7): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (8): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (9): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (10): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (11): EncoderLayer( + (self_attn): RelPositionMultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (linear_pos): Linear(in_features=256, out_features=256, bias=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (feed_forward_macaron): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): Swish() + ) + (conv_module): ConvolutionModule( + (pointwise_conv1): Conv1d(256, 512, kernel_size=(1,), stride=(1,)) + (depthwise_conv): Conv1d(256, 256, kernel_size=(31,), stride=(1,), padding=(15,), groups=256) + (norm): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True) + (pointwise_conv2): Conv1d(256, 256, kernel_size=(1,), stride=(1,)) + (activation): Swish() + ) + (norm_ff): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_mha): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_ff_macaron): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_conv): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm_final): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (after_norm): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + ) + (decoder): TransformerDecoder( + (embed): Sequential( + (0): Embedding(32, 256) + (1): PositionalEncoding( + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + (after_norm): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (output_layer): Linear(in_features=256, out_features=32, bias=True) + (decoders): MultiSequential( + (0): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (1): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (2): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (3): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (4): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (5): DecoderLayer( + (self_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (src_attn): MultiHeadedAttention( + (linear_q): Linear(in_features=256, out_features=256, bias=True) + (linear_k): Linear(in_features=256, out_features=256, bias=True) + (linear_v): Linear(in_features=256, out_features=256, bias=True) + (linear_out): Linear(in_features=256, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + (feed_forward): PositionwiseFeedForward( + (w_1): Linear(in_features=256, out_features=2048, bias=True) + (w_2): Linear(in_features=2048, out_features=256, bias=True) + (dropout): Dropout(p=0.1, inplace=False) + (activation): ReLU() + ) + (norm1): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm2): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (norm3): LayerNorm((256,), eps=1e-12, elementwise_affine=True) + (dropout): Dropout(p=0.1, inplace=False) + ) + ) + ) + (criterion_att): LabelSmoothingLoss( + (criterion): KLDivLoss() + ) +) + +Model summary: + Class Name: ESPnetASRModel + Total Number of model parameters: 43.00 M + Number of trainable parameters: 43.00 M (100.0%) + Size: 172.01 MB + Type: torch.float32 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:21,594 (abs_task:1233) INFO: Optimizer: +Adam ( +Parameter Group 0 + amsgrad: False + betas: (0.9, 0.999) + capturable: False + eps: 1e-08 + foreach: None + initial_lr: 0.002 + lr: 1e-07 + maximize: False + weight_decay: 1e-06 +) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:21,594 (abs_task:1234) INFO: Scheduler: WarmupLR(warmup_steps=20000) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:21,596 (abs_task:1243) INFO: Saving the configuration in exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/config.yaml +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:21,610 (abs_task:1304) INFO: Loading pretrained params from /star-home/jinzengrui/dev/espnet/egs2/librimix/sot_asr1_pretrain/exp/asr_train_sot_asr_conformer_raw_en_char_sp/45epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:22,637 (asr:461) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4') +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,255 (abs_task:1614) INFO: [train] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/small_w_whamr_sp/wav.scp", "type": "kaldi_ark"} + text: {"path": "dump/raw/small_w_whamr_sp/text", "type": "text"} + preprocess: ) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,255 (abs_task:1615) INFO: [train] Batch sampler: NumElementsBatchSampler(N-batch=2879, batch_bins=10000000, sort_in_batch=descending, sort_batch=descending) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,256 (abs_task:1616) INFO: [train] mini-batch sizes summary: N-batch=2879, mean=42.8, min=10, max=128 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,329 (asr:461) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4') +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,367 (abs_task:1614) INFO: [valid] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/cv_mix_clean_reverb_max_16k/wav.scp", "type": "kaldi_ark"} + text: {"path": "dump/raw/cv_mix_clean_reverb_max_16k/text", "type": "text"} + preprocess: ) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,367 (abs_task:1615) INFO: [valid] Batch sampler: NumElementsBatchSampler(N-batch=88, batch_bins=10000000, sort_in_batch=descending, sort_batch=descending) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,367 (abs_task:1616) INFO: [valid] mini-batch sizes summary: N-batch=88, mean=56.8, min=5, max=102 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,380 (asr:461) INFO: Optional Data Names: ('text_spk2', 'text_spk3', 'text_spk4') +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,407 (abs_task:1614) INFO: [plot_att] dataset: +ESPnetDataset( + speech: {"path": "dump/raw/cv_mix_clean_reverb_max_16k/wav.scp", "type": "kaldi_ark"} + text: {"path": "dump/raw/cv_mix_clean_reverb_max_16k/text", "type": "text"} + preprocess: ) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,408 (abs_task:1615) INFO: [plot_att] Batch sampler: UnsortedBatchSampler(N-batch=5000, batch_size=1, key_file=exp/asr_stats_raw_en_char/valid/speech_shape, +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:24,408 (abs_task:1616) INFO: [plot_att] mini-batch sizes summary: N-batch=3, mean=1.0, min=1, max=1 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] NCCL INFO Bootstrap : Using eth0:10.177.63.83<0> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_0 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_1 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] NCCL INFO NET/IB : No device found. +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] NCCL INFO NET/Socket : Using [0]eth0:10.177.63.83<0> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] NCCL INFO Using network Socket +NCCL version 2.10.3+cuda11.6 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146344 [2] NCCL INFO Bootstrap : Using eth0:10.177.63.83<0> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146345 [3] NCCL INFO Bootstrap : Using eth0:10.177.63.83<0> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146344 [2] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146345 [3] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146344 [2] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146344 [2] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_0 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146344 [2] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146344 [2] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_1 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146344 [2] NCCL INFO NET/IB : No device found. +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146344 [2] NCCL INFO NET/Socket : Using [0]eth0:10.177.63.83<0> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146344 [2] NCCL INFO Using network Socket + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146345 [3] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146345 [3] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_0 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146345 [3] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146345 [3] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_1 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146345 [3] NCCL INFO NET/IB : No device found. +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146345 [3] NCCL INFO NET/Socket : Using [0]eth0:10.177.63.83<0> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146345 [3] NCCL INFO Using network Socket +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146343 [1] NCCL INFO Bootstrap : Using eth0:10.177.63.83<0> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146343 [1] NCCL INFO NET/Plugin : No plugin found (libnccl-net.so), using internal implementation + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146343 [1] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146343 [1] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_0 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146343 [1] misc/ibvwrap.cc:212 NCCL WARN Call to ibv_open_device failed + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146343 [1] transport/net_ib.cc:149 NCCL WARN NET/IB : Unable to open device mlx5_1 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146343 [1] NCCL INFO NET/IB : No device found. +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146343 [1] NCCL INFO NET/Socket : Using [0]eth0:10.177.63.83<0> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146343 [1] NCCL INFO Using network Socket +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO Channel 00/02 : 0 1 2 3 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO Trees [0] 3/-1/-1->2->1 [1] 3/-1/-1->2->1 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO Channel 01/02 : 0 1 2 3 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO Trees [0] 2/-1/-1->1->0 [1] 2/-1/-1->1->0 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO Trees [0] -1/-1/-1->3->2 [1] -1/-1/-1->3->2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO Setting affinity for GPU 6 to ff,ffc0000f,fffc0000 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO Trees [0] 1/-1/-1->0->-1 [1] 1/-1/-1->0->-1 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO Setting affinity for GPU 5 to ff,ffc0000f,fffc0000 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO Setting affinity for GPU 7 to ff,ffc0000f,fffc0000 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO Setting affinity for GPU 4 to ff,ffc0000f,fffc0000 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO Channel 00 : 0[b1000] -> 1[b2000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO Channel 00 : 2[b4000] -> 3[b5000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO Channel 00 : 1[b2000] -> 2[b4000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO Channel 00 : 3[b5000] -> 0[b1000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO Channel 01 : 0[b1000] -> 1[b2000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO Channel 01 : 2[b4000] -> 3[b5000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO Channel 01 : 1[b2000] -> 2[b4000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO Channel 01 : 3[b5000] -> 0[b1000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO Connected all rings +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO Channel 00 : 3[b5000] -> 2[b4000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO Channel 01 : 3[b5000] -> 2[b4000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO Connected all rings +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO Connected all rings +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO Connected all rings +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO Channel 00 : 2[b4000] -> 1[b2000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO Channel 00 : 1[b2000] -> 0[b1000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO Channel 01 : 2[b4000] -> 1[b2000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO Channel 01 : 1[b2000] -> 0[b1000] via P2P/IPC +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO Connected all trees +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 8/8/512 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO Connected all trees +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 8/8/512 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO Connected all trees +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO Connected all trees +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 8/8/512 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO threadThresholds 8/8/64 | 32/8/64 | 8/8/512 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO 2 coll channels, 2 p2p channels, 2 p2p channels per peer +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146529 [2] NCCL INFO comm 0x7f6580002f70 rank 2 nranks 4 cudaDev 2 busId b4000 - Init COMPLETE +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146525 [0] NCCL INFO comm 0x7f62bc002f70 rank 0 nranks 4 cudaDev 0 busId b1000 - Init COMPLETE +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146530 [3] NCCL INFO comm 0x7f37a4002f70 rank 3 nranks 4 cudaDev 3 busId b5000 - Init COMPLETE +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146534 [1] NCCL INFO comm 0x7f8fc0002f70 rank 1 nranks 4 cudaDev 1 busId b2000 - Init COMPLETE +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146342 [0] NCCL INFO Launch mode Parallel +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:09:25,588 (trainer:284) INFO: 1/60epoch started +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:13:09,283 (distributed:995) INFO: Reducer buckets have been rebuilt in this iteration. +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:14:15,148 (trainer:732) INFO: 1epoch:train:1-143batch: iter_time=0.003, forward_time=0.156, loss_att=660.818, acc=0.421, loss=660.818, backward_time=0.192, grad_norm=622.901, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=1.900e-06, train_time=8.236 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:15:21,144 (trainer:732) INFO: 1epoch:train:144-286batch: iter_time=2.484e-04, forward_time=0.127, loss_att=613.054, acc=0.436, loss=613.054, backward_time=0.191, grad_norm=487.797, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.450e-06, train_time=1.843 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:16:32,646 (trainer:732) INFO: 1epoch:train:287-429batch: iter_time=2.563e-04, forward_time=0.129, loss_att=567.550, acc=0.453, loss=567.550, backward_time=0.193, grad_norm=328.722, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=9.050e-06, train_time=1.996 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:18:11,271 (trainer:732) INFO: 1epoch:train:430-572batch: iter_time=2.515e-04, forward_time=0.127, loss_att=472.169, acc=0.474, loss=472.169, backward_time=0.191, grad_norm=164.314, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.265e-05, train_time=2.753 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:19:53,297 (trainer:732) INFO: 1epoch:train:573-715batch: iter_time=2.593e-04, forward_time=0.127, loss_att=463.406, acc=0.499, loss=463.406, backward_time=0.191, grad_norm=112.020, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.620e-05, train_time=2.864 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:21:33,791 (trainer:732) INFO: 1epoch:train:716-858batch: iter_time=2.687e-04, forward_time=0.127, loss_att=412.979, acc=0.516, loss=412.979, backward_time=0.191, grad_norm=106.209, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=1.975e-05, train_time=2.807 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:22:57,063 (trainer:732) INFO: 1epoch:train:859-1001batch: iter_time=2.672e-04, forward_time=0.126, loss_att=394.956, acc=0.530, loss=394.956, backward_time=0.190, grad_norm=117.806, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.335e-05, train_time=2.338 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:24:03,198 (trainer:732) INFO: 1epoch:train:1002-1144batch: iter_time=2.569e-04, forward_time=0.128, loss_att=396.924, acc=0.540, loss=396.924, backward_time=0.192, grad_norm=110.145, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.695e-05, train_time=1.845 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:25:08,640 (trainer:732) INFO: 1epoch:train:1145-1287batch: iter_time=2.453e-04, forward_time=0.127, loss_att=373.740, acc=0.548, loss=373.740, backward_time=0.191, grad_norm=83.148, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.050e-05, train_time=1.833 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:26:14,071 (trainer:732) INFO: 1epoch:train:1288-1430batch: iter_time=2.625e-04, forward_time=0.127, loss_att=369.671, acc=0.554, loss=369.671, backward_time=0.190, grad_norm=80.278, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.405e-05, train_time=1.827 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:27:20,397 (trainer:732) INFO: 1epoch:train:1431-1573batch: iter_time=2.462e-04, forward_time=0.128, loss_att=366.033, acc=0.561, loss=366.033, backward_time=0.192, grad_norm=115.579, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.765e-05, train_time=1.859 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:28:27,277 (trainer:732) INFO: 1epoch:train:1574-1716batch: iter_time=2.545e-04, forward_time=0.130, loss_att=381.327, acc=0.570, loss=381.327, backward_time=0.195, grad_norm=102.321, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=4.125e-05, train_time=1.866 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:29:33,573 (trainer:732) INFO: 1epoch:train:1717-1859batch: iter_time=2.621e-04, forward_time=0.129, loss_att=369.785, acc=0.577, loss=369.785, backward_time=0.194, grad_norm=90.270, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=4.480e-05, train_time=1.853 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:30:39,127 (trainer:732) INFO: 1epoch:train:1860-2002batch: iter_time=2.527e-04, forward_time=0.127, loss_att=349.762, acc=0.578, loss=349.762, backward_time=0.190, grad_norm=80.329, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.835e-05, train_time=1.837 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:31:45,452 (trainer:732) INFO: 1epoch:train:2003-2145batch: iter_time=2.871e-04, forward_time=0.129, loss_att=351.090, acc=0.583, loss=351.090, backward_time=0.192, grad_norm=86.665, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=5.195e-05, train_time=1.856 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:33:13,679 (trainer:732) INFO: 1epoch:train:2146-2288batch: iter_time=3.092e-04, forward_time=0.129, loss_att=340.559, acc=0.587, loss=340.559, backward_time=0.191, grad_norm=80.211, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=5.555e-05, train_time=2.460 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:34:50,918 (trainer:732) INFO: 1epoch:train:2289-2431batch: iter_time=3.143e-04, forward_time=0.132, loss_att=368.878, acc=0.593, loss=368.878, backward_time=0.196, grad_norm=79.204, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=5.910e-05, train_time=2.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:36:29,295 (trainer:732) INFO: 1epoch:train:2432-2574batch: iter_time=3.095e-04, forward_time=0.131, loss_att=354.647, acc=0.598, loss=354.647, backward_time=0.194, grad_norm=75.848, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=6.265e-05, train_time=2.753 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:38:06,739 (trainer:732) INFO: 1epoch:train:2575-2717batch: iter_time=3.171e-04, forward_time=0.130, loss_att=333.119, acc=0.599, loss=333.119, backward_time=0.193, grad_norm=78.836, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=6.625e-05, train_time=2.729 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:39:43,741 (trainer:732) INFO: 1epoch:train:2718-2860batch: iter_time=3.061e-04, forward_time=0.129, loss_att=328.937, acc=0.599, loss=328.937, backward_time=0.191, grad_norm=80.499, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=6.985e-05, train_time=2.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:48:29,840 (trainer:338) INFO: 1epoch results: [train] iter_time=4.043e-04, forward_time=0.130, loss_att=412.612, acc=0.541, loss=412.612, backward_time=0.192, grad_norm=153.433, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.610e-05, train_time=2.544, time=30 minutes and 34.02 seconds, total_count=2879, gpu_max_cached_mem_GB=23.359, [valid] loss_att=197.048, acc=0.622, cer=0.446, wer=0.814, loss=197.048, time=3 minutes and 55.24 seconds, total_count=88, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 34.97 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:48:34,448 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:48:34,449 (trainer:272) INFO: 2/60epoch started. Estimated time to finish: 1 day, 14 hours and 29 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:52:04,938 (trainer:732) INFO: 2epoch:train:1-143batch: iter_time=0.003, forward_time=0.129, loss_att=321.751, acc=0.607, loss=321.751, backward_time=0.189, grad_norm=77.384, clip=100.000, loss_scale=1.000, optim_step_time=0.067, optim0_lr0=7.380e-05, train_time=5.979 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:53:10,955 (trainer:732) INFO: 2epoch:train:144-286batch: iter_time=2.655e-04, forward_time=0.128, loss_att=324.809, acc=0.612, loss=324.809, backward_time=0.191, grad_norm=77.004, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.735e-05, train_time=1.843 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:54:17,214 (trainer:732) INFO: 2epoch:train:287-429batch: iter_time=2.548e-04, forward_time=0.129, loss_att=327.951, acc=0.617, loss=327.951, backward_time=0.191, grad_norm=72.872, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=8.095e-05, train_time=1.853 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:55:23,429 (trainer:732) INFO: 2epoch:train:430-572batch: iter_time=2.927e-04, forward_time=0.129, loss_att=323.467, acc=0.618, loss=323.467, backward_time=0.191, grad_norm=72.741, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.455e-05, train_time=1.849 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:56:28,710 (trainer:732) INFO: 2epoch:train:573-715batch: iter_time=2.702e-04, forward_time=0.127, loss_att=317.101, acc=0.618, loss=317.101, backward_time=0.188, grad_norm=73.599, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.810e-05, train_time=1.830 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:57:35,076 (trainer:732) INFO: 2epoch:train:716-858batch: iter_time=2.840e-04, forward_time=0.128, loss_att=312.526, acc=0.623, loss=312.526, backward_time=0.190, grad_norm=73.338, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=9.165e-05, train_time=1.857 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:58:41,003 (trainer:732) INFO: 2epoch:train:859-1001batch: iter_time=2.746e-04, forward_time=0.129, loss_att=315.574, acc=0.628, loss=315.574, backward_time=0.191, grad_norm=71.877, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=9.525e-05, train_time=1.841 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 17:59:46,920 (trainer:732) INFO: 2epoch:train:1002-1144batch: iter_time=2.635e-04, forward_time=0.128, loss_att=316.882, acc=0.629, loss=316.882, backward_time=0.191, grad_norm=72.282, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=9.885e-05, train_time=1.841 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:00:53,090 (trainer:732) INFO: 2epoch:train:1145-1287batch: iter_time=2.736e-04, forward_time=0.129, loss_att=313.595, acc=0.634, loss=313.595, backward_time=0.192, grad_norm=89.645, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.024e-04, train_time=1.856 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:01:59,078 (trainer:732) INFO: 2epoch:train:1288-1430batch: iter_time=2.584e-04, forward_time=0.128, loss_att=297.565, acc=0.634, loss=297.565, backward_time=0.191, grad_norm=66.847, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.059e-04, train_time=1.843 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:03:05,392 (trainer:732) INFO: 2epoch:train:1431-1573batch: iter_time=2.718e-04, forward_time=0.129, loss_att=322.738, acc=0.638, loss=322.738, backward_time=0.192, grad_norm=75.149, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.095e-04, train_time=1.855 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:04:11,831 (trainer:732) INFO: 2epoch:train:1574-1716batch: iter_time=2.739e-04, forward_time=0.130, loss_att=313.433, acc=0.640, loss=313.433, backward_time=0.193, grad_norm=80.285, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.131e-04, train_time=1.855 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:05:17,321 (trainer:732) INFO: 2epoch:train:1717-1859batch: iter_time=2.816e-04, forward_time=0.128, loss_att=293.604, acc=0.642, loss=293.604, backward_time=0.190, grad_norm=70.754, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.167e-04, train_time=1.836 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:06:23,502 (trainer:732) INFO: 2epoch:train:1860-2002batch: iter_time=2.719e-04, forward_time=0.129, loss_att=304.124, acc=0.645, loss=304.124, backward_time=0.192, grad_norm=69.073, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.202e-04, train_time=1.848 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:07:29,751 (trainer:732) INFO: 2epoch:train:2003-2145batch: iter_time=2.704e-04, forward_time=0.129, loss_att=302.364, acc=0.647, loss=302.364, backward_time=0.192, grad_norm=70.628, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.239e-04, train_time=1.853 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:08:35,758 (trainer:732) INFO: 2epoch:train:2146-2288batch: iter_time=2.811e-04, forward_time=0.129, loss_att=302.167, acc=0.650, loss=302.167, backward_time=0.192, grad_norm=71.432, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=1.274e-04, train_time=1.844 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:09:42,404 (trainer:732) INFO: 2epoch:train:2289-2431batch: iter_time=2.802e-04, forward_time=0.129, loss_att=319.213, acc=0.655, loss=319.213, backward_time=0.193, grad_norm=78.693, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=1.310e-04, train_time=1.866 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:10:48,476 (trainer:732) INFO: 2epoch:train:2432-2574batch: iter_time=2.725e-04, forward_time=0.128, loss_att=288.659, acc=0.655, loss=288.659, backward_time=0.190, grad_norm=66.394, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.345e-04, train_time=1.849 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:11:55,311 (trainer:732) INFO: 2epoch:train:2575-2717batch: iter_time=2.726e-04, forward_time=0.128, loss_att=299.569, acc=0.657, loss=299.569, backward_time=0.190, grad_norm=70.665, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=1.381e-04, train_time=1.870 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:13:01,080 (trainer:732) INFO: 2epoch:train:2718-2860batch: iter_time=2.494e-04, forward_time=0.127, loss_att=281.792, acc=0.656, loss=281.792, backward_time=0.189, grad_norm=70.780, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=1.418e-04, train_time=1.836 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:21:18,794 (trainer:338) INFO: 2epoch results: [train] iter_time=4.045e-04, forward_time=0.128, loss_att=309.006, acc=0.635, loss=309.006, backward_time=0.191, grad_norm=73.469, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.080e-04, train_time=2.049, time=24 minutes and 36.85 seconds, total_count=5758, gpu_max_cached_mem_GB=23.359, [valid] loss_att=160.334, acc=0.691, cer=0.365, wer=0.728, loss=160.334, time=3 minutes and 42.28 seconds, total_count=176, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.22 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:21:22,206 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:21:22,207 (trainer:272) INFO: 3/60epoch started. Estimated time to finish: 1 day, 10 hours and 46 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:24:53,248 (trainer:732) INFO: 3epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=292.343, acc=0.663, loss=292.343, backward_time=0.192, grad_norm=73.587, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=1.457e-04, train_time=5.995 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:26:00,335 (trainer:732) INFO: 3epoch:train:144-286batch: iter_time=2.890e-04, forward_time=0.130, loss_att=304.829, acc=0.667, loss=304.829, backward_time=0.195, grad_norm=74.597, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.492e-04, train_time=1.872 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:27:05,414 (trainer:732) INFO: 3epoch:train:287-429batch: iter_time=2.762e-04, forward_time=0.126, loss_att=270.138, acc=0.667, loss=270.138, backward_time=0.188, grad_norm=65.879, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.529e-04, train_time=1.822 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:28:11,546 (trainer:732) INFO: 3epoch:train:430-572batch: iter_time=2.847e-04, forward_time=0.128, loss_att=285.271, acc=0.670, loss=285.271, backward_time=0.191, grad_norm=68.041, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=1.564e-04, train_time=1.847 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:29:16,570 (trainer:732) INFO: 3epoch:train:573-715batch: iter_time=2.527e-04, forward_time=0.126, loss_att=266.762, acc=0.672, loss=266.762, backward_time=0.188, grad_norm=74.842, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=1.600e-04, train_time=1.821 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:30:21,914 (trainer:732) INFO: 3epoch:train:716-858batch: iter_time=2.650e-04, forward_time=0.127, loss_att=275.619, acc=0.673, loss=275.619, backward_time=0.189, grad_norm=71.582, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=1.636e-04, train_time=1.828 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:31:27,907 (trainer:732) INFO: 3epoch:train:859-1001batch: iter_time=2.566e-04, forward_time=0.128, loss_att=279.412, acc=0.675, loss=279.412, backward_time=0.192, grad_norm=77.380, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=1.671e-04, train_time=1.844 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:32:33,259 (trainer:732) INFO: 3epoch:train:1002-1144batch: iter_time=2.612e-04, forward_time=0.127, loss_att=258.931, acc=0.678, loss=258.931, backward_time=0.190, grad_norm=73.995, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=1.708e-04, train_time=1.825 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:33:38,450 (trainer:732) INFO: 3epoch:train:1145-1287batch: iter_time=2.622e-04, forward_time=0.127, loss_att=261.296, acc=0.680, loss=261.296, backward_time=0.188, grad_norm=66.445, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=1.743e-04, train_time=1.827 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:34:44,087 (trainer:732) INFO: 3epoch:train:1288-1430batch: iter_time=2.712e-04, forward_time=0.128, loss_att=271.049, acc=0.683, loss=271.049, backward_time=0.191, grad_norm=78.378, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=1.778e-04, train_time=1.836 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:35:49,413 (trainer:732) INFO: 3epoch:train:1431-1573batch: iter_time=2.544e-04, forward_time=0.127, loss_att=268.753, acc=0.682, loss=268.753, backward_time=0.189, grad_norm=70.415, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=1.814e-04, train_time=1.826 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:36:55,618 (trainer:732) INFO: 3epoch:train:1574-1716batch: iter_time=2.668e-04, forward_time=0.129, loss_att=282.862, acc=0.684, loss=282.862, backward_time=0.192, grad_norm=79.160, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=1.850e-04, train_time=1.848 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:38:01,491 (trainer:732) INFO: 3epoch:train:1717-1859batch: iter_time=2.734e-04, forward_time=0.128, loss_att=262.133, acc=0.686, loss=262.133, backward_time=0.190, grad_norm=77.054, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=1.886e-04, train_time=1.842 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:39:06,973 (trainer:732) INFO: 3epoch:train:1860-2002batch: iter_time=2.625e-04, forward_time=0.126, loss_att=256.266, acc=0.690, loss=256.266, backward_time=0.189, grad_norm=74.556, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=1.921e-04, train_time=1.833 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:40:12,753 (trainer:732) INFO: 3epoch:train:2003-2145batch: iter_time=2.780e-04, forward_time=0.128, loss_att=268.281, acc=0.689, loss=268.281, backward_time=0.192, grad_norm=83.920, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=1.957e-04, train_time=1.839 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:41:18,658 (trainer:732) INFO: 3epoch:train:2146-2288batch: iter_time=2.783e-04, forward_time=0.128, loss_att=269.497, acc=0.691, loss=269.497, backward_time=0.192, grad_norm=78.191, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=1.993e-04, train_time=1.843 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:42:24,906 (trainer:732) INFO: 3epoch:train:2289-2431batch: iter_time=2.535e-04, forward_time=0.130, loss_att=277.890, acc=0.692, loss=277.890, backward_time=0.193, grad_norm=81.714, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=2.029e-04, train_time=1.858 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:43:30,598 (trainer:732) INFO: 3epoch:train:2432-2574batch: iter_time=2.854e-04, forward_time=0.129, loss_att=257.624, acc=0.695, loss=257.624, backward_time=0.191, grad_norm=70.893, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.065e-04, train_time=1.836 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:44:36,761 (trainer:732) INFO: 3epoch:train:2575-2717batch: iter_time=2.798e-04, forward_time=0.128, loss_att=267.072, acc=0.697, loss=267.072, backward_time=0.192, grad_norm=84.597, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.100e-04, train_time=1.851 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:45:42,189 (trainer:732) INFO: 3epoch:train:2718-2860batch: iter_time=2.507e-04, forward_time=0.128, loss_att=255.773, acc=0.696, loss=255.773, backward_time=0.191, grad_norm=76.792, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.137e-04, train_time=1.826 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:53:54,903 (trainer:338) INFO: 3epoch results: [train] iter_time=4.098e-04, forward_time=0.128, loss_att=271.039, acc=0.681, loss=271.039, backward_time=0.191, grad_norm=75.060, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=1.799e-04, train_time=2.040, time=24 minutes and 30.37 seconds, total_count=8637, gpu_max_cached_mem_GB=23.359, [valid] loss_att=136.914, acc=0.736, cer=0.314, wer=0.670, loss=136.914, time=3 minutes and 41.96 seconds, total_count=264, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 20.36 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:53:58,480 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:53:58,501 (trainer:272) INFO: 4/60epoch started. Estimated time to finish: 1 day, 9 hours and 6 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:57:22,498 (trainer:732) INFO: 4epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=256.548, acc=0.704, loss=256.548, backward_time=0.194, grad_norm=75.214, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.176e-04, train_time=5.796 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:58:27,824 (trainer:732) INFO: 4epoch:train:144-286batch: iter_time=2.697e-04, forward_time=0.126, loss_att=246.997, acc=0.705, loss=246.997, backward_time=0.189, grad_norm=73.729, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.211e-04, train_time=1.823 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 18:59:33,576 (trainer:732) INFO: 4epoch:train:287-429batch: iter_time=2.654e-04, forward_time=0.128, loss_att=248.860, acc=0.706, loss=248.860, backward_time=0.190, grad_norm=72.140, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.248e-04, train_time=1.838 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:00:39,368 (trainer:732) INFO: 4epoch:train:430-572batch: iter_time=2.788e-04, forward_time=0.127, loss_att=253.453, acc=0.706, loss=253.453, backward_time=0.190, grad_norm=75.480, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.283e-04, train_time=1.838 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:01:44,825 (trainer:732) INFO: 4epoch:train:573-715batch: iter_time=2.659e-04, forward_time=0.127, loss_att=244.412, acc=0.708, loss=244.412, backward_time=0.190, grad_norm=76.559, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=2.319e-04, train_time=1.837 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:02:50,626 (trainer:732) INFO: 4epoch:train:716-858batch: iter_time=2.702e-04, forward_time=0.128, loss_att=257.030, acc=0.709, loss=257.030, backward_time=0.191, grad_norm=73.354, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.355e-04, train_time=1.835 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:03:55,785 (trainer:732) INFO: 4epoch:train:859-1001batch: iter_time=2.792e-04, forward_time=0.127, loss_att=233.429, acc=0.710, loss=233.429, backward_time=0.189, grad_norm=81.016, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.391e-04, train_time=1.823 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:05:01,373 (trainer:732) INFO: 4epoch:train:1002-1144batch: iter_time=2.611e-04, forward_time=0.127, loss_att=237.710, acc=0.711, loss=237.710, backward_time=0.189, grad_norm=77.885, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.427e-04, train_time=1.833 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:06:07,873 (trainer:732) INFO: 4epoch:train:1145-1287batch: iter_time=2.716e-04, forward_time=0.130, loss_att=266.048, acc=0.711, loss=266.048, backward_time=0.194, grad_norm=81.295, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.462e-04, train_time=1.864 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:07:13,114 (trainer:732) INFO: 4epoch:train:1288-1430batch: iter_time=2.800e-04, forward_time=0.127, loss_att=240.186, acc=0.715, loss=240.186, backward_time=0.190, grad_norm=79.360, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.497e-04, train_time=1.824 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:08:19,170 (trainer:732) INFO: 4epoch:train:1431-1573batch: iter_time=2.728e-04, forward_time=0.130, loss_att=246.487, acc=0.717, loss=246.487, backward_time=0.192, grad_norm=75.213, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.534e-04, train_time=1.845 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:09:24,216 (trainer:732) INFO: 4epoch:train:1574-1716batch: iter_time=2.779e-04, forward_time=0.127, loss_att=230.598, acc=0.719, loss=230.598, backward_time=0.188, grad_norm=68.705, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.569e-04, train_time=1.818 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:10:30,802 (trainer:732) INFO: 4epoch:train:1717-1859batch: iter_time=2.796e-04, forward_time=0.131, loss_att=264.628, acc=0.716, loss=264.628, backward_time=0.195, grad_norm=83.326, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.605e-04, train_time=1.866 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:11:36,433 (trainer:732) INFO: 4epoch:train:1860-2002batch: iter_time=2.659e-04, forward_time=0.127, loss_att=244.655, acc=0.721, loss=244.655, backward_time=0.190, grad_norm=79.498, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.640e-04, train_time=1.834 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:12:42,152 (trainer:732) INFO: 4epoch:train:2003-2145batch: iter_time=2.764e-04, forward_time=0.128, loss_att=239.532, acc=0.722, loss=239.532, backward_time=0.191, grad_norm=86.432, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.676e-04, train_time=1.838 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:13:47,221 (trainer:732) INFO: 4epoch:train:2146-2288batch: iter_time=2.775e-04, forward_time=0.127, loss_att=224.895, acc=0.723, loss=224.895, backward_time=0.188, grad_norm=70.630, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.713e-04, train_time=1.818 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:14:53,364 (trainer:732) INFO: 4epoch:train:2289-2431batch: iter_time=2.751e-04, forward_time=0.130, loss_att=247.228, acc=0.724, loss=247.228, backward_time=0.193, grad_norm=86.879, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=2.748e-04, train_time=1.852 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:15:59,507 (trainer:732) INFO: 4epoch:train:2432-2574batch: iter_time=2.750e-04, forward_time=0.128, loss_att=236.955, acc=0.725, loss=236.955, backward_time=0.191, grad_norm=86.086, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.784e-04, train_time=1.850 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:17:04,841 (trainer:732) INFO: 4epoch:train:2575-2717batch: iter_time=2.817e-04, forward_time=0.127, loss_att=226.157, acc=0.727, loss=226.157, backward_time=0.189, grad_norm=76.758, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=2.819e-04, train_time=1.826 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:18:11,033 (trainer:732) INFO: 4epoch:train:2718-2860batch: iter_time=2.467e-04, forward_time=0.127, loss_att=237.363, acc=0.730, loss=237.363, backward_time=0.190, grad_norm=80.878, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.855e-04, train_time=1.849 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:26:29,374 (trainer:338) INFO: 4epoch results: [train] iter_time=3.975e-04, forward_time=0.128, loss_att=243.851, acc=0.715, loss=243.851, backward_time=0.191, grad_norm=78.020, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.518e-04, train_time=2.030, time=24 minutes and 23.05 seconds, total_count=11516, gpu_max_cached_mem_GB=23.359, [valid] loss_att=118.993, acc=0.773, cer=0.271, wer=0.621, loss=118.993, time=3 minutes and 44.4 seconds, total_count=352, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 23.41 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:26:33,095 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:26:33,104 (trainer:272) INFO: 5/60epoch started. Estimated time to finish: 1 day, 7 hours and 59 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:29:55,947 (trainer:732) INFO: 5epoch:train:1-143batch: iter_time=0.003, forward_time=0.127, loss_att=218.375, acc=0.732, loss=218.375, backward_time=0.188, grad_norm=75.146, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=2.895e-04, train_time=5.757 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:31:01,326 (trainer:732) INFO: 5epoch:train:144-286batch: iter_time=2.627e-04, forward_time=0.127, loss_att=223.827, acc=0.735, loss=223.827, backward_time=0.190, grad_norm=75.409, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=2.930e-04, train_time=1.831 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:32:06,889 (trainer:732) INFO: 5epoch:train:287-429batch: iter_time=2.570e-04, forward_time=0.127, loss_att=228.390, acc=0.735, loss=228.390, backward_time=0.190, grad_norm=82.289, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=2.966e-04, train_time=1.832 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:33:13,072 (trainer:732) INFO: 5epoch:train:430-572batch: iter_time=2.645e-04, forward_time=0.129, loss_att=239.080, acc=0.734, loss=239.080, backward_time=0.193, grad_norm=80.551, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=3.002e-04, train_time=1.848 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:34:18,537 (trainer:732) INFO: 5epoch:train:573-715batch: iter_time=2.755e-04, forward_time=0.128, loss_att=227.286, acc=0.739, loss=227.286, backward_time=0.191, grad_norm=86.408, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=3.038e-04, train_time=1.836 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:35:24,527 (trainer:732) INFO: 5epoch:train:716-858batch: iter_time=2.566e-04, forward_time=0.128, loss_att=223.236, acc=0.739, loss=223.236, backward_time=0.191, grad_norm=88.077, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.074e-04, train_time=1.845 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:36:29,673 (trainer:732) INFO: 5epoch:train:859-1001batch: iter_time=2.838e-04, forward_time=0.127, loss_att=215.945, acc=0.739, loss=215.945, backward_time=0.189, grad_norm=75.308, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.109e-04, train_time=1.821 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:37:35,579 (trainer:732) INFO: 5epoch:train:1002-1144batch: iter_time=2.734e-04, forward_time=0.129, loss_att=220.195, acc=0.742, loss=220.195, backward_time=0.192, grad_norm=86.366, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.146e-04, train_time=1.840 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:38:41,162 (trainer:732) INFO: 5epoch:train:1145-1287batch: iter_time=2.624e-04, forward_time=0.128, loss_att=223.375, acc=0.742, loss=223.375, backward_time=0.190, grad_norm=81.278, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.181e-04, train_time=1.839 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:39:46,662 (trainer:732) INFO: 5epoch:train:1288-1430batch: iter_time=2.805e-04, forward_time=0.127, loss_att=217.536, acc=0.743, loss=217.536, backward_time=0.190, grad_norm=71.497, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.216e-04, train_time=1.831 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:40:51,789 (trainer:732) INFO: 5epoch:train:1431-1573batch: iter_time=2.723e-04, forward_time=0.127, loss_att=220.727, acc=0.741, loss=220.727, backward_time=0.189, grad_norm=76.244, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=3.252e-04, train_time=1.820 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:41:58,637 (trainer:732) INFO: 5epoch:train:1574-1716batch: iter_time=2.792e-04, forward_time=0.130, loss_att=230.786, acc=0.745, loss=230.786, backward_time=0.193, grad_norm=89.221, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.289e-04, train_time=1.867 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:43:05,559 (trainer:732) INFO: 5epoch:train:1717-1859batch: iter_time=2.907e-04, forward_time=0.130, loss_att=235.768, acc=0.743, loss=235.768, backward_time=0.193, grad_norm=85.662, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.324e-04, train_time=1.878 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:44:39,361 (trainer:732) INFO: 5epoch:train:1860-2002batch: iter_time=2.780e-04, forward_time=0.129, loss_att=223.518, acc=0.746, loss=223.518, backward_time=0.191, grad_norm=86.477, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.360e-04, train_time=2.595 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:46:51,538 (trainer:732) INFO: 5epoch:train:2003-2145batch: iter_time=2.781e-04, forward_time=0.128, loss_att=213.157, acc=0.749, loss=213.157, backward_time=0.190, grad_norm=78.989, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.395e-04, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:49:02,531 (trainer:732) INFO: 5epoch:train:2146-2288batch: iter_time=2.701e-04, forward_time=0.127, loss_att=210.667, acc=0.749, loss=210.667, backward_time=0.189, grad_norm=81.559, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.432e-04, train_time=3.656 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:51:14,035 (trainer:732) INFO: 5epoch:train:2289-2431batch: iter_time=2.658e-04, forward_time=0.128, loss_att=221.163, acc=0.747, loss=221.163, backward_time=0.191, grad_norm=82.835, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=3.467e-04, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:53:21,320 (trainer:732) INFO: 5epoch:train:2432-2574batch: iter_time=2.751e-04, forward_time=0.128, loss_att=224.264, acc=0.748, loss=224.264, backward_time=0.190, grad_norm=97.348, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=3.502e-04, train_time=3.556 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:55:34,083 (trainer:732) INFO: 5epoch:train:2575-2717batch: iter_time=2.790e-04, forward_time=0.128, loss_att=211.792, acc=0.753, loss=211.792, backward_time=0.191, grad_norm=81.149, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.538e-04, train_time=3.713 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 19:57:48,566 (trainer:732) INFO: 5epoch:train:2718-2860batch: iter_time=2.548e-04, forward_time=0.128, loss_att=222.025, acc=0.754, loss=222.025, backward_time=0.192, grad_norm=92.922, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.574e-04, train_time=3.752 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:06:43,734 (trainer:338) INFO: 5epoch results: [train] iter_time=3.858e-04, forward_time=0.128, loss_att=222.269, acc=0.743, loss=222.269, backward_time=0.191, grad_norm=82.699, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.237e-04, train_time=2.629, time=31 minutes and 34.89 seconds, total_count=14395, gpu_max_cached_mem_GB=23.359, [valid] loss_att=105.772, acc=0.802, cer=0.241, wer=0.568, loss=105.772, time=4 minutes and 10.9 seconds, total_count=440, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 24.84 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:06:47,328 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:06:47,331 (trainer:272) INFO: 6/60epoch started. Estimated time to finish: 1 day, 8 hours and 30 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:11:17,703 (trainer:732) INFO: 6epoch:train:1-143batch: iter_time=0.003, forward_time=0.126, loss_att=199.976, acc=0.757, loss=199.976, backward_time=0.186, grad_norm=74.569, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.614e-04, train_time=7.655 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:13:31,252 (trainer:732) INFO: 6epoch:train:144-286batch: iter_time=2.673e-04, forward_time=0.128, loss_att=221.305, acc=0.756, loss=221.305, backward_time=0.191, grad_norm=87.761, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=3.649e-04, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:15:43,792 (trainer:732) INFO: 6epoch:train:287-429batch: iter_time=2.752e-04, forward_time=0.127, loss_att=202.071, acc=0.762, loss=202.071, backward_time=0.190, grad_norm=90.033, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.685e-04, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:17:50,555 (trainer:732) INFO: 6epoch:train:430-572batch: iter_time=2.788e-04, forward_time=0.127, loss_att=202.397, acc=0.763, loss=202.397, backward_time=0.189, grad_norm=87.360, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=3.722e-04, train_time=3.537 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:20:02,843 (trainer:732) INFO: 6epoch:train:573-715batch: iter_time=2.664e-04, forward_time=0.129, loss_att=206.333, acc=0.763, loss=206.333, backward_time=0.192, grad_norm=81.821, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.757e-04, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:22:14,652 (trainer:732) INFO: 6epoch:train:716-858batch: iter_time=2.736e-04, forward_time=0.126, loss_att=196.571, acc=0.762, loss=196.571, backward_time=0.188, grad_norm=78.118, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=3.793e-04, train_time=3.685 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:24:21,358 (trainer:732) INFO: 6epoch:train:859-1001batch: iter_time=2.824e-04, forward_time=0.127, loss_att=204.150, acc=0.764, loss=204.150, backward_time=0.190, grad_norm=83.289, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=3.828e-04, train_time=3.548 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:26:33,489 (trainer:732) INFO: 6epoch:train:1002-1144batch: iter_time=2.544e-04, forward_time=0.127, loss_att=216.302, acc=0.759, loss=216.302, backward_time=0.190, grad_norm=86.841, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=3.864e-04, train_time=3.686 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:28:46,874 (trainer:732) INFO: 6epoch:train:1145-1287batch: iter_time=2.638e-04, forward_time=0.129, loss_att=218.381, acc=0.763, loss=218.381, backward_time=0.193, grad_norm=87.817, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=3.900e-04, train_time=3.738 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:30:58,764 (trainer:732) INFO: 6epoch:train:1288-1430batch: iter_time=2.803e-04, forward_time=0.127, loss_att=195.071, acc=0.768, loss=195.071, backward_time=0.188, grad_norm=82.627, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.935e-04, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:33:07,091 (trainer:732) INFO: 6epoch:train:1431-1573batch: iter_time=2.617e-04, forward_time=0.130, loss_att=214.289, acc=0.764, loss=214.289, backward_time=0.193, grad_norm=97.995, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=3.971e-04, train_time=3.587 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:35:20,717 (trainer:732) INFO: 6epoch:train:1574-1716batch: iter_time=2.697e-04, forward_time=0.128, loss_att=205.847, acc=0.768, loss=205.847, backward_time=0.191, grad_norm=87.799, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=4.007e-04, train_time=3.726 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:37:33,094 (trainer:732) INFO: 6epoch:train:1717-1859batch: iter_time=2.737e-04, forward_time=0.128, loss_att=206.317, acc=0.767, loss=206.317, backward_time=0.190, grad_norm=77.670, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=4.043e-04, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:39:39,789 (trainer:732) INFO: 6epoch:train:1860-2002batch: iter_time=2.656e-04, forward_time=0.128, loss_att=209.853, acc=0.766, loss=209.853, backward_time=0.191, grad_norm=79.648, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=4.078e-04, train_time=3.542 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:41:51,428 (trainer:732) INFO: 6epoch:train:2003-2145batch: iter_time=2.640e-04, forward_time=0.127, loss_att=202.420, acc=0.768, loss=202.420, backward_time=0.189, grad_norm=74.720, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=4.115e-04, train_time=3.685 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:44:04,343 (trainer:732) INFO: 6epoch:train:2146-2288batch: iter_time=2.820e-04, forward_time=0.128, loss_att=197.731, acc=0.771, loss=197.731, backward_time=0.190, grad_norm=90.897, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=4.150e-04, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:46:16,482 (trainer:732) INFO: 6epoch:train:2289-2431batch: iter_time=2.556e-04, forward_time=0.128, loss_att=193.132, acc=0.774, loss=193.132, backward_time=0.191, grad_norm=90.500, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.186e-04, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:48:24,957 (trainer:732) INFO: 6epoch:train:2432-2574batch: iter_time=2.603e-04, forward_time=0.130, loss_att=217.623, acc=0.766, loss=217.623, backward_time=0.194, grad_norm=84.591, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=4.221e-04, train_time=3.596 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:50:37,575 (trainer:732) INFO: 6epoch:train:2575-2717batch: iter_time=2.717e-04, forward_time=0.127, loss_att=196.363, acc=0.774, loss=196.363, backward_time=0.190, grad_norm=84.904, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=4.258e-04, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 20:52:50,080 (trainer:732) INFO: 6epoch:train:2718-2860batch: iter_time=2.504e-04, forward_time=0.128, loss_att=198.371, acc=0.772, loss=198.371, backward_time=0.190, grad_norm=78.992, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=4.294e-04, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:01:44,549 (trainer:338) INFO: 6epoch results: [train] iter_time=4.123e-04, forward_time=0.128, loss_att=204.871, acc=0.765, loss=204.871, backward_time=0.190, grad_norm=84.396, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=3.956e-04, train_time=3.863, time=46 minutes and 22.54 seconds, total_count=17274, gpu_max_cached_mem_GB=23.359, [valid] loss_att=93.410, acc=0.830, cer=0.212, wer=0.499, loss=93.410, time=4 minutes and 6.4 seconds, total_count=528, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 28.28 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:01:48,080 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:01:48,083 (trainer:272) INFO: 7/60epoch started. Estimated time to finish: 1 day, 10 hours and 51 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:06:17,042 (trainer:732) INFO: 7epoch:train:1-143batch: iter_time=0.003, forward_time=0.126, loss_att=182.712, acc=0.783, loss=182.712, backward_time=0.188, grad_norm=78.038, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.333e-04, train_time=7.615 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:08:28,684 (trainer:732) INFO: 7epoch:train:144-286batch: iter_time=2.666e-04, forward_time=0.127, loss_att=190.029, acc=0.782, loss=190.029, backward_time=0.189, grad_norm=80.358, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=4.369e-04, train_time=3.681 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:10:41,303 (trainer:732) INFO: 7epoch:train:287-429batch: iter_time=2.526e-04, forward_time=0.127, loss_att=194.098, acc=0.778, loss=194.098, backward_time=0.189, grad_norm=81.238, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=4.404e-04, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:12:50,258 (trainer:732) INFO: 7epoch:train:430-572batch: iter_time=2.744e-04, forward_time=0.129, loss_att=212.937, acc=0.777, loss=212.937, backward_time=0.194, grad_norm=94.698, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=4.440e-04, train_time=3.600 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:15:01,798 (trainer:732) INFO: 7epoch:train:573-715batch: iter_time=2.713e-04, forward_time=0.127, loss_att=194.420, acc=0.780, loss=194.420, backward_time=0.190, grad_norm=84.720, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=4.476e-04, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:17:13,126 (trainer:732) INFO: 7epoch:train:716-858batch: iter_time=2.550e-04, forward_time=0.127, loss_att=182.392, acc=0.785, loss=182.392, backward_time=0.189, grad_norm=90.303, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=4.511e-04, train_time=3.670 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:19:18,899 (trainer:732) INFO: 7epoch:train:859-1001batch: iter_time=2.684e-04, forward_time=0.128, loss_att=187.082, acc=0.784, loss=187.082, backward_time=0.191, grad_norm=85.484, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=4.548e-04, train_time=3.521 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:21:32,146 (trainer:732) INFO: 7epoch:train:1002-1144batch: iter_time=2.595e-04, forward_time=0.128, loss_att=194.184, acc=0.785, loss=194.184, backward_time=0.191, grad_norm=86.921, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=4.583e-04, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:23:43,924 (trainer:732) INFO: 7epoch:train:1145-1287batch: iter_time=2.577e-04, forward_time=0.129, loss_att=191.462, acc=0.786, loss=191.462, backward_time=0.192, grad_norm=83.650, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=4.619e-04, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:25:58,342 (trainer:732) INFO: 7epoch:train:1288-1430batch: iter_time=2.644e-04, forward_time=0.129, loss_att=203.514, acc=0.782, loss=203.514, backward_time=0.193, grad_norm=87.706, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=4.654e-04, train_time=3.756 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:28:05,300 (trainer:732) INFO: 7epoch:train:1431-1573batch: iter_time=2.541e-04, forward_time=0.127, loss_att=182.822, acc=0.788, loss=182.822, backward_time=0.189, grad_norm=95.722, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=4.690e-04, train_time=3.556 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:30:19,296 (trainer:732) INFO: 7epoch:train:1574-1716batch: iter_time=2.634e-04, forward_time=0.128, loss_att=193.613, acc=0.783, loss=193.613, backward_time=0.191, grad_norm=87.493, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=4.727e-04, train_time=3.735 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:32:33,183 (trainer:732) INFO: 7epoch:train:1717-1859batch: iter_time=2.648e-04, forward_time=0.128, loss_att=194.259, acc=0.784, loss=194.259, backward_time=0.192, grad_norm=87.013, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=4.762e-04, train_time=3.752 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:34:40,657 (trainer:732) INFO: 7epoch:train:1860-2002batch: iter_time=2.545e-04, forward_time=0.128, loss_att=195.489, acc=0.785, loss=195.489, backward_time=0.191, grad_norm=87.927, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=4.797e-04, train_time=3.568 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:36:52,885 (trainer:732) INFO: 7epoch:train:2003-2145batch: iter_time=2.581e-04, forward_time=0.126, loss_att=179.717, acc=0.791, loss=179.717, backward_time=0.188, grad_norm=85.080, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=4.833e-04, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:39:05,772 (trainer:732) INFO: 7epoch:train:2146-2288batch: iter_time=2.758e-04, forward_time=0.128, loss_att=190.799, acc=0.789, loss=190.799, backward_time=0.191, grad_norm=85.810, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=4.869e-04, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:41:18,282 (trainer:732) INFO: 7epoch:train:2289-2431batch: iter_time=2.596e-04, forward_time=0.126, loss_att=180.209, acc=0.795, loss=180.209, backward_time=0.188, grad_norm=87.354, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=4.905e-04, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:43:24,609 (trainer:732) INFO: 7epoch:train:2432-2574batch: iter_time=2.694e-04, forward_time=0.127, loss_att=185.500, acc=0.788, loss=185.500, backward_time=0.190, grad_norm=86.317, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=4.940e-04, train_time=3.538 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:45:37,796 (trainer:732) INFO: 7epoch:train:2575-2717batch: iter_time=2.594e-04, forward_time=0.127, loss_att=182.460, acc=0.790, loss=182.460, backward_time=0.189, grad_norm=90.075, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=4.976e-04, train_time=3.723 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:47:51,340 (trainer:732) INFO: 7epoch:train:2718-2860batch: iter_time=2.346e-04, forward_time=0.128, loss_att=191.701, acc=0.789, loss=191.701, backward_time=0.192, grad_norm=87.526, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=5.013e-04, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:56:41,058 (trainer:338) INFO: 7epoch results: [train] iter_time=4.048e-04, forward_time=0.128, loss_att=190.227, acc=0.785, loss=190.227, backward_time=0.190, grad_norm=86.673, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=4.675e-04, train_time=3.864, time=46 minutes and 22.66 seconds, total_count=20153, gpu_max_cached_mem_GB=23.359, [valid] loss_att=83.910, acc=0.850, cer=0.179, wer=0.432, loss=83.910, time=4 minutes and 7.1 seconds, total_count=616, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 23.22 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:56:44,770 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 21:56:44,773 (trainer:272) INFO: 8/60epoch started. Estimated time to finish: 1 day, 12 hours and 15 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:01:19,132 (trainer:732) INFO: 8epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=185.677, acc=0.795, loss=185.677, backward_time=0.191, grad_norm=96.486, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.052e-04, train_time=7.773 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:03:31,629 (trainer:732) INFO: 8epoch:train:144-286batch: iter_time=2.743e-04, forward_time=0.129, loss_att=191.824, acc=0.795, loss=191.824, backward_time=0.192, grad_norm=86.930, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.087e-04, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:05:43,780 (trainer:732) INFO: 8epoch:train:287-429batch: iter_time=2.610e-04, forward_time=0.127, loss_att=169.780, acc=0.803, loss=169.780, backward_time=0.188, grad_norm=82.970, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.124e-04, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:07:49,281 (trainer:732) INFO: 8epoch:train:430-572batch: iter_time=2.688e-04, forward_time=0.126, loss_att=174.248, acc=0.802, loss=174.248, backward_time=0.188, grad_norm=87.669, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.159e-04, train_time=3.503 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:10:00,453 (trainer:732) INFO: 8epoch:train:573-715batch: iter_time=2.883e-04, forward_time=0.127, loss_att=171.265, acc=0.801, loss=171.265, backward_time=0.189, grad_norm=80.418, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.195e-04, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:12:13,400 (trainer:732) INFO: 8epoch:train:716-858batch: iter_time=2.514e-04, forward_time=0.128, loss_att=186.032, acc=0.798, loss=186.032, backward_time=0.192, grad_norm=84.940, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.230e-04, train_time=3.722 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:14:22,454 (trainer:732) INFO: 8epoch:train:859-1001batch: iter_time=2.676e-04, forward_time=0.125, loss_att=174.971, acc=0.799, loss=174.971, backward_time=0.187, grad_norm=83.251, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=5.267e-04, train_time=3.615 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:16:32,154 (trainer:732) INFO: 8epoch:train:1002-1144batch: iter_time=2.739e-04, forward_time=0.129, loss_att=186.347, acc=0.799, loss=186.347, backward_time=0.192, grad_norm=95.699, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.302e-04, train_time=3.612 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:18:44,555 (trainer:732) INFO: 8epoch:train:1145-1287batch: iter_time=2.619e-04, forward_time=0.129, loss_att=179.451, acc=0.800, loss=179.451, backward_time=0.192, grad_norm=87.822, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=5.338e-04, train_time=3.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:20:56,594 (trainer:732) INFO: 8epoch:train:1288-1430batch: iter_time=2.630e-04, forward_time=0.128, loss_att=172.491, acc=0.804, loss=172.491, backward_time=0.191, grad_norm=89.156, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.374e-04, train_time=3.690 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:23:03,961 (trainer:732) INFO: 8epoch:train:1431-1573batch: iter_time=2.445e-04, forward_time=0.129, loss_att=194.726, acc=0.795, loss=194.726, backward_time=0.193, grad_norm=93.715, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=5.410e-04, train_time=3.565 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:25:14,706 (trainer:732) INFO: 8epoch:train:1574-1716batch: iter_time=2.655e-04, forward_time=0.126, loss_att=164.692, acc=0.803, loss=164.692, backward_time=0.187, grad_norm=81.246, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=5.445e-04, train_time=3.646 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:27:26,431 (trainer:732) INFO: 8epoch:train:1717-1859batch: iter_time=2.713e-04, forward_time=0.127, loss_att=180.940, acc=0.799, loss=180.940, backward_time=0.189, grad_norm=80.244, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=5.481e-04, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:29:40,800 (trainer:732) INFO: 8epoch:train:1860-2002batch: iter_time=2.600e-04, forward_time=0.130, loss_att=198.608, acc=0.798, loss=198.608, backward_time=0.196, grad_norm=91.101, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.516e-04, train_time=3.760 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:31:47,329 (trainer:732) INFO: 8epoch:train:2003-2145batch: iter_time=2.747e-04, forward_time=0.127, loss_att=170.100, acc=0.806, loss=170.100, backward_time=0.189, grad_norm=81.110, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=5.552e-04, train_time=3.539 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:34:00,966 (trainer:732) INFO: 8epoch:train:2146-2288batch: iter_time=2.805e-04, forward_time=0.129, loss_att=178.301, acc=0.802, loss=178.301, backward_time=0.192, grad_norm=89.497, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=5.588e-04, train_time=3.727 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:36:12,520 (trainer:732) INFO: 8epoch:train:2289-2431batch: iter_time=2.633e-04, forward_time=0.126, loss_att=163.950, acc=0.811, loss=163.950, backward_time=0.188, grad_norm=81.453, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.624e-04, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:38:19,926 (trainer:732) INFO: 8epoch:train:2432-2574batch: iter_time=2.812e-04, forward_time=0.128, loss_att=177.267, acc=0.804, loss=177.267, backward_time=0.191, grad_norm=90.385, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.659e-04, train_time=3.559 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:40:31,665 (trainer:732) INFO: 8epoch:train:2575-2717batch: iter_time=2.824e-04, forward_time=0.128, loss_att=175.013, acc=0.804, loss=175.013, backward_time=0.190, grad_norm=82.812, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=5.695e-04, train_time=3.685 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:42:44,422 (trainer:732) INFO: 8epoch:train:2718-2860batch: iter_time=2.676e-04, forward_time=0.128, loss_att=174.691, acc=0.804, loss=174.691, backward_time=0.190, grad_norm=86.091, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=5.732e-04, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:51:33,297 (trainer:338) INFO: 8epoch results: [train] iter_time=3.982e-04, forward_time=0.128, loss_att=178.415, acc=0.801, loss=178.415, backward_time=0.190, grad_norm=86.860, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.394e-04, train_time=3.858, time=46 minutes and 18.38 seconds, total_count=23032, gpu_max_cached_mem_GB=23.359, [valid] loss_att=76.779, acc=0.862, cer=0.166, wer=0.403, loss=76.779, time=4 minutes and 3.48 seconds, total_count=704, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 26.65 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:51:37,704 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:51:37,708 (trainer:272) INFO: 9/60epoch started. Estimated time to finish: 1 day, 13 hours and 4 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:56:00,973 (trainer:732) INFO: 9epoch:train:1-143batch: iter_time=0.003, forward_time=0.127, loss_att=164.071, acc=0.815, loss=164.071, backward_time=0.189, grad_norm=88.968, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.771e-04, train_time=7.457 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 22:58:15,058 (trainer:732) INFO: 9epoch:train:144-286batch: iter_time=2.605e-04, forward_time=0.128, loss_att=177.519, acc=0.811, loss=177.519, backward_time=0.192, grad_norm=87.928, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.806e-04, train_time=3.747 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:00:26,762 (trainer:732) INFO: 9epoch:train:287-429batch: iter_time=2.761e-04, forward_time=0.126, loss_att=165.797, acc=0.811, loss=165.797, backward_time=0.187, grad_norm=78.631, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.842e-04, train_time=3.681 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:02:36,894 (trainer:732) INFO: 9epoch:train:430-572batch: iter_time=2.776e-04, forward_time=0.129, loss_att=181.294, acc=0.810, loss=181.294, backward_time=0.194, grad_norm=94.193, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=5.878e-04, train_time=3.632 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:04:47,943 (trainer:732) INFO: 9epoch:train:573-715batch: iter_time=2.644e-04, forward_time=0.130, loss_att=185.608, acc=0.807, loss=185.608, backward_time=0.195, grad_norm=86.606, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=5.914e-04, train_time=3.674 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:06:59,377 (trainer:732) INFO: 9epoch:train:716-858batch: iter_time=2.516e-04, forward_time=0.126, loss_att=160.951, acc=0.811, loss=160.951, backward_time=0.188, grad_norm=73.433, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=5.949e-04, train_time=3.678 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:09:13,484 (trainer:732) INFO: 9epoch:train:859-1001batch: iter_time=2.703e-04, forward_time=0.129, loss_att=181.841, acc=0.807, loss=181.841, backward_time=0.193, grad_norm=86.782, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=5.985e-04, train_time=3.749 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:11:20,812 (trainer:732) INFO: 9epoch:train:1002-1144batch: iter_time=2.675e-04, forward_time=0.128, loss_att=179.807, acc=0.807, loss=179.807, backward_time=0.191, grad_norm=79.818, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=6.021e-04, train_time=3.553 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:13:34,268 (trainer:732) INFO: 9epoch:train:1145-1287batch: iter_time=2.578e-04, forward_time=0.130, loss_att=183.784, acc=0.807, loss=183.784, backward_time=0.194, grad_norm=88.992, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.057e-04, train_time=3.747 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:15:46,527 (trainer:732) INFO: 9epoch:train:1288-1430batch: iter_time=2.664e-04, forward_time=0.127, loss_att=164.346, acc=0.815, loss=164.346, backward_time=0.189, grad_norm=91.027, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.092e-04, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:17:55,646 (trainer:732) INFO: 9epoch:train:1431-1573batch: iter_time=2.549e-04, forward_time=0.129, loss_att=173.350, acc=0.809, loss=173.350, backward_time=0.191, grad_norm=91.218, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.128e-04, train_time=3.615 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:20:06,071 (trainer:732) INFO: 9epoch:train:1574-1716batch: iter_time=2.697e-04, forward_time=0.127, loss_att=166.965, acc=0.812, loss=166.965, backward_time=0.189, grad_norm=83.265, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.164e-04, train_time=3.632 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:22:16,037 (trainer:732) INFO: 9epoch:train:1717-1859batch: iter_time=2.725e-04, forward_time=0.126, loss_att=153.178, acc=0.820, loss=153.178, backward_time=0.187, grad_norm=81.379, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.200e-04, train_time=3.645 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:24:27,878 (trainer:732) INFO: 9epoch:train:1860-2002batch: iter_time=2.593e-04, forward_time=0.128, loss_att=161.423, acc=0.816, loss=161.423, backward_time=0.190, grad_norm=84.594, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.235e-04, train_time=3.689 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:26:33,485 (trainer:732) INFO: 9epoch:train:2003-2145batch: iter_time=2.742e-04, forward_time=0.126, loss_att=161.789, acc=0.815, loss=161.789, backward_time=0.187, grad_norm=86.587, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.272e-04, train_time=3.511 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:28:47,150 (trainer:732) INFO: 9epoch:train:2146-2288batch: iter_time=2.849e-04, forward_time=0.129, loss_att=165.389, acc=0.815, loss=165.389, backward_time=0.192, grad_norm=95.235, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.307e-04, train_time=3.729 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:30:59,639 (trainer:732) INFO: 9epoch:train:2289-2431batch: iter_time=2.556e-04, forward_time=0.128, loss_att=170.596, acc=0.811, loss=170.596, backward_time=0.191, grad_norm=89.650, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.343e-04, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:33:11,104 (trainer:732) INFO: 9epoch:train:2432-2574batch: iter_time=2.490e-04, forward_time=0.127, loss_att=161.197, acc=0.817, loss=161.197, backward_time=0.189, grad_norm=86.457, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.379e-04, train_time=3.690 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:35:18,102 (trainer:732) INFO: 9epoch:train:2575-2717batch: iter_time=2.583e-04, forward_time=0.128, loss_att=175.919, acc=0.809, loss=175.919, backward_time=0.191, grad_norm=85.749, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.415e-04, train_time=3.539 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:37:28,988 (trainer:732) INFO: 9epoch:train:2718-2860batch: iter_time=2.584e-04, forward_time=0.127, loss_att=151.772, acc=0.822, loss=151.772, backward_time=0.188, grad_norm=78.033, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.450e-04, train_time=3.651 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:46:18,869 (trainer:338) INFO: 9epoch results: [train] iter_time=4.119e-04, forward_time=0.128, loss_att=168.969, acc=0.813, loss=168.969, backward_time=0.190, grad_norm=85.911, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.113e-04, train_time=3.847, time=46 minutes and 10.29 seconds, total_count=25911, gpu_max_cached_mem_GB=23.359, [valid] loss_att=74.292, acc=0.865, cer=0.165, wer=0.410, loss=74.292, time=4 minutes and 2.6 seconds, total_count=792, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 28.27 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:46:23,047 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:46:23,051 (trainer:272) INFO: 10/60epoch started. Estimated time to finish: 1 day, 13 hours and 29 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:50:49,053 (trainer:732) INFO: 10epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=166.937, acc=0.815, loss=166.937, backward_time=0.190, grad_norm=87.242, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.490e-04, train_time=7.530 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:53:00,585 (trainer:732) INFO: 10epoch:train:144-286batch: iter_time=2.814e-04, forward_time=0.127, loss_att=159.430, acc=0.819, loss=159.430, backward_time=0.190, grad_norm=80.462, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=6.526e-04, train_time=3.683 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:55:13,662 (trainer:732) INFO: 10epoch:train:287-429batch: iter_time=2.660e-04, forward_time=0.130, loss_att=176.684, acc=0.812, loss=176.684, backward_time=0.194, grad_norm=86.976, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.561e-04, train_time=3.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:57:29,319 (trainer:732) INFO: 10epoch:train:430-572batch: iter_time=2.692e-04, forward_time=0.129, loss_att=181.158, acc=0.815, loss=181.158, backward_time=0.193, grad_norm=96.201, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.598e-04, train_time=3.785 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-20 23:59:36,174 (trainer:732) INFO: 10epoch:train:573-715batch: iter_time=2.801e-04, forward_time=0.127, loss_att=157.302, acc=0.823, loss=157.302, backward_time=0.189, grad_norm=81.672, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.633e-04, train_time=3.553 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:01:50,157 (trainer:732) INFO: 10epoch:train:716-858batch: iter_time=2.584e-04, forward_time=0.128, loss_att=164.068, acc=0.822, loss=164.068, backward_time=0.191, grad_norm=84.844, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.668e-04, train_time=3.751 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:04:04,203 (trainer:732) INFO: 10epoch:train:859-1001batch: iter_time=2.827e-04, forward_time=0.130, loss_att=177.890, acc=0.816, loss=177.890, backward_time=0.194, grad_norm=93.502, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=6.704e-04, train_time=3.749 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:06:11,920 (trainer:732) INFO: 10epoch:train:1002-1144batch: iter_time=2.585e-04, forward_time=0.128, loss_att=164.472, acc=0.819, loss=164.472, backward_time=0.192, grad_norm=88.224, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.740e-04, train_time=3.562 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:08:23,449 (trainer:732) INFO: 10epoch:train:1145-1287batch: iter_time=2.683e-04, forward_time=0.126, loss_att=154.609, acc=0.824, loss=154.609, backward_time=0.187, grad_norm=76.849, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.776e-04, train_time=3.688 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:10:36,767 (trainer:732) INFO: 10epoch:train:1288-1430batch: iter_time=2.777e-04, forward_time=0.128, loss_att=166.898, acc=0.819, loss=166.898, backward_time=0.191, grad_norm=89.189, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=6.811e-04, train_time=3.731 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:12:48,090 (trainer:732) INFO: 10epoch:train:1431-1573batch: iter_time=2.567e-04, forward_time=0.127, loss_att=147.127, acc=0.828, loss=147.127, backward_time=0.188, grad_norm=74.346, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=6.847e-04, train_time=3.669 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:14:54,529 (trainer:732) INFO: 10epoch:train:1574-1716batch: iter_time=2.644e-04, forward_time=0.126, loss_att=149.137, acc=0.826, loss=149.137, backward_time=0.187, grad_norm=82.804, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=6.883e-04, train_time=3.529 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:17:05,452 (trainer:732) INFO: 10epoch:train:1717-1859batch: iter_time=2.682e-04, forward_time=0.127, loss_att=156.058, acc=0.825, loss=156.058, backward_time=0.189, grad_norm=85.190, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.919e-04, train_time=3.672 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:19:18,236 (trainer:732) INFO: 10epoch:train:1860-2002batch: iter_time=2.785e-04, forward_time=0.128, loss_att=165.653, acc=0.822, loss=165.653, backward_time=0.190, grad_norm=83.947, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.954e-04, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:21:23,371 (trainer:732) INFO: 10epoch:train:2003-2145batch: iter_time=2.681e-04, forward_time=0.126, loss_att=148.282, acc=0.827, loss=148.282, backward_time=0.187, grad_norm=81.535, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=6.990e-04, train_time=3.498 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:23:35,816 (trainer:732) INFO: 10epoch:train:2146-2288batch: iter_time=2.766e-04, forward_time=0.129, loss_att=163.239, acc=0.821, loss=163.239, backward_time=0.191, grad_norm=83.674, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=7.026e-04, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:25:46,025 (trainer:732) INFO: 10epoch:train:2289-2431batch: iter_time=2.719e-04, forward_time=0.127, loss_att=153.433, acc=0.825, loss=153.433, backward_time=0.188, grad_norm=74.485, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.062e-04, train_time=3.656 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:27:59,756 (trainer:732) INFO: 10epoch:train:2432-2574batch: iter_time=2.803e-04, forward_time=0.129, loss_att=165.077, acc=0.821, loss=165.077, backward_time=0.193, grad_norm=82.588, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.097e-04, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:30:06,420 (trainer:732) INFO: 10epoch:train:2575-2717batch: iter_time=2.870e-04, forward_time=0.128, loss_att=160.979, acc=0.824, loss=160.979, backward_time=0.191, grad_norm=90.294, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.133e-04, train_time=3.546 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:32:19,270 (trainer:732) INFO: 10epoch:train:2718-2860batch: iter_time=2.569e-04, forward_time=0.127, loss_att=155.459, acc=0.826, loss=155.459, backward_time=0.190, grad_norm=81.516, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.169e-04, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:41:06,896 (trainer:338) INFO: 10epoch results: [train] iter_time=4.254e-04, forward_time=0.128, loss_att=161.382, acc=0.822, loss=161.382, backward_time=0.190, grad_norm=84.340, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=6.832e-04, train_time=3.854, time=46 minutes and 15.73 seconds, total_count=28790, gpu_max_cached_mem_GB=23.359, [valid] loss_att=70.293, acc=0.874, cer=0.153, wer=0.372, loss=70.293, time=4 minutes and 3.18 seconds, total_count=880, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 24.94 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:41:10,983 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:41:10,988 (trainer:272) INFO: 11/60epoch started. Estimated time to finish: 1 day, 13 hours and 38 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:45:36,320 (trainer:732) INFO: 11epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=157.049, acc=0.825, loss=157.049, backward_time=0.190, grad_norm=89.441, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.209e-04, train_time=7.511 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:47:47,203 (trainer:732) INFO: 11epoch:train:144-286batch: iter_time=2.728e-04, forward_time=0.126, loss_att=145.191, acc=0.828, loss=145.191, backward_time=0.187, grad_norm=76.197, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=7.244e-04, train_time=3.659 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:50:00,280 (trainer:732) INFO: 11epoch:train:287-429batch: iter_time=2.773e-04, forward_time=0.129, loss_att=153.269, acc=0.830, loss=153.269, backward_time=0.191, grad_norm=82.941, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.280e-04, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:52:14,045 (trainer:732) INFO: 11epoch:train:430-572batch: iter_time=3.053e-04, forward_time=0.129, loss_att=165.207, acc=0.824, loss=165.207, backward_time=0.193, grad_norm=92.136, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=7.316e-04, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:54:20,652 (trainer:732) INFO: 11epoch:train:573-715batch: iter_time=2.756e-04, forward_time=0.128, loss_att=152.214, acc=0.830, loss=152.214, backward_time=0.190, grad_norm=89.106, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.352e-04, train_time=3.548 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:56:32,132 (trainer:732) INFO: 11epoch:train:716-858batch: iter_time=2.978e-04, forward_time=0.127, loss_att=143.933, acc=0.834, loss=143.933, backward_time=0.189, grad_norm=87.703, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.388e-04, train_time=3.678 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 00:58:45,108 (trainer:732) INFO: 11epoch:train:859-1001batch: iter_time=3.053e-04, forward_time=0.129, loss_att=161.257, acc=0.826, loss=161.257, backward_time=0.193, grad_norm=86.713, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.423e-04, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:00:54,128 (trainer:732) INFO: 11epoch:train:1002-1144batch: iter_time=2.880e-04, forward_time=0.130, loss_att=169.170, acc=0.826, loss=169.170, backward_time=0.195, grad_norm=95.487, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=7.459e-04, train_time=3.602 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:03:06,103 (trainer:732) INFO: 11epoch:train:1145-1287batch: iter_time=2.830e-04, forward_time=0.128, loss_att=159.466, acc=0.827, loss=159.466, backward_time=0.191, grad_norm=84.549, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.495e-04, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:05:18,520 (trainer:732) INFO: 11epoch:train:1288-1430batch: iter_time=2.929e-04, forward_time=0.127, loss_att=153.717, acc=0.828, loss=153.717, backward_time=0.189, grad_norm=82.058, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.530e-04, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:07:31,313 (trainer:732) INFO: 11epoch:train:1431-1573batch: iter_time=2.707e-04, forward_time=0.127, loss_att=157.264, acc=0.827, loss=157.264, backward_time=0.190, grad_norm=79.008, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.566e-04, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:09:39,665 (trainer:732) INFO: 11epoch:train:1574-1716batch: iter_time=2.901e-04, forward_time=0.129, loss_att=169.394, acc=0.823, loss=169.394, backward_time=0.194, grad_norm=93.958, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.602e-04, train_time=3.582 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:11:51,430 (trainer:732) INFO: 11epoch:train:1717-1859batch: iter_time=2.928e-04, forward_time=0.127, loss_att=153.699, acc=0.829, loss=153.699, backward_time=0.189, grad_norm=78.611, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=7.638e-04, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:14:03,717 (trainer:732) INFO: 11epoch:train:1860-2002batch: iter_time=2.685e-04, forward_time=0.128, loss_att=147.538, acc=0.834, loss=147.538, backward_time=0.190, grad_norm=83.274, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=7.673e-04, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:16:11,042 (trainer:732) INFO: 11epoch:train:2003-2145batch: iter_time=2.844e-04, forward_time=0.128, loss_att=152.461, acc=0.832, loss=152.461, backward_time=0.191, grad_norm=84.878, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=7.709e-04, train_time=3.562 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:18:22,086 (trainer:732) INFO: 11epoch:train:2146-2288batch: iter_time=2.851e-04, forward_time=0.126, loss_att=143.653, acc=0.834, loss=143.653, backward_time=0.188, grad_norm=84.076, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.746e-04, train_time=3.653 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:20:33,842 (trainer:732) INFO: 11epoch:train:2289-2431batch: iter_time=2.751e-04, forward_time=0.127, loss_att=149.326, acc=0.833, loss=149.326, backward_time=0.189, grad_norm=93.731, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.781e-04, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:22:47,135 (trainer:732) INFO: 11epoch:train:2432-2574batch: iter_time=2.860e-04, forward_time=0.128, loss_att=153.807, acc=0.829, loss=153.807, backward_time=0.191, grad_norm=81.078, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=7.816e-04, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:24:52,736 (trainer:732) INFO: 11epoch:train:2575-2717batch: iter_time=2.904e-04, forward_time=0.126, loss_att=147.357, acc=0.833, loss=147.357, backward_time=0.188, grad_norm=82.591, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=7.852e-04, train_time=3.518 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:27:06,117 (trainer:732) INFO: 11epoch:train:2718-2860batch: iter_time=2.624e-04, forward_time=0.129, loss_att=166.019, acc=0.826, loss=166.019, backward_time=0.193, grad_norm=90.134, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=7.889e-04, train_time=3.719 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:35:51,764 (trainer:338) INFO: 11epoch results: [train] iter_time=4.058e-04, forward_time=0.128, loss_att=154.866, acc=0.829, loss=154.866, backward_time=0.190, grad_norm=85.889, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=7.551e-04, train_time=3.853, time=46 minutes and 14.72 seconds, total_count=31669, gpu_max_cached_mem_GB=23.359, [valid] loss_att=66.136, acc=0.880, cer=0.144, wer=0.360, loss=66.136, time=4 minutes and 0.65 seconds, total_count=968, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.4 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:35:55,413 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:35:55,418 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/1epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:35:55,419 (trainer:272) INFO: 12/60epoch started. Estimated time to finish: 1 day, 13 hours and 36 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:40:26,737 (trainer:732) INFO: 12epoch:train:1-143batch: iter_time=0.003, forward_time=0.132, loss_att=148.789, acc=0.835, loss=148.789, backward_time=0.191, grad_norm=74.171, clip=100.000, loss_scale=1.000, optim_step_time=0.069, optim0_lr0=7.928e-04, train_time=7.685 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:42:34,204 (trainer:732) INFO: 12epoch:train:144-286batch: iter_time=2.973e-04, forward_time=0.130, loss_att=150.108, acc=0.835, loss=150.108, backward_time=0.191, grad_norm=81.015, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=7.963e-04, train_time=3.562 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:44:47,361 (trainer:732) INFO: 12epoch:train:287-429batch: iter_time=2.837e-04, forward_time=0.130, loss_att=157.939, acc=0.833, loss=157.939, backward_time=0.193, grad_norm=88.250, clip=100.000, loss_scale=1.000, optim_step_time=0.067, optim0_lr0=7.999e-04, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:47:00,440 (trainer:732) INFO: 12epoch:train:430-572batch: iter_time=2.952e-04, forward_time=0.128, loss_att=149.768, acc=0.832, loss=149.768, backward_time=0.189, grad_norm=78.553, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.035e-04, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:49:05,860 (trainer:732) INFO: 12epoch:train:573-715batch: iter_time=3.137e-04, forward_time=0.128, loss_att=140.307, acc=0.839, loss=140.307, backward_time=0.188, grad_norm=83.273, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.071e-04, train_time=3.526 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:51:19,173 (trainer:732) INFO: 12epoch:train:716-858batch: iter_time=3.147e-04, forward_time=0.130, loss_att=165.538, acc=0.828, loss=165.538, backward_time=0.194, grad_norm=84.456, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.106e-04, train_time=3.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:53:30,972 (trainer:732) INFO: 12epoch:train:859-1001batch: iter_time=3.391e-04, forward_time=0.127, loss_att=141.281, acc=0.838, loss=141.281, backward_time=0.189, grad_norm=80.164, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.142e-04, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:55:42,578 (trainer:732) INFO: 12epoch:train:1002-1144batch: iter_time=3.228e-04, forward_time=0.126, loss_att=141.587, acc=0.838, loss=141.587, backward_time=0.187, grad_norm=88.502, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.178e-04, train_time=3.671 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 01:57:49,859 (trainer:732) INFO: 12epoch:train:1145-1287batch: iter_time=3.213e-04, forward_time=0.129, loss_att=152.187, acc=0.835, loss=152.187, backward_time=0.192, grad_norm=84.386, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.214e-04, train_time=3.567 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:00:03,425 (trainer:732) INFO: 12epoch:train:1288-1430batch: iter_time=3.208e-04, forward_time=0.128, loss_att=151.684, acc=0.834, loss=151.684, backward_time=0.190, grad_norm=84.672, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.249e-04, train_time=3.735 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:02:15,197 (trainer:732) INFO: 12epoch:train:1431-1573batch: iter_time=3.032e-04, forward_time=0.127, loss_att=147.667, acc=0.834, loss=147.667, backward_time=0.190, grad_norm=80.670, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.285e-04, train_time=3.688 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:04:23,621 (trainer:732) INFO: 12epoch:train:1574-1716batch: iter_time=3.207e-04, forward_time=0.130, loss_att=162.004, acc=0.831, loss=162.004, backward_time=0.193, grad_norm=85.291, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.321e-04, train_time=3.581 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:06:35,149 (trainer:732) INFO: 12epoch:train:1717-1859batch: iter_time=3.152e-04, forward_time=0.127, loss_att=144.612, acc=0.837, loss=144.612, backward_time=0.189, grad_norm=79.921, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.357e-04, train_time=3.688 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:08:47,218 (trainer:732) INFO: 12epoch:train:1860-2002batch: iter_time=2.964e-04, forward_time=0.128, loss_att=144.198, acc=0.838, loss=144.198, backward_time=0.191, grad_norm=82.278, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.392e-04, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:10:59,707 (trainer:732) INFO: 12epoch:train:2003-2145batch: iter_time=3.092e-04, forward_time=0.128, loss_att=149.627, acc=0.835, loss=149.627, backward_time=0.190, grad_norm=80.348, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.428e-04, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:13:06,319 (trainer:732) INFO: 12epoch:train:2146-2288batch: iter_time=3.163e-04, forward_time=0.129, loss_att=147.419, acc=0.835, loss=147.419, backward_time=0.191, grad_norm=75.765, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.464e-04, train_time=3.532 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:15:18,574 (trainer:732) INFO: 12epoch:train:2289-2431batch: iter_time=3.114e-04, forward_time=0.128, loss_att=148.868, acc=0.834, loss=148.868, backward_time=0.191, grad_norm=84.382, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.500e-04, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:17:32,222 (trainer:732) INFO: 12epoch:train:2432-2574batch: iter_time=3.151e-04, forward_time=0.130, loss_att=159.306, acc=0.830, loss=159.306, backward_time=0.195, grad_norm=84.276, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.536e-04, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:19:40,292 (trainer:732) INFO: 12epoch:train:2575-2717batch: iter_time=3.095e-04, forward_time=0.127, loss_att=144.810, acc=0.836, loss=144.810, backward_time=0.188, grad_norm=81.058, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.571e-04, train_time=3.586 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:21:50,791 (trainer:732) INFO: 12epoch:train:2718-2860batch: iter_time=2.909e-04, forward_time=0.128, loss_att=147.385, acc=0.837, loss=147.385, backward_time=0.190, grad_norm=79.814, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.607e-04, train_time=3.634 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:30:44,466 (trainer:338) INFO: 12epoch results: [train] iter_time=4.394e-04, forward_time=0.129, loss_att=149.839, acc=0.835, loss=149.839, backward_time=0.191, grad_norm=82.292, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.270e-04, train_time=3.853, time=46 minutes and 15.29 seconds, total_count=34548, gpu_max_cached_mem_GB=23.359, [valid] loss_att=64.479, acc=0.884, cer=0.140, wer=0.355, loss=64.479, time=4 minutes and 3.35 seconds, total_count=1056, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 30.41 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:30:48,323 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:30:48,330 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/2epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:30:48,331 (trainer:272) INFO: 13/60epoch started. Estimated time to finish: 1 day, 13 hours and 25 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:35:24,430 (trainer:732) INFO: 13epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=152.747, acc=0.835, loss=152.747, backward_time=0.193, grad_norm=83.885, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.647e-04, train_time=7.817 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:37:31,736 (trainer:732) INFO: 13epoch:train:144-286batch: iter_time=3.027e-04, forward_time=0.129, loss_att=159.710, acc=0.833, loss=159.710, backward_time=0.192, grad_norm=83.294, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.682e-04, train_time=3.558 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:39:44,817 (trainer:732) INFO: 13epoch:train:287-429batch: iter_time=3.084e-04, forward_time=0.127, loss_att=143.550, acc=0.839, loss=143.550, backward_time=0.189, grad_norm=75.129, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.718e-04, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:41:56,846 (trainer:732) INFO: 13epoch:train:430-572batch: iter_time=3.240e-04, forward_time=0.127, loss_att=130.009, acc=0.849, loss=130.009, backward_time=0.188, grad_norm=76.991, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=8.754e-04, train_time=3.684 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:44:10,664 (trainer:732) INFO: 13epoch:train:573-715batch: iter_time=3.109e-04, forward_time=0.129, loss_att=155.420, acc=0.838, loss=155.420, backward_time=0.192, grad_norm=84.271, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.790e-04, train_time=3.757 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:46:17,906 (trainer:732) INFO: 13epoch:train:716-858batch: iter_time=2.995e-04, forward_time=0.129, loss_att=146.807, acc=0.842, loss=146.807, backward_time=0.192, grad_norm=81.925, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.825e-04, train_time=3.558 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:48:31,621 (trainer:732) INFO: 13epoch:train:859-1001batch: iter_time=3.275e-04, forward_time=0.129, loss_att=151.971, acc=0.837, loss=151.971, backward_time=0.193, grad_norm=83.513, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.861e-04, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:50:44,596 (trainer:732) INFO: 13epoch:train:1002-1144batch: iter_time=3.024e-04, forward_time=0.128, loss_att=150.726, acc=0.834, loss=150.726, backward_time=0.191, grad_norm=78.702, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=8.897e-04, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:52:50,014 (trainer:732) INFO: 13epoch:train:1145-1287batch: iter_time=3.135e-04, forward_time=0.127, loss_att=135.738, acc=0.844, loss=135.738, backward_time=0.189, grad_norm=76.598, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=8.933e-04, train_time=3.524 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:55:02,774 (trainer:732) INFO: 13epoch:train:1288-1430batch: iter_time=3.241e-04, forward_time=0.128, loss_att=145.646, acc=0.839, loss=145.646, backward_time=0.191, grad_norm=79.864, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.968e-04, train_time=3.704 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:57:15,048 (trainer:732) INFO: 13epoch:train:1431-1573batch: iter_time=3.092e-04, forward_time=0.128, loss_att=142.721, acc=0.840, loss=142.721, backward_time=0.190, grad_norm=75.868, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.004e-04, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 02:59:27,631 (trainer:732) INFO: 13epoch:train:1574-1716batch: iter_time=3.014e-04, forward_time=0.128, loss_att=143.815, acc=0.841, loss=143.815, backward_time=0.190, grad_norm=80.520, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.040e-04, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:01:33,270 (trainer:732) INFO: 13epoch:train:1717-1859batch: iter_time=2.983e-04, forward_time=0.128, loss_att=145.821, acc=0.841, loss=145.821, backward_time=0.191, grad_norm=80.473, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.076e-04, train_time=3.522 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:03:45,616 (trainer:732) INFO: 13epoch:train:1860-2002batch: iter_time=2.997e-04, forward_time=0.128, loss_att=146.081, acc=0.840, loss=146.081, backward_time=0.191, grad_norm=79.379, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.111e-04, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:05:59,009 (trainer:732) INFO: 13epoch:train:2003-2145batch: iter_time=3.081e-04, forward_time=0.129, loss_att=148.210, acc=0.839, loss=148.210, backward_time=0.192, grad_norm=83.007, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.148e-04, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:08:04,796 (trainer:732) INFO: 13epoch:train:2146-2288batch: iter_time=3.122e-04, forward_time=0.126, loss_att=137.026, acc=0.846, loss=137.026, backward_time=0.188, grad_norm=81.170, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.183e-04, train_time=3.508 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:10:16,714 (trainer:732) INFO: 13epoch:train:2289-2431batch: iter_time=3.035e-04, forward_time=0.128, loss_att=129.210, acc=0.850, loss=129.210, backward_time=0.190, grad_norm=79.063, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.219e-04, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:12:29,777 (trainer:732) INFO: 13epoch:train:2432-2574batch: iter_time=3.064e-04, forward_time=0.130, loss_att=149.296, acc=0.839, loss=149.296, backward_time=0.193, grad_norm=87.881, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.254e-04, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:14:42,074 (trainer:732) INFO: 13epoch:train:2575-2717batch: iter_time=3.224e-04, forward_time=0.128, loss_att=150.210, acc=0.838, loss=150.210, backward_time=0.191, grad_norm=78.384, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.290e-04, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:16:48,435 (trainer:732) INFO: 13epoch:train:2718-2860batch: iter_time=2.836e-04, forward_time=0.127, loss_att=138.575, acc=0.842, loss=138.575, backward_time=0.189, grad_norm=83.698, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.326e-04, train_time=3.524 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:25:41,964 (trainer:338) INFO: 13epoch results: [train] iter_time=4.430e-04, forward_time=0.128, loss_att=144.813, acc=0.840, loss=144.813, backward_time=0.191, grad_norm=80.649, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=8.989e-04, train_time=3.859, time=46 minutes and 19.3 seconds, total_count=37427, gpu_max_cached_mem_GB=23.359, [valid] loss_att=62.490, acc=0.887, cer=0.137, wer=0.352, loss=62.490, time=4 minutes and 4.56 seconds, total_count=1144, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 29.77 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:25:46,235 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:25:46,242 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/3epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:25:46,242 (trainer:272) INFO: 14/60epoch started. Estimated time to finish: 1 day, 13 hours and 8 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:30:22,037 (trainer:732) INFO: 14epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=156.234, acc=0.839, loss=156.234, backward_time=0.194, grad_norm=85.536, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.366e-04, train_time=7.803 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:32:29,703 (trainer:732) INFO: 14epoch:train:144-286batch: iter_time=3.005e-04, forward_time=0.127, loss_att=139.253, acc=0.845, loss=139.253, backward_time=0.190, grad_norm=74.455, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.401e-04, train_time=3.580 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:34:42,908 (trainer:732) INFO: 14epoch:train:287-429batch: iter_time=2.993e-04, forward_time=0.128, loss_att=142.558, acc=0.843, loss=142.558, backward_time=0.191, grad_norm=83.118, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.437e-04, train_time=3.726 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:36:56,374 (trainer:732) INFO: 14epoch:train:430-572batch: iter_time=3.063e-04, forward_time=0.129, loss_att=147.512, acc=0.842, loss=147.512, backward_time=0.193, grad_norm=88.815, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.474e-04, train_time=3.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:39:09,560 (trainer:732) INFO: 14epoch:train:573-715batch: iter_time=2.959e-04, forward_time=0.129, loss_att=143.313, acc=0.846, loss=143.313, backward_time=0.192, grad_norm=86.512, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.509e-04, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:41:15,721 (trainer:732) INFO: 14epoch:train:716-858batch: iter_time=3.045e-04, forward_time=0.126, loss_att=131.225, acc=0.852, loss=131.225, backward_time=0.187, grad_norm=82.363, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.544e-04, train_time=3.535 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:43:27,587 (trainer:732) INFO: 14epoch:train:859-1001batch: iter_time=3.632e-04, forward_time=0.128, loss_att=134.788, acc=0.849, loss=134.788, backward_time=0.191, grad_norm=73.056, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.581e-04, train_time=3.686 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:45:41,290 (trainer:732) INFO: 14epoch:train:1002-1144batch: iter_time=2.983e-04, forward_time=0.130, loss_att=151.003, acc=0.839, loss=151.003, backward_time=0.194, grad_norm=81.264, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=9.616e-04, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:47:45,621 (trainer:732) INFO: 14epoch:train:1145-1287batch: iter_time=3.067e-04, forward_time=0.125, loss_att=130.049, acc=0.849, loss=130.049, backward_time=0.185, grad_norm=73.846, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.652e-04, train_time=3.484 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:49:57,173 (trainer:732) INFO: 14epoch:train:1288-1430batch: iter_time=3.031e-04, forward_time=0.128, loss_att=135.972, acc=0.844, loss=135.972, backward_time=0.190, grad_norm=75.403, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.687e-04, train_time=3.680 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:52:09,466 (trainer:732) INFO: 14epoch:train:1431-1573batch: iter_time=2.979e-04, forward_time=0.127, loss_att=133.253, acc=0.851, loss=133.253, backward_time=0.189, grad_norm=81.691, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=9.723e-04, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:54:22,692 (trainer:732) INFO: 14epoch:train:1574-1716batch: iter_time=3.107e-04, forward_time=0.128, loss_att=139.559, acc=0.848, loss=139.559, backward_time=0.191, grad_norm=81.299, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.759e-04, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:56:28,915 (trainer:732) INFO: 14epoch:train:1717-1859batch: iter_time=3.179e-04, forward_time=0.127, loss_att=141.592, acc=0.845, loss=141.592, backward_time=0.190, grad_norm=80.899, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=9.795e-04, train_time=3.533 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 03:58:41,614 (trainer:732) INFO: 14epoch:train:1860-2002batch: iter_time=2.953e-04, forward_time=0.128, loss_att=141.063, acc=0.844, loss=141.063, backward_time=0.190, grad_norm=79.763, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.831e-04, train_time=3.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:00:54,557 (trainer:732) INFO: 14epoch:train:2003-2145batch: iter_time=3.301e-04, forward_time=0.128, loss_att=140.207, acc=0.846, loss=140.207, backward_time=0.191, grad_norm=84.870, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.866e-04, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:03:02,184 (trainer:732) INFO: 14epoch:train:2146-2288batch: iter_time=3.119e-04, forward_time=0.128, loss_att=138.895, acc=0.848, loss=138.895, backward_time=0.192, grad_norm=79.070, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.903e-04, train_time=3.560 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:05:15,324 (trainer:732) INFO: 14epoch:train:2289-2431batch: iter_time=3.076e-04, forward_time=0.128, loss_att=146.635, acc=0.840, loss=146.635, backward_time=0.191, grad_norm=76.175, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.938e-04, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:07:27,428 (trainer:732) INFO: 14epoch:train:2432-2574batch: iter_time=3.174e-04, forward_time=0.128, loss_att=136.762, acc=0.848, loss=136.762, backward_time=0.190, grad_norm=74.733, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.973e-04, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:09:39,763 (trainer:732) INFO: 14epoch:train:2575-2717batch: iter_time=3.097e-04, forward_time=0.128, loss_att=141.619, acc=0.843, loss=141.619, backward_time=0.190, grad_norm=73.475, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:11:47,120 (trainer:732) INFO: 14epoch:train:2718-2860batch: iter_time=2.891e-04, forward_time=0.130, loss_att=146.200, acc=0.843, loss=146.200, backward_time=0.193, grad_norm=78.805, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.554 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:20:40,033 (trainer:338) INFO: 14epoch results: [train] iter_time=4.360e-04, forward_time=0.128, loss_att=140.571, acc=0.845, loss=140.571, backward_time=0.191, grad_norm=79.767, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=9.708e-04, train_time=3.860, time=46 minutes and 19.93 seconds, total_count=40306, gpu_max_cached_mem_GB=23.359, [valid] loss_att=59.505, acc=0.892, cer=0.132, wer=0.340, loss=59.505, time=4 minutes and 7.26 seconds, total_count=1232, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 26.6 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:20:44,038 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:20:44,044 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/4epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:20:44,045 (trainer:272) INFO: 15/60epoch started. Estimated time to finish: 1 day, 12 hours and 45 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:25:18,283 (trainer:732) INFO: 15epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=131.787, acc=0.855, loss=131.787, backward_time=0.189, grad_norm=78.394, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=7.763 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:27:23,815 (trainer:732) INFO: 15epoch:train:144-286batch: iter_time=3.026e-04, forward_time=0.126, loss_att=125.955, acc=0.852, loss=125.955, backward_time=0.187, grad_norm=72.659, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.516 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:29:37,133 (trainer:732) INFO: 15epoch:train:287-429batch: iter_time=3.094e-04, forward_time=0.129, loss_att=145.863, acc=0.846, loss=145.863, backward_time=0.193, grad_norm=79.661, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.727 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:31:51,158 (trainer:732) INFO: 15epoch:train:430-572batch: iter_time=3.067e-04, forward_time=0.130, loss_att=149.209, acc=0.847, loss=149.209, backward_time=0.194, grad_norm=83.967, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.739 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:34:02,501 (trainer:732) INFO: 15epoch:train:573-715batch: iter_time=2.936e-04, forward_time=0.126, loss_att=123.880, acc=0.856, loss=123.880, backward_time=0.188, grad_norm=74.211, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.682 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:36:08,741 (trainer:732) INFO: 15epoch:train:716-858batch: iter_time=3.060e-04, forward_time=0.129, loss_att=136.148, acc=0.851, loss=136.148, backward_time=0.191, grad_norm=76.356, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.534 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:38:21,462 (trainer:732) INFO: 15epoch:train:859-1001batch: iter_time=3.448e-04, forward_time=0.127, loss_att=133.351, acc=0.851, loss=133.351, backward_time=0.189, grad_norm=80.423, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:40:34,854 (trainer:732) INFO: 15epoch:train:1002-1144batch: iter_time=3.129e-04, forward_time=0.129, loss_att=137.671, acc=0.849, loss=137.671, backward_time=0.191, grad_norm=77.721, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:42:46,192 (trainer:732) INFO: 15epoch:train:1145-1287batch: iter_time=3.057e-04, forward_time=0.128, loss_att=137.780, acc=0.847, loss=137.780, backward_time=0.191, grad_norm=72.248, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:44:55,130 (trainer:732) INFO: 15epoch:train:1288-1430batch: iter_time=3.146e-04, forward_time=0.129, loss_att=141.806, acc=0.848, loss=141.806, backward_time=0.192, grad_norm=81.892, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=3.576 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:47:08,107 (trainer:732) INFO: 15epoch:train:1431-1573batch: iter_time=2.898e-04, forward_time=0.129, loss_att=142.181, acc=0.847, loss=142.181, backward_time=0.191, grad_norm=76.878, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:49:20,861 (trainer:732) INFO: 15epoch:train:1574-1716batch: iter_time=3.040e-04, forward_time=0.127, loss_att=133.413, acc=0.854, loss=133.413, backward_time=0.189, grad_norm=82.703, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.704 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:51:27,352 (trainer:732) INFO: 15epoch:train:1717-1859batch: iter_time=3.013e-04, forward_time=0.129, loss_att=139.789, acc=0.847, loss=139.789, backward_time=0.192, grad_norm=79.125, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.543 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:53:40,995 (trainer:732) INFO: 15epoch:train:1860-2002batch: iter_time=2.842e-04, forward_time=0.128, loss_att=136.882, acc=0.849, loss=136.882, backward_time=0.191, grad_norm=79.954, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.739 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:55:54,042 (trainer:732) INFO: 15epoch:train:2003-2145batch: iter_time=3.115e-04, forward_time=0.130, loss_att=147.249, acc=0.846, loss=147.249, backward_time=0.193, grad_norm=76.771, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 04:58:05,574 (trainer:732) INFO: 15epoch:train:2146-2288batch: iter_time=2.977e-04, forward_time=0.130, loss_att=131.635, acc=0.856, loss=131.635, backward_time=0.194, grad_norm=78.206, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.669 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:00:12,000 (trainer:732) INFO: 15epoch:train:2289-2431batch: iter_time=2.883e-04, forward_time=0.127, loss_att=125.354, acc=0.854, loss=125.354, backward_time=0.188, grad_norm=82.419, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.543 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:02:24,453 (trainer:732) INFO: 15epoch:train:2432-2574batch: iter_time=3.069e-04, forward_time=0.128, loss_att=138.710, acc=0.851, loss=138.710, backward_time=0.191, grad_norm=78.957, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:04:36,393 (trainer:732) INFO: 15epoch:train:2575-2717batch: iter_time=3.108e-04, forward_time=0.126, loss_att=131.965, acc=0.852, loss=131.965, backward_time=0.188, grad_norm=71.026, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:06:44,001 (trainer:732) INFO: 15epoch:train:2718-2860batch: iter_time=2.800e-04, forward_time=0.128, loss_att=135.833, acc=0.851, loss=135.833, backward_time=0.191, grad_norm=81.734, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.560 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:15:36,693 (trainer:338) INFO: 15epoch results: [train] iter_time=4.036e-04, forward_time=0.128, loss_att=136.036, acc=0.851, loss=136.036, backward_time=0.191, grad_norm=78.290, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.859, time=46 minutes and 18.95 seconds, total_count=43185, gpu_max_cached_mem_GB=23.359, [valid] loss_att=55.454, acc=0.899, cer=0.123, wer=0.331, loss=55.454, time=4 minutes and 4.97 seconds, total_count=1320, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 28.73 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:15:40,821 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:15:40,828 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/5epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:15:40,828 (trainer:272) INFO: 16/60epoch started. Estimated time to finish: 1 day, 12 hours and 18 minutes + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<64350> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<64424> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<20195> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<40922> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 151) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 151) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 151) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 151) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<49082> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 151) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 151) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<49093> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 150) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 151) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 151) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 151) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<20577> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 153) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<20825> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 153) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 154) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 154) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 154) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 154) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 154) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 154) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:20:16,409 (trainer:732) INFO: 16epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=135.899, acc=0.855, loss=135.899, backward_time=0.193, grad_norm=76.734, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=7.805 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:22:30,515 (trainer:732) INFO: 16epoch:train:144-286batch: iter_time=3.063e-04, forward_time=0.129, loss_att=141.488, acc=0.850, loss=141.488, backward_time=0.193, grad_norm=82.128, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.750 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<54251> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<54415> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<16664> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<43695> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<59986> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<59998> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<37224> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 141) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<37722> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 141) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 141) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 141) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 141) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 141) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<40188> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<40274> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:24:36,300 (trainer:732) INFO: 16epoch:train:287-429batch: iter_time=2.915e-04, forward_time=0.128, loss_att=127.164, acc=0.855, loss=127.164, backward_time=0.190, grad_norm=77.541, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.520 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<52782> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<29070> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<44659> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<44647> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<22107> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<22203> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:26:49,234 (trainer:732) INFO: 16epoch:train:430-572batch: iter_time=3.130e-04, forward_time=0.128, loss_att=132.981, acc=0.857, loss=132.981, backward_time=0.191, grad_norm=74.527, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.706 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<50753> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<50879> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<64259> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<41754> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<59222> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<59208> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 1, fd 139) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 141) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 141) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<38022> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 142) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 142) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 142) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.13.248<26814> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 142) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 4, fd 143) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 4, fd 143) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 4, fd 143) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 4, fd 143) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 4, fd 143) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:29:01,594 (trainer:732) INFO: 16epoch:train:573-715batch: iter_time=3.047e-04, forward_time=0.127, loss_att=127.114, acc=0.859, loss=127.114, backward_time=0.189, grad_norm=82.817, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:31:07,303 (trainer:732) INFO: 16epoch:train:716-858batch: iter_time=3.035e-04, forward_time=0.127, loss_att=119.225, acc=0.863, loss=119.225, backward_time=0.189, grad_norm=81.670, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.517 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:33:20,472 (trainer:732) INFO: 16epoch:train:859-1001batch: iter_time=3.089e-04, forward_time=0.129, loss_att=137.540, acc=0.853, loss=137.540, backward_time=0.193, grad_norm=76.835, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:35:32,088 (trainer:732) INFO: 16epoch:train:1002-1144batch: iter_time=2.917e-04, forward_time=0.127, loss_att=118.602, acc=0.862, loss=118.602, backward_time=0.188, grad_norm=81.425, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.672 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:37:45,496 (trainer:732) INFO: 16epoch:train:1145-1287batch: iter_time=3.033e-04, forward_time=0.129, loss_att=143.278, acc=0.850, loss=143.278, backward_time=0.192, grad_norm=76.961, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.738 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:39:50,838 (trainer:732) INFO: 16epoch:train:1288-1430batch: iter_time=2.989e-04, forward_time=0.126, loss_att=123.576, acc=0.858, loss=123.576, backward_time=0.188, grad_norm=74.408, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.510 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:42:02,842 (trainer:732) INFO: 16epoch:train:1431-1573batch: iter_time=3.449e-04, forward_time=0.128, loss_att=129.961, acc=0.857, loss=129.961, backward_time=0.190, grad_norm=78.776, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:44:16,969 (trainer:732) INFO: 16epoch:train:1574-1716batch: iter_time=3.082e-04, forward_time=0.129, loss_att=141.813, acc=0.852, loss=141.813, backward_time=0.193, grad_norm=79.156, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:46:24,786 (trainer:732) INFO: 16epoch:train:1717-1859batch: iter_time=3.179e-04, forward_time=0.130, loss_att=143.891, acc=0.851, loss=143.891, backward_time=0.194, grad_norm=76.552, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.584 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:48:35,830 (trainer:732) INFO: 16epoch:train:1860-2002batch: iter_time=2.912e-04, forward_time=0.126, loss_att=122.110, acc=0.860, loss=122.110, backward_time=0.187, grad_norm=69.746, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.664 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:50:48,256 (trainer:732) INFO: 16epoch:train:2003-2145batch: iter_time=2.937e-04, forward_time=0.127, loss_att=127.472, acc=0.855, loss=127.472, backward_time=0.189, grad_norm=81.270, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.703 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:53:00,830 (trainer:732) INFO: 16epoch:train:2146-2288batch: iter_time=3.199e-04, forward_time=0.127, loss_att=129.498, acc=0.854, loss=129.498, backward_time=0.189, grad_norm=78.866, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:55:07,399 (trainer:732) INFO: 16epoch:train:2289-2431batch: iter_time=2.921e-04, forward_time=0.129, loss_att=135.795, acc=0.855, loss=135.795, backward_time=0.192, grad_norm=77.960, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.548 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:57:20,547 (trainer:732) INFO: 16epoch:train:2432-2574batch: iter_time=2.963e-04, forward_time=0.129, loss_att=137.697, acc=0.854, loss=137.697, backward_time=0.192, grad_norm=77.073, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 05:59:32,969 (trainer:732) INFO: 16epoch:train:2575-2717batch: iter_time=3.085e-04, forward_time=0.127, loss_att=128.954, acc=0.857, loss=128.954, backward_time=0.190, grad_norm=86.661, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:01:40,524 (trainer:732) INFO: 16epoch:train:2718-2860batch: iter_time=2.905e-04, forward_time=0.129, loss_att=132.302, acc=0.857, loss=132.302, backward_time=0.192, grad_norm=86.228, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.560 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:10:29,399 (trainer:338) INFO: 16epoch results: [train] iter_time=4.546e-04, forward_time=0.128, loss_att=131.632, acc=0.856, loss=131.632, backward_time=0.191, grad_norm=79.144, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.859, time=46 minutes and 18.94 seconds, total_count=46064, gpu_max_cached_mem_GB=23.359, [valid] loss_att=52.763, acc=0.904, cer=0.114, wer=0.325, loss=52.763, time=4 minutes and 0.48 seconds, total_count=1408, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 29.14 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:10:33,528 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:10:33,555 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/6epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:10:33,555 (trainer:272) INFO: 17/60epoch started. Estimated time to finish: 1 day, 11 hours and 48 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:15:06,624 (trainer:732) INFO: 17epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=120.597, acc=0.865, loss=120.597, backward_time=0.190, grad_norm=80.219, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=7.731 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:17:20,785 (trainer:732) INFO: 17epoch:train:144-286batch: iter_time=2.752e-04, forward_time=0.128, loss_att=133.771, acc=0.864, loss=133.771, backward_time=0.192, grad_norm=83.503, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.753 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:19:27,728 (trainer:732) INFO: 17epoch:train:287-429batch: iter_time=2.751e-04, forward_time=0.128, loss_att=130.202, acc=0.861, loss=130.202, backward_time=0.191, grad_norm=80.881, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.552 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:21:41,409 (trainer:732) INFO: 17epoch:train:430-572batch: iter_time=3.049e-04, forward_time=0.129, loss_att=130.839, acc=0.858, loss=130.839, backward_time=0.191, grad_norm=73.086, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.728 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:23:53,856 (trainer:732) INFO: 17epoch:train:573-715batch: iter_time=2.987e-04, forward_time=0.129, loss_att=127.662, acc=0.862, loss=127.662, backward_time=0.191, grad_norm=81.894, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:25:59,600 (trainer:732) INFO: 17epoch:train:716-858batch: iter_time=2.924e-04, forward_time=0.127, loss_att=116.448, acc=0.866, loss=116.448, backward_time=0.188, grad_norm=72.160, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.514 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:28:11,337 (trainer:732) INFO: 17epoch:train:859-1001batch: iter_time=3.144e-04, forward_time=0.127, loss_att=121.071, acc=0.863, loss=121.071, backward_time=0.189, grad_norm=77.969, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:30:25,806 (trainer:732) INFO: 17epoch:train:1002-1144batch: iter_time=2.855e-04, forward_time=0.130, loss_att=142.297, acc=0.853, loss=142.297, backward_time=0.194, grad_norm=81.354, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.752 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:32:37,925 (trainer:732) INFO: 17epoch:train:1145-1287batch: iter_time=2.822e-04, forward_time=0.128, loss_att=127.321, acc=0.862, loss=127.321, backward_time=0.191, grad_norm=73.877, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.703 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:34:44,392 (trainer:732) INFO: 17epoch:train:1288-1430batch: iter_time=2.831e-04, forward_time=0.128, loss_att=132.686, acc=0.858, loss=132.686, backward_time=0.191, grad_norm=73.187, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.541 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:36:55,933 (trainer:732) INFO: 17epoch:train:1431-1573batch: iter_time=2.789e-04, forward_time=0.127, loss_att=118.246, acc=0.864, loss=118.246, backward_time=0.188, grad_norm=73.216, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.678 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:39:09,722 (trainer:732) INFO: 17epoch:train:1574-1716batch: iter_time=2.930e-04, forward_time=0.130, loss_att=132.278, acc=0.860, loss=132.278, backward_time=0.194, grad_norm=84.353, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.732 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:41:15,950 (trainer:732) INFO: 17epoch:train:1717-1859batch: iter_time=2.905e-04, forward_time=0.128, loss_att=125.630, acc=0.865, loss=125.630, backward_time=0.191, grad_norm=76.474, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.534 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:43:27,590 (trainer:732) INFO: 17epoch:train:1860-2002batch: iter_time=2.853e-04, forward_time=0.127, loss_att=110.530, acc=0.871, loss=110.530, backward_time=0.189, grad_norm=71.839, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.689 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:45:39,456 (trainer:732) INFO: 17epoch:train:2003-2145batch: iter_time=2.969e-04, forward_time=0.128, loss_att=121.505, acc=0.865, loss=121.505, backward_time=0.191, grad_norm=81.893, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.685 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:47:52,296 (trainer:732) INFO: 17epoch:train:2146-2288batch: iter_time=3.094e-04, forward_time=0.128, loss_att=126.290, acc=0.865, loss=126.290, backward_time=0.192, grad_norm=83.280, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:49:58,057 (trainer:732) INFO: 17epoch:train:2289-2431batch: iter_time=2.859e-04, forward_time=0.128, loss_att=132.461, acc=0.857, loss=132.461, backward_time=0.191, grad_norm=72.668, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.526 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:52:10,036 (trainer:732) INFO: 17epoch:train:2432-2574batch: iter_time=2.963e-04, forward_time=0.127, loss_att=119.672, acc=0.866, loss=119.672, backward_time=0.189, grad_norm=75.578, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:54:22,952 (trainer:732) INFO: 17epoch:train:2575-2717batch: iter_time=2.937e-04, forward_time=0.128, loss_att=123.428, acc=0.865, loss=123.428, backward_time=0.191, grad_norm=75.698, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 06:56:31,817 (trainer:732) INFO: 17epoch:train:2718-2860batch: iter_time=2.773e-04, forward_time=0.128, loss_att=136.024, acc=0.859, loss=136.024, backward_time=0.192, grad_norm=85.943, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.597 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:05:16,559 (trainer:338) INFO: 17epoch results: [train] iter_time=4.204e-04, forward_time=0.128, loss_att=126.080, acc=0.862, loss=126.080, backward_time=0.191, grad_norm=77.910, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.856, time=46 minutes and 17.05 seconds, total_count=48943, gpu_max_cached_mem_GB=23.359, [valid] loss_att=46.766, acc=0.916, cer=0.102, wer=0.303, loss=46.766, time=3 minutes and 56.88 seconds, total_count=1496, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 29.07 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:05:20,704 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:05:20,710 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/7epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:05:20,711 (trainer:272) INFO: 18/60epoch started. Estimated time to finish: 1 day, 11 hours and 14 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:09:54,651 (trainer:732) INFO: 18epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=126.274, acc=0.866, loss=126.274, backward_time=0.191, grad_norm=77.353, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=7.760 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:12:05,959 (trainer:732) INFO: 18epoch:train:144-286batch: iter_time=2.922e-04, forward_time=0.127, loss_att=112.348, acc=0.872, loss=112.348, backward_time=0.190, grad_norm=81.031, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.670 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:14:12,788 (trainer:732) INFO: 18epoch:train:287-429batch: iter_time=3.038e-04, forward_time=0.128, loss_att=117.838, acc=0.871, loss=117.838, backward_time=0.190, grad_norm=78.415, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.547 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:16:25,244 (trainer:732) INFO: 18epoch:train:430-572batch: iter_time=2.926e-04, forward_time=0.128, loss_att=117.446, acc=0.870, loss=117.446, backward_time=0.190, grad_norm=72.609, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:18:39,249 (trainer:732) INFO: 18epoch:train:573-715batch: iter_time=2.988e-04, forward_time=0.129, loss_att=125.543, acc=0.868, loss=125.543, backward_time=0.193, grad_norm=79.428, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.756 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:20:46,381 (trainer:732) INFO: 18epoch:train:716-858batch: iter_time=2.869e-04, forward_time=0.127, loss_att=121.442, acc=0.867, loss=121.442, backward_time=0.189, grad_norm=77.679, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.563 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:22:58,934 (trainer:732) INFO: 18epoch:train:859-1001batch: iter_time=3.173e-04, forward_time=0.129, loss_att=122.602, acc=0.869, loss=122.602, backward_time=0.193, grad_norm=79.265, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:25:12,307 (trainer:732) INFO: 18epoch:train:1002-1144batch: iter_time=2.964e-04, forward_time=0.130, loss_att=126.487, acc=0.866, loss=126.487, backward_time=0.194, grad_norm=84.041, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.722 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:27:23,668 (trainer:732) INFO: 18epoch:train:1145-1287batch: iter_time=2.983e-04, forward_time=0.127, loss_att=112.163, acc=0.874, loss=112.163, backward_time=0.188, grad_norm=69.243, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=3.684 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:29:30,909 (trainer:732) INFO: 18epoch:train:1288-1430batch: iter_time=2.924e-04, forward_time=0.128, loss_att=122.938, acc=0.867, loss=122.938, backward_time=0.191, grad_norm=80.023, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.558 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:31:43,328 (trainer:732) INFO: 18epoch:train:1431-1573batch: iter_time=2.850e-04, forward_time=0.128, loss_att=114.490, acc=0.873, loss=114.490, backward_time=0.190, grad_norm=84.142, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:33:56,326 (trainer:732) INFO: 18epoch:train:1574-1716batch: iter_time=2.898e-04, forward_time=0.128, loss_att=126.272, acc=0.866, loss=126.272, backward_time=0.191, grad_norm=79.215, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:36:09,063 (trainer:732) INFO: 18epoch:train:1717-1859batch: iter_time=2.977e-04, forward_time=0.129, loss_att=120.907, acc=0.870, loss=120.907, backward_time=0.191, grad_norm=81.781, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:38:15,135 (trainer:732) INFO: 18epoch:train:1860-2002batch: iter_time=3.028e-04, forward_time=0.128, loss_att=113.684, acc=0.876, loss=113.684, backward_time=0.191, grad_norm=75.570, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.531 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:40:27,212 (trainer:732) INFO: 18epoch:train:2003-2145batch: iter_time=3.026e-04, forward_time=0.128, loss_att=118.445, acc=0.869, loss=118.445, backward_time=0.191, grad_norm=71.519, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:42:41,951 (trainer:732) INFO: 18epoch:train:2146-2288batch: iter_time=3.148e-04, forward_time=0.129, loss_att=128.485, acc=0.866, loss=128.485, backward_time=0.193, grad_norm=84.020, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.759 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:44:47,010 (trainer:732) INFO: 18epoch:train:2289-2431batch: iter_time=2.919e-04, forward_time=0.127, loss_att=115.366, acc=0.871, loss=115.366, backward_time=0.188, grad_norm=70.092, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.001, train_time=3.503 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:46:59,019 (trainer:732) INFO: 18epoch:train:2432-2574batch: iter_time=3.066e-04, forward_time=0.127, loss_att=118.697, acc=0.870, loss=118.697, backward_time=0.189, grad_norm=79.352, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:49:12,996 (trainer:732) INFO: 18epoch:train:2575-2717batch: iter_time=3.135e-04, forward_time=0.129, loss_att=119.309, acc=0.872, loss=119.309, backward_time=0.192, grad_norm=78.437, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 07:51:26,526 (trainer:732) INFO: 18epoch:train:2718-2860batch: iter_time=3.304e-04, forward_time=0.127, loss_att=115.358, acc=0.872, loss=115.358, backward_time=0.189, grad_norm=75.649, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:00:13,396 (trainer:338) INFO: 18epoch results: [train] iter_time=4.306e-04, forward_time=0.128, loss_att=119.729, acc=0.870, loss=119.729, backward_time=0.191, grad_norm=77.942, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.867, time=46 minutes and 23.62 seconds, total_count=51822, gpu_max_cached_mem_GB=23.359, [valid] loss_att=43.617, acc=0.922, cer=0.097, wer=0.290, loss=43.617, time=4 minutes and 1.47 seconds, total_count=1584, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.59 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:00:17,304 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:00:17,311 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/8epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:00:17,312 (trainer:272) INFO: 19/60epoch started. Estimated time to finish: 1 day, 10 hours and 38 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:04:54,062 (trainer:732) INFO: 19epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=119.445, acc=0.875, loss=119.445, backward_time=0.190, grad_norm=76.616, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=7.845 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:07:05,346 (trainer:732) INFO: 19epoch:train:144-286batch: iter_time=3.067e-04, forward_time=0.126, loss_att=101.021, acc=0.883, loss=101.021, backward_time=0.187, grad_norm=68.391, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.670 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:09:10,490 (trainer:732) INFO: 19epoch:train:287-429batch: iter_time=2.922e-04, forward_time=0.126, loss_att=112.749, acc=0.875, loss=112.749, backward_time=0.188, grad_norm=77.717, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.498 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:11:23,829 (trainer:732) INFO: 19epoch:train:430-572batch: iter_time=3.091e-04, forward_time=0.128, loss_att=117.869, acc=0.876, loss=117.869, backward_time=0.190, grad_norm=79.794, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:13:35,839 (trainer:732) INFO: 19epoch:train:573-715batch: iter_time=3.043e-04, forward_time=0.130, loss_att=120.364, acc=0.871, loss=120.364, backward_time=0.193, grad_norm=76.345, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:15:50,019 (trainer:732) INFO: 19epoch:train:716-858batch: iter_time=3.052e-04, forward_time=0.129, loss_att=119.934, acc=0.876, loss=119.934, backward_time=0.193, grad_norm=82.921, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.758 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:17:57,979 (trainer:732) INFO: 19epoch:train:859-1001batch: iter_time=3.141e-04, forward_time=0.129, loss_att=111.522, acc=0.880, loss=111.522, backward_time=0.192, grad_norm=76.470, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.579 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:20:10,914 (trainer:732) INFO: 19epoch:train:1002-1144batch: iter_time=3.132e-04, forward_time=0.127, loss_att=112.372, acc=0.875, loss=112.372, backward_time=0.189, grad_norm=78.058, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:22:24,267 (trainer:732) INFO: 19epoch:train:1145-1287batch: iter_time=2.961e-04, forward_time=0.129, loss_att=115.706, acc=0.876, loss=115.706, backward_time=0.192, grad_norm=80.546, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:24:30,350 (trainer:732) INFO: 19epoch:train:1288-1430batch: iter_time=3.190e-04, forward_time=0.128, loss_att=117.851, acc=0.871, loss=117.851, backward_time=0.190, grad_norm=67.547, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.530 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:26:42,392 (trainer:732) INFO: 19epoch:train:1431-1573batch: iter_time=3.012e-04, forward_time=0.130, loss_att=117.086, acc=0.874, loss=117.086, backward_time=0.193, grad_norm=76.788, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:28:56,995 (trainer:732) INFO: 19epoch:train:1574-1716batch: iter_time=3.152e-04, forward_time=0.130, loss_att=117.749, acc=0.876, loss=117.749, backward_time=0.193, grad_norm=78.049, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.752 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:31:07,361 (trainer:732) INFO: 19epoch:train:1717-1859batch: iter_time=3.051e-04, forward_time=0.128, loss_att=105.993, acc=0.880, loss=105.993, backward_time=0.189, grad_norm=73.577, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.657 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:33:13,479 (trainer:732) INFO: 19epoch:train:1860-2002batch: iter_time=3.547e-04, forward_time=0.128, loss_att=113.244, acc=0.876, loss=113.244, backward_time=0.189, grad_norm=76.273, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.527 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:35:24,965 (trainer:732) INFO: 19epoch:train:2003-2145batch: iter_time=3.072e-04, forward_time=0.127, loss_att=104.481, acc=0.882, loss=104.481, backward_time=0.188, grad_norm=75.216, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.677 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:37:36,891 (trainer:732) INFO: 19epoch:train:2146-2288batch: iter_time=3.031e-04, forward_time=0.128, loss_att=117.324, acc=0.874, loss=117.324, backward_time=0.190, grad_norm=76.796, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.681 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:39:43,590 (trainer:732) INFO: 19epoch:train:2289-2431batch: iter_time=3.130e-04, forward_time=0.130, loss_att=118.447, acc=0.876, loss=118.447, backward_time=0.194, grad_norm=82.765, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.556 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:41:56,154 (trainer:732) INFO: 19epoch:train:2432-2574batch: iter_time=3.032e-04, forward_time=0.128, loss_att=116.177, acc=0.877, loss=116.177, backward_time=0.191, grad_norm=78.762, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:44:09,053 (trainer:732) INFO: 19epoch:train:2575-2717batch: iter_time=3.080e-04, forward_time=0.128, loss_att=106.090, acc=0.883, loss=106.090, backward_time=0.190, grad_norm=77.616, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:46:22,255 (trainer:732) INFO: 19epoch:train:2718-2860batch: iter_time=2.922e-04, forward_time=0.129, loss_att=114.254, acc=0.877, loss=114.254, backward_time=0.193, grad_norm=82.940, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:55:08,455 (trainer:338) INFO: 19epoch results: [train] iter_time=4.126e-04, forward_time=0.128, loss_att=113.881, acc=0.877, loss=113.881, backward_time=0.191, grad_norm=77.178, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.866, time=46 minutes and 24.39 seconds, total_count=54701, gpu_max_cached_mem_GB=23.359, [valid] loss_att=40.272, acc=0.928, cer=0.089, wer=0.277, loss=40.272, time=3 minutes and 58.85 seconds, total_count=1672, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.9 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:55:12,502 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:55:12,509 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/9epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:55:12,510 (trainer:272) INFO: 20/60epoch started. Estimated time to finish: 1 day, 10 hours and 53.88 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 08:59:49,011 (trainer:732) INFO: 20epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=112.712, acc=0.884, loss=112.712, backward_time=0.193, grad_norm=82.052, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=7.830 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:02:01,938 (trainer:732) INFO: 20epoch:train:144-286batch: iter_time=3.199e-04, forward_time=0.129, loss_att=112.434, acc=0.880, loss=112.434, backward_time=0.192, grad_norm=77.020, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:04:08,857 (trainer:732) INFO: 20epoch:train:287-429batch: iter_time=3.167e-04, forward_time=0.128, loss_att=105.121, acc=0.886, loss=105.121, backward_time=0.191, grad_norm=77.849, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.547 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:06:21,463 (trainer:732) INFO: 20epoch:train:430-572batch: iter_time=3.257e-04, forward_time=0.128, loss_att=112.710, acc=0.879, loss=112.710, backward_time=0.190, grad_norm=81.082, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:08:34,045 (trainer:732) INFO: 20epoch:train:573-715batch: iter_time=3.116e-04, forward_time=0.128, loss_att=111.531, acc=0.881, loss=111.531, backward_time=0.191, grad_norm=81.062, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:10:45,039 (trainer:732) INFO: 20epoch:train:716-858batch: iter_time=3.136e-04, forward_time=0.128, loss_att=95.736, acc=0.891, loss=95.736, backward_time=0.189, grad_norm=74.238, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.666 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:12:49,844 (trainer:732) INFO: 20epoch:train:859-1001batch: iter_time=3.666e-04, forward_time=0.127, loss_att=102.460, acc=0.884, loss=102.460, backward_time=0.188, grad_norm=73.239, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.490 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:15:01,783 (trainer:732) INFO: 20epoch:train:1002-1144batch: iter_time=3.126e-04, forward_time=0.127, loss_att=101.376, acc=0.887, loss=101.376, backward_time=0.189, grad_norm=78.712, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.679 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:17:14,398 (trainer:732) INFO: 20epoch:train:1145-1287batch: iter_time=3.239e-04, forward_time=0.128, loss_att=109.886, acc=0.881, loss=109.886, backward_time=0.190, grad_norm=72.299, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:19:21,208 (trainer:732) INFO: 20epoch:train:1288-1430batch: iter_time=3.248e-04, forward_time=0.128, loss_att=105.955, acc=0.882, loss=105.955, backward_time=0.190, grad_norm=75.505, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.564 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:21:33,892 (trainer:732) INFO: 20epoch:train:1431-1573batch: iter_time=2.987e-04, forward_time=0.129, loss_att=115.975, acc=0.882, loss=115.975, backward_time=0.193, grad_norm=80.209, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:23:46,723 (trainer:732) INFO: 20epoch:train:1574-1716batch: iter_time=2.999e-04, forward_time=0.128, loss_att=107.361, acc=0.883, loss=107.361, backward_time=0.191, grad_norm=76.832, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:25:59,216 (trainer:732) INFO: 20epoch:train:1717-1859batch: iter_time=3.190e-04, forward_time=0.129, loss_att=113.339, acc=0.882, loss=113.339, backward_time=0.192, grad_norm=76.958, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:28:07,266 (trainer:732) INFO: 20epoch:train:1860-2002batch: iter_time=2.936e-04, forward_time=0.130, loss_att=116.050, acc=0.883, loss=116.050, backward_time=0.195, grad_norm=82.939, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.582 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:30:20,571 (trainer:732) INFO: 20epoch:train:2003-2145batch: iter_time=3.276e-04, forward_time=0.129, loss_att=111.270, acc=0.885, loss=111.270, backward_time=0.192, grad_norm=77.815, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.731 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:32:33,395 (trainer:732) INFO: 20epoch:train:2146-2288batch: iter_time=3.195e-04, forward_time=0.128, loss_att=107.324, acc=0.883, loss=107.324, backward_time=0.191, grad_norm=79.837, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:34:40,577 (trainer:732) INFO: 20epoch:train:2289-2431batch: iter_time=3.062e-04, forward_time=0.128, loss_att=104.560, acc=0.885, loss=104.560, backward_time=0.190, grad_norm=74.646, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.597 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:36:50,407 (trainer:732) INFO: 20epoch:train:2432-2574batch: iter_time=3.268e-04, forward_time=0.127, loss_att=102.956, acc=0.885, loss=102.956, backward_time=0.188, grad_norm=75.148, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.598 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:39:03,322 (trainer:732) INFO: 20epoch:train:2575-2717batch: iter_time=3.192e-04, forward_time=0.129, loss_att=108.398, acc=0.881, loss=108.398, backward_time=0.191, grad_norm=85.645, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:41:15,865 (trainer:732) INFO: 20epoch:train:2718-2860batch: iter_time=2.970e-04, forward_time=0.128, loss_att=106.041, acc=0.885, loss=106.041, backward_time=0.190, grad_norm=79.401, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:50:01,602 (trainer:338) INFO: 20epoch results: [train] iter_time=4.567e-04, forward_time=0.128, loss_att=107.950, acc=0.883, loss=107.950, backward_time=0.191, grad_norm=78.060, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.863, time=46 minutes and 22.33 seconds, total_count=57580, gpu_max_cached_mem_GB=23.359, [valid] loss_att=36.672, acc=0.934, cer=0.079, wer=0.258, loss=36.672, time=4 minutes and 1 second, total_count=1760, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.76 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:50:05,404 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:50:05,412 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/10epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:50:05,412 (trainer:272) INFO: 21/60epoch started. Estimated time to finish: 1 day, 9 hours and 21 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:54:40,011 (trainer:732) INFO: 21epoch:train:1-143batch: iter_time=0.003, forward_time=0.129, loss_att=103.670, acc=0.889, loss=103.670, backward_time=0.191, grad_norm=77.593, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=7.774 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:56:52,968 (trainer:732) INFO: 21epoch:train:144-286batch: iter_time=2.985e-04, forward_time=0.130, loss_att=105.885, acc=0.886, loss=105.885, backward_time=0.192, grad_norm=76.509, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.723 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 09:59:06,366 (trainer:732) INFO: 21epoch:train:287-429batch: iter_time=3.226e-04, forward_time=0.129, loss_att=108.496, acc=0.886, loss=108.496, backward_time=0.192, grad_norm=81.539, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.729 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:01:12,691 (trainer:732) INFO: 21epoch:train:430-572batch: iter_time=3.432e-04, forward_time=0.127, loss_att=92.265, acc=0.895, loss=92.265, backward_time=0.188, grad_norm=78.545, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.523 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:03:24,376 (trainer:732) INFO: 21epoch:train:573-715batch: iter_time=3.226e-04, forward_time=0.127, loss_att=100.147, acc=0.889, loss=100.147, backward_time=0.188, grad_norm=75.300, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:05:35,277 (trainer:732) INFO: 21epoch:train:716-858batch: iter_time=3.116e-04, forward_time=0.127, loss_att=93.131, acc=0.896, loss=93.131, backward_time=0.188, grad_norm=72.855, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.657 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:07:42,806 (trainer:732) INFO: 21epoch:train:859-1001batch: iter_time=3.252e-04, forward_time=0.129, loss_att=110.603, acc=0.884, loss=110.603, backward_time=0.192, grad_norm=80.062, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.566 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:09:55,277 (trainer:732) INFO: 21epoch:train:1002-1144batch: iter_time=2.939e-04, forward_time=0.128, loss_att=102.169, acc=0.888, loss=102.169, backward_time=0.190, grad_norm=74.758, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:12:05,765 (trainer:732) INFO: 21epoch:train:1145-1287batch: iter_time=3.093e-04, forward_time=0.129, loss_att=101.098, acc=0.889, loss=101.098, backward_time=0.192, grad_norm=75.342, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.660 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:14:17,891 (trainer:732) INFO: 21epoch:train:1288-1430batch: iter_time=3.450e-04, forward_time=0.128, loss_att=98.360, acc=0.894, loss=98.360, backward_time=0.190, grad_norm=76.964, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:16:25,505 (trainer:732) INFO: 21epoch:train:1431-1573batch: iter_time=3.216e-04, forward_time=0.129, loss_att=107.121, acc=0.887, loss=107.121, backward_time=0.192, grad_norm=82.415, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.574 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:18:39,501 (trainer:732) INFO: 21epoch:train:1574-1716batch: iter_time=3.074e-04, forward_time=0.129, loss_att=105.771, acc=0.889, loss=105.771, backward_time=0.192, grad_norm=83.075, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.737 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:20:50,992 (trainer:732) INFO: 21epoch:train:1717-1859batch: iter_time=3.092e-04, forward_time=0.129, loss_att=101.642, acc=0.891, loss=101.642, backward_time=0.191, grad_norm=77.174, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:22:58,465 (trainer:732) INFO: 21epoch:train:1860-2002batch: iter_time=2.982e-04, forward_time=0.129, loss_att=108.021, acc=0.888, loss=108.021, backward_time=0.192, grad_norm=84.112, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.562 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:25:10,499 (trainer:732) INFO: 21epoch:train:2003-2145batch: iter_time=3.129e-04, forward_time=0.129, loss_att=108.837, acc=0.885, loss=108.837, backward_time=0.193, grad_norm=79.490, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:27:23,233 (trainer:732) INFO: 21epoch:train:2146-2288batch: iter_time=3.165e-04, forward_time=0.128, loss_att=103.610, acc=0.888, loss=103.610, backward_time=0.191, grad_norm=77.921, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:29:34,568 (trainer:732) INFO: 21epoch:train:2289-2431batch: iter_time=2.886e-04, forward_time=0.127, loss_att=101.654, acc=0.889, loss=101.654, backward_time=0.189, grad_norm=86.075, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:31:42,594 (trainer:732) INFO: 21epoch:train:2432-2574batch: iter_time=2.925e-04, forward_time=0.131, loss_att=113.703, acc=0.885, loss=113.703, backward_time=0.195, grad_norm=82.457, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.574 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:33:54,883 (trainer:732) INFO: 21epoch:train:2575-2717batch: iter_time=3.037e-04, forward_time=0.128, loss_att=98.502, acc=0.893, loss=98.502, backward_time=0.190, grad_norm=79.977, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.703 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:36:07,350 (trainer:732) INFO: 21epoch:train:2718-2860batch: iter_time=2.885e-04, forward_time=0.127, loss_att=90.972, acc=0.897, loss=90.972, backward_time=0.188, grad_norm=71.330, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:44:56,674 (trainer:338) INFO: 21epoch results: [train] iter_time=4.534e-04, forward_time=0.128, loss_att=102.638, acc=0.890, loss=102.638, backward_time=0.191, grad_norm=78.762, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.861, time=46 minutes and 21.3 seconds, total_count=60459, gpu_max_cached_mem_GB=23.359, [valid] loss_att=34.862, acc=0.939, cer=0.075, wer=0.245, loss=34.862, time=4 minutes and 1.25 seconds, total_count=1848, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 28.71 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:45:00,582 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:45:00,588 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/11epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:45:00,589 (trainer:272) INFO: 22/60epoch started. Estimated time to finish: 1 day, 8 hours and 40 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:49:30,375 (trainer:732) INFO: 22epoch:train:1-143batch: iter_time=0.003, forward_time=0.129, loss_att=98.950, acc=0.895, loss=98.950, backward_time=0.192, grad_norm=78.386, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=7.642 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:51:41,430 (trainer:732) INFO: 22epoch:train:144-286batch: iter_time=3.292e-04, forward_time=0.129, loss_att=95.817, acc=0.893, loss=95.817, backward_time=0.190, grad_norm=76.747, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.668 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:53:53,424 (trainer:732) INFO: 22epoch:train:287-429batch: iter_time=3.290e-04, forward_time=0.127, loss_att=91.810, acc=0.897, loss=91.810, backward_time=0.187, grad_norm=75.718, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.685 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:56:00,874 (trainer:732) INFO: 22epoch:train:430-572batch: iter_time=3.383e-04, forward_time=0.128, loss_att=96.292, acc=0.896, loss=96.292, backward_time=0.190, grad_norm=83.865, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.557 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 10:58:15,210 (trainer:732) INFO: 22epoch:train:573-715batch: iter_time=3.240e-04, forward_time=0.131, loss_att=106.707, acc=0.894, loss=106.707, backward_time=0.195, grad_norm=84.853, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.773 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:00:28,106 (trainer:732) INFO: 22epoch:train:716-858batch: iter_time=3.294e-04, forward_time=0.129, loss_att=105.751, acc=0.890, loss=105.751, backward_time=0.192, grad_norm=77.182, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:02:40,027 (trainer:732) INFO: 22epoch:train:859-1001batch: iter_time=3.382e-04, forward_time=0.128, loss_att=100.197, acc=0.895, loss=100.197, backward_time=0.191, grad_norm=81.547, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:04:47,016 (trainer:732) INFO: 22epoch:train:1002-1144batch: iter_time=3.387e-04, forward_time=0.129, loss_att=93.965, acc=0.897, loss=93.965, backward_time=0.191, grad_norm=77.157, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.543 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:07:00,015 (trainer:732) INFO: 22epoch:train:1145-1287batch: iter_time=3.265e-04, forward_time=0.130, loss_att=102.880, acc=0.892, loss=102.880, backward_time=0.193, grad_norm=81.935, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.732 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:09:13,125 (trainer:732) INFO: 22epoch:train:1288-1430batch: iter_time=3.469e-04, forward_time=0.129, loss_att=96.333, acc=0.897, loss=96.333, backward_time=0.191, grad_norm=82.718, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:11:20,085 (trainer:732) INFO: 22epoch:train:1431-1573batch: iter_time=3.344e-04, forward_time=0.129, loss_att=97.200, acc=0.895, loss=97.200, backward_time=0.190, grad_norm=77.730, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.555 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:13:32,042 (trainer:732) INFO: 22epoch:train:1574-1716batch: iter_time=3.340e-04, forward_time=0.127, loss_att=98.887, acc=0.893, loss=98.887, backward_time=0.189, grad_norm=74.490, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.681 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:15:44,321 (trainer:732) INFO: 22epoch:train:1717-1859batch: iter_time=3.204e-04, forward_time=0.129, loss_att=104.032, acc=0.892, loss=104.032, backward_time=0.192, grad_norm=78.039, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:17:56,090 (trainer:732) INFO: 22epoch:train:1860-2002batch: iter_time=3.140e-04, forward_time=0.128, loss_att=91.253, acc=0.896, loss=91.253, backward_time=0.190, grad_norm=78.793, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:20:00,764 (trainer:732) INFO: 22epoch:train:2003-2145batch: iter_time=3.278e-04, forward_time=0.127, loss_att=90.015, acc=0.897, loss=90.015, backward_time=0.188, grad_norm=75.815, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.487 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:22:12,436 (trainer:732) INFO: 22epoch:train:2146-2288batch: iter_time=3.381e-04, forward_time=0.127, loss_att=94.878, acc=0.896, loss=94.878, backward_time=0.188, grad_norm=71.979, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.673 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:24:25,246 (trainer:732) INFO: 22epoch:train:2289-2431batch: iter_time=3.219e-04, forward_time=0.129, loss_att=101.885, acc=0.896, loss=101.885, backward_time=0.192, grad_norm=80.967, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.729 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:26:31,676 (trainer:732) INFO: 22epoch:train:2432-2574batch: iter_time=3.312e-04, forward_time=0.128, loss_att=96.998, acc=0.895, loss=96.998, backward_time=0.190, grad_norm=83.692, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.535 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:28:45,600 (trainer:732) INFO: 22epoch:train:2575-2717batch: iter_time=3.341e-04, forward_time=0.130, loss_att=110.509, acc=0.889, loss=110.509, backward_time=0.195, grad_norm=81.415, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.747 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:30:58,366 (trainer:732) INFO: 22epoch:train:2718-2860batch: iter_time=3.104e-04, forward_time=0.128, loss_att=94.042, acc=0.897, loss=94.042, backward_time=0.190, grad_norm=74.900, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:39:47,554 (trainer:338) INFO: 22epoch results: [train] iter_time=4.584e-04, forward_time=0.129, loss_att=98.258, acc=0.895, loss=98.258, backward_time=0.191, grad_norm=78.909, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.855, time=46 minutes and 16.99 seconds, total_count=63338, gpu_max_cached_mem_GB=23.359, [valid] loss_att=31.777, acc=0.944, cer=0.068, wer=0.230, loss=31.777, time=4 minutes and 1.29 seconds, total_count=1936, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 28.68 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:39:52,211 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:39:52,223 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/12epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:39:52,223 (trainer:272) INFO: 23/60epoch started. Estimated time to finish: 1 day, 7 hours and 58 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:45:04,308 (trainer:732) INFO: 23epoch:train:1-143batch: iter_time=0.006, forward_time=0.133, loss_att=91.826, acc=0.900, loss=91.826, backward_time=0.191, grad_norm=76.999, clip=100.000, loss_scale=1.000, optim_step_time=0.075, optim0_lr0=0.002, train_time=8.851 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:47:17,680 (trainer:732) INFO: 23epoch:train:144-286batch: iter_time=3.518e-04, forward_time=0.130, loss_att=98.099, acc=0.895, loss=98.099, backward_time=0.193, grad_norm=79.984, clip=100.000, loss_scale=1.000, optim_step_time=0.065, optim0_lr0=0.002, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:49:32,969 (trainer:732) INFO: 23epoch:train:287-429batch: iter_time=5.100e-04, forward_time=0.143, loss_att=97.816, acc=0.899, loss=97.816, backward_time=0.196, grad_norm=84.096, clip=100.000, loss_scale=1.000, optim_step_time=0.072, optim0_lr0=0.002, train_time=3.776 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:51:39,250 (trainer:732) INFO: 23epoch:train:430-572batch: iter_time=5.499e-04, forward_time=0.151, loss_att=93.017, acc=0.898, loss=93.017, backward_time=0.199, grad_norm=74.271, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.002, train_time=3.526 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:53:53,044 (trainer:732) INFO: 23epoch:train:573-715batch: iter_time=5.853e-04, forward_time=0.150, loss_att=98.882, acc=0.898, loss=98.882, backward_time=0.201, grad_norm=81.088, clip=100.000, loss_scale=1.000, optim_step_time=0.097, optim0_lr0=0.002, train_time=3.751 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:56:05,257 (trainer:732) INFO: 23epoch:train:716-858batch: iter_time=5.711e-04, forward_time=0.149, loss_att=90.271, acc=0.902, loss=90.271, backward_time=0.198, grad_norm=74.377, clip=100.000, loss_scale=1.000, optim_step_time=0.094, optim0_lr0=0.002, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 11:58:17,719 (trainer:732) INFO: 23epoch:train:859-1001batch: iter_time=6.645e-04, forward_time=0.148, loss_att=89.186, acc=0.902, loss=89.186, backward_time=0.199, grad_norm=74.669, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.002, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:00:23,702 (trainer:732) INFO: 23epoch:train:1002-1144batch: iter_time=6.226e-04, forward_time=0.148, loss_att=94.904, acc=0.899, loss=94.904, backward_time=0.200, grad_norm=74.698, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.002, train_time=3.513 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:02:37,339 (trainer:732) INFO: 23epoch:train:1145-1287batch: iter_time=5.124e-04, forward_time=0.145, loss_att=101.245, acc=0.894, loss=101.245, backward_time=0.199, grad_norm=79.756, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.002, train_time=3.749 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:04:50,530 (trainer:732) INFO: 23epoch:train:1288-1430batch: iter_time=4.280e-04, forward_time=0.136, loss_att=98.927, acc=0.896, loss=98.927, backward_time=0.195, grad_norm=76.050, clip=100.000, loss_scale=1.000, optim_step_time=0.072, optim0_lr0=0.002, train_time=3.727 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:06:56,852 (trainer:732) INFO: 23epoch:train:1431-1573batch: iter_time=4.015e-04, forward_time=0.138, loss_att=92.338, acc=0.902, loss=92.338, backward_time=0.198, grad_norm=77.258, clip=100.000, loss_scale=1.000, optim_step_time=0.076, optim0_lr0=0.002, train_time=3.534 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:09:09,920 (trainer:732) INFO: 23epoch:train:1574-1716batch: iter_time=5.505e-04, forward_time=0.147, loss_att=98.454, acc=0.899, loss=98.454, backward_time=0.199, grad_norm=80.810, clip=100.000, loss_scale=1.000, optim_step_time=0.101, optim0_lr0=0.002, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:11:23,393 (trainer:732) INFO: 23epoch:train:1717-1859batch: iter_time=5.705e-04, forward_time=0.146, loss_att=94.843, acc=0.897, loss=94.843, backward_time=0.200, grad_norm=72.053, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.002, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:13:35,296 (trainer:732) INFO: 23epoch:train:1860-2002batch: iter_time=5.657e-04, forward_time=0.147, loss_att=92.189, acc=0.898, loss=92.189, backward_time=0.198, grad_norm=77.834, clip=100.000, loss_scale=1.000, optim_step_time=0.100, optim0_lr0=0.002, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:15:41,818 (trainer:732) INFO: 23epoch:train:2003-2145batch: iter_time=5.322e-04, forward_time=0.148, loss_att=98.119, acc=0.895, loss=98.119, backward_time=0.199, grad_norm=87.345, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.002, train_time=3.540 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:17:55,058 (trainer:732) INFO: 23epoch:train:2146-2288batch: iter_time=6.422e-04, forward_time=0.146, loss_att=88.680, acc=0.903, loss=88.680, backward_time=0.199, grad_norm=81.525, clip=100.000, loss_scale=1.000, optim_step_time=0.095, optim0_lr0=0.002, train_time=3.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:20:04,820 (trainer:732) INFO: 23epoch:train:2289-2431batch: iter_time=5.126e-04, forward_time=0.140, loss_att=87.262, acc=0.903, loss=87.262, backward_time=0.191, grad_norm=68.528, clip=100.000, loss_scale=1.000, optim_step_time=0.090, optim0_lr0=0.002, train_time=3.643 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:22:16,948 (trainer:732) INFO: 23epoch:train:2432-2574batch: iter_time=4.124e-04, forward_time=0.133, loss_att=95.548, acc=0.901, loss=95.548, backward_time=0.192, grad_norm=80.965, clip=100.000, loss_scale=1.000, optim_step_time=0.070, optim0_lr0=0.002, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:24:23,497 (trainer:732) INFO: 23epoch:train:2575-2717batch: iter_time=5.225e-04, forward_time=0.141, loss_att=91.848, acc=0.900, loss=91.848, backward_time=0.195, grad_norm=74.613, clip=100.000, loss_scale=1.000, optim_step_time=0.080, optim0_lr0=0.002, train_time=3.536 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:26:38,681 (trainer:732) INFO: 23epoch:train:2718-2860batch: iter_time=6.094e-04, forward_time=0.149, loss_att=90.863, acc=0.904, loss=90.863, backward_time=0.199, grad_norm=78.232, clip=100.000, loss_scale=1.000, optim_step_time=0.095, optim0_lr0=0.002, train_time=3.773 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:40:42,826 (trainer:338) INFO: 23epoch results: [train] iter_time=8.029e-04, forward_time=0.143, loss_att=94.241, acc=0.899, loss=94.241, backward_time=0.197, grad_norm=77.826, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=0.002, train_time=3.924, time=47 minutes and 8.08 seconds, total_count=66217, gpu_max_cached_mem_GB=23.359, [valid] loss_att=30.267, acc=0.946, cer=0.065, wer=0.222, loss=30.267, time=6 minutes and 23.3 seconds, total_count=2024, gpu_max_cached_mem_GB=23.359, [att_plot] time=7 minutes and 19.22 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:40:48,590 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:40:48,601 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/13epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:40:48,602 (trainer:272) INFO: 24/60epoch started. Estimated time to finish: 1 day, 7 hours and 24 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:46:47,415 (trainer:732) INFO: 24epoch:train:1-143batch: iter_time=0.006, forward_time=0.149, loss_att=94.133, acc=0.899, loss=94.133, backward_time=0.199, grad_norm=82.479, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.002, train_time=10.182 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:48:59,362 (trainer:732) INFO: 24epoch:train:144-286batch: iter_time=5.597e-04, forward_time=0.148, loss_att=82.534, acc=0.910, loss=82.534, backward_time=0.197, grad_norm=75.255, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.002, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:51:12,278 (trainer:732) INFO: 24epoch:train:287-429batch: iter_time=6.363e-04, forward_time=0.148, loss_att=85.021, acc=0.909, loss=85.021, backward_time=0.200, grad_norm=77.093, clip=100.000, loss_scale=1.000, optim_step_time=0.102, optim0_lr0=0.002, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:53:26,307 (trainer:732) INFO: 24epoch:train:430-572batch: iter_time=5.935e-04, forward_time=0.147, loss_att=99.185, acc=0.900, loss=99.185, backward_time=0.199, grad_norm=82.835, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=0.002, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:55:31,642 (trainer:732) INFO: 24epoch:train:573-715batch: iter_time=5.331e-04, forward_time=0.147, loss_att=85.245, acc=0.907, loss=85.245, backward_time=0.195, grad_norm=70.629, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.002, train_time=3.503 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:57:44,434 (trainer:732) INFO: 24epoch:train:716-858batch: iter_time=5.117e-04, forward_time=0.149, loss_att=87.547, acc=0.904, loss=87.547, backward_time=0.197, grad_norm=74.088, clip=100.000, loss_scale=1.000, optim_step_time=0.099, optim0_lr0=0.002, train_time=3.722 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 12:59:55,778 (trainer:732) INFO: 24epoch:train:859-1001batch: iter_time=4.725e-04, forward_time=0.140, loss_att=88.451, acc=0.903, loss=88.451, backward_time=0.193, grad_norm=74.720, clip=100.000, loss_scale=1.000, optim_step_time=0.087, optim0_lr0=0.002, train_time=3.671 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:02:08,736 (trainer:732) INFO: 24epoch:train:1002-1144batch: iter_time=4.142e-04, forward_time=0.136, loss_att=93.864, acc=0.902, loss=93.864, backward_time=0.194, grad_norm=78.714, clip=100.000, loss_scale=1.000, optim_step_time=0.074, optim0_lr0=0.002, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:04:16,741 (trainer:732) INFO: 24epoch:train:1145-1287batch: iter_time=3.714e-04, forward_time=0.133, loss_att=93.426, acc=0.900, loss=93.426, backward_time=0.194, grad_norm=86.716, clip=100.000, loss_scale=1.000, optim_step_time=0.070, optim0_lr0=0.002, train_time=3.592 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:06:29,932 (trainer:732) INFO: 24epoch:train:1288-1430batch: iter_time=5.983e-04, forward_time=0.144, loss_att=87.567, acc=0.901, loss=87.567, backward_time=0.198, grad_norm=70.492, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.002, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:08:43,140 (trainer:732) INFO: 24epoch:train:1431-1573batch: iter_time=5.039e-04, forward_time=0.147, loss_att=89.043, acc=0.904, loss=89.043, backward_time=0.196, grad_norm=80.031, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.002, train_time=3.726 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:10:50,230 (trainer:732) INFO: 24epoch:train:1574-1716batch: iter_time=5.700e-04, forward_time=0.146, loss_att=84.481, acc=0.908, loss=84.481, backward_time=0.195, grad_norm=79.748, clip=100.000, loss_scale=1.000, optim_step_time=0.093, optim0_lr0=0.002, train_time=3.547 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:13:03,551 (trainer:732) INFO: 24epoch:train:1717-1859batch: iter_time=5.579e-04, forward_time=0.147, loss_att=100.723, acc=0.897, loss=100.723, backward_time=0.202, grad_norm=83.063, clip=100.000, loss_scale=1.000, optim_step_time=0.095, optim0_lr0=0.002, train_time=3.745 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:15:15,059 (trainer:732) INFO: 24epoch:train:1860-2002batch: iter_time=5.668e-04, forward_time=0.142, loss_att=85.911, acc=0.906, loss=85.911, backward_time=0.195, grad_norm=81.605, clip=100.000, loss_scale=1.000, optim_step_time=0.091, optim0_lr0=0.002, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:17:29,647 (trainer:732) INFO: 24epoch:train:2003-2145batch: iter_time=5.888e-04, forward_time=0.145, loss_att=94.505, acc=0.903, loss=94.505, backward_time=0.201, grad_norm=85.741, clip=100.000, loss_scale=1.000, optim_step_time=0.098, optim0_lr0=0.002, train_time=3.758 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:19:37,970 (trainer:732) INFO: 24epoch:train:2146-2288batch: iter_time=6.133e-04, forward_time=0.148, loss_att=96.420, acc=0.901, loss=96.420, backward_time=0.200, grad_norm=79.929, clip=100.000, loss_scale=1.000, optim_step_time=0.096, optim0_lr0=0.002, train_time=3.581 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:21:49,486 (trainer:732) INFO: 24epoch:train:2289-2431batch: iter_time=5.358e-04, forward_time=0.141, loss_att=88.703, acc=0.903, loss=88.703, backward_time=0.194, grad_norm=86.343, clip=100.000, loss_scale=1.000, optim_step_time=0.086, optim0_lr0=0.002, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:24:01,267 (trainer:732) INFO: 24epoch:train:2432-2574batch: iter_time=4.200e-04, forward_time=0.132, loss_att=90.235, acc=0.904, loss=90.235, backward_time=0.192, grad_norm=84.119, clip=100.000, loss_scale=1.000, optim_step_time=0.073, optim0_lr0=0.002, train_time=3.682 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:26:08,057 (trainer:732) INFO: 24epoch:train:2575-2717batch: iter_time=3.972e-04, forward_time=0.132, loss_att=91.568, acc=0.903, loss=91.568, backward_time=0.192, grad_norm=82.098, clip=100.000, loss_scale=1.000, optim_step_time=0.071, optim0_lr0=0.002, train_time=3.553 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:28:20,623 (trainer:732) INFO: 24epoch:train:2718-2860batch: iter_time=3.803e-04, forward_time=0.136, loss_att=96.155, acc=0.902, loss=96.155, backward_time=0.194, grad_norm=83.036, clip=100.000, loss_scale=1.000, optim_step_time=0.077, optim0_lr0=0.002, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:42:47,767 (trainer:338) INFO: 24epoch results: [train] iter_time=7.975e-04, forward_time=0.143, loss_att=90.541, acc=0.903, loss=90.541, backward_time=0.196, grad_norm=79.832, clip=100.000, loss_scale=1.000, optim_step_time=0.089, optim0_lr0=0.002, train_time=3.986, time=47 minutes and 52.95 seconds, total_count=69096, gpu_max_cached_mem_GB=23.359, [valid] loss_att=29.119, acc=0.949, cer=0.061, wer=0.211, loss=29.119, time=6 minutes and 28.66 seconds, total_count=2112, gpu_max_cached_mem_GB=23.359, [att_plot] time=7 minutes and 37.54 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:42:56,010 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:42:56,027 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/14epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:42:56,028 (trainer:272) INFO: 25/60epoch started. Estimated time to finish: 1 day, 6 hours and 50 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:50:14,291 (trainer:732) INFO: 25epoch:train:1-143batch: iter_time=0.004, forward_time=0.133, loss_att=82.272, acc=0.909, loss=82.272, backward_time=0.191, grad_norm=80.549, clip=100.000, loss_scale=1.000, optim_step_time=0.073, optim0_lr0=0.002, train_time=12.484 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:52:23,196 (trainer:732) INFO: 25epoch:train:144-286batch: iter_time=3.382e-04, forward_time=0.129, loss_att=83.973, acc=0.909, loss=83.973, backward_time=0.190, grad_norm=79.603, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.573 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:54:36,235 (trainer:732) INFO: 25epoch:train:287-429batch: iter_time=3.194e-04, forward_time=0.128, loss_att=87.433, acc=0.907, loss=87.433, backward_time=0.190, grad_norm=80.202, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:56:49,217 (trainer:732) INFO: 25epoch:train:430-572batch: iter_time=3.278e-04, forward_time=0.128, loss_att=84.780, acc=0.909, loss=84.780, backward_time=0.190, grad_norm=76.826, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 13:58:55,656 (trainer:732) INFO: 25epoch:train:573-715batch: iter_time=3.329e-04, forward_time=0.129, loss_att=88.385, acc=0.904, loss=88.385, backward_time=0.192, grad_norm=79.045, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.543 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:01:08,615 (trainer:732) INFO: 25epoch:train:716-858batch: iter_time=3.165e-04, forward_time=0.130, loss_att=87.986, acc=0.907, loss=87.986, backward_time=0.192, grad_norm=80.391, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:03:20,791 (trainer:732) INFO: 25epoch:train:859-1001batch: iter_time=3.341e-04, forward_time=0.128, loss_att=84.107, acc=0.909, loss=84.107, backward_time=0.189, grad_norm=79.236, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:05:31,268 (trainer:732) INFO: 25epoch:train:1002-1144batch: iter_time=3.249e-04, forward_time=0.128, loss_att=87.063, acc=0.905, loss=87.063, backward_time=0.189, grad_norm=77.199, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.639 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:07:38,889 (trainer:732) INFO: 25epoch:train:1145-1287batch: iter_time=3.171e-04, forward_time=0.129, loss_att=91.219, acc=0.904, loss=91.219, backward_time=0.191, grad_norm=80.515, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.576 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:09:51,695 (trainer:732) INFO: 25epoch:train:1288-1430batch: iter_time=3.316e-04, forward_time=0.130, loss_att=91.047, acc=0.905, loss=91.047, backward_time=0.192, grad_norm=78.398, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:12:03,061 (trainer:732) INFO: 25epoch:train:1431-1573batch: iter_time=3.291e-04, forward_time=0.127, loss_att=82.897, acc=0.909, loss=82.897, backward_time=0.188, grad_norm=80.697, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.674 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:14:09,175 (trainer:732) INFO: 25epoch:train:1574-1716batch: iter_time=3.187e-04, forward_time=0.127, loss_att=87.621, acc=0.905, loss=87.621, backward_time=0.189, grad_norm=80.098, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.522 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:16:20,497 (trainer:732) INFO: 25epoch:train:1717-1859batch: iter_time=3.226e-04, forward_time=0.127, loss_att=81.387, acc=0.909, loss=81.387, backward_time=0.187, grad_norm=73.645, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:18:33,083 (trainer:732) INFO: 25epoch:train:1860-2002batch: iter_time=3.140e-04, forward_time=0.127, loss_att=87.644, acc=0.906, loss=87.644, backward_time=0.189, grad_norm=74.677, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:20:46,174 (trainer:732) INFO: 25epoch:train:2003-2145batch: iter_time=3.244e-04, forward_time=0.130, loss_att=96.064, acc=0.903, loss=96.064, backward_time=0.193, grad_norm=84.004, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:22:53,245 (trainer:732) INFO: 25epoch:train:2146-2288batch: iter_time=3.350e-04, forward_time=0.129, loss_att=93.108, acc=0.903, loss=93.108, backward_time=0.193, grad_norm=83.500, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.546 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:25:05,763 (trainer:732) INFO: 25epoch:train:2289-2431batch: iter_time=3.208e-04, forward_time=0.129, loss_att=90.813, acc=0.905, loss=90.813, backward_time=0.192, grad_norm=78.013, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.719 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:27:17,145 (trainer:732) INFO: 25epoch:train:2432-2574batch: iter_time=3.344e-04, forward_time=0.127, loss_att=81.308, acc=0.912, loss=81.308, backward_time=0.189, grad_norm=83.806, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.674 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:29:24,627 (trainer:732) INFO: 25epoch:train:2575-2717batch: iter_time=3.304e-04, forward_time=0.130, loss_att=88.722, acc=0.908, loss=88.722, backward_time=0.194, grad_norm=84.790, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.563 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:31:39,988 (trainer:732) INFO: 25epoch:train:2718-2860batch: iter_time=2.987e-04, forward_time=0.130, loss_att=93.964, acc=0.906, loss=93.964, backward_time=0.195, grad_norm=86.971, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.777 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:40:30,865 (trainer:338) INFO: 25epoch results: [train] iter_time=4.893e-04, forward_time=0.129, loss_att=87.391, acc=0.907, loss=87.391, backward_time=0.191, grad_norm=80.096, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=4.086, time=49 minutes and 2.98 seconds, total_count=71975, gpu_max_cached_mem_GB=23.359, [valid] loss_att=27.477, acc=0.952, cer=0.059, wer=0.203, loss=27.477, time=4 minutes and 4.09 seconds, total_count=2200, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.76 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:40:34,794 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:40:34,808 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/15epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:40:34,808 (trainer:272) INFO: 26/60epoch started. Estimated time to finish: 1 day, 6 hours and 7 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:45:10,936 (trainer:732) INFO: 26epoch:train:1-143batch: iter_time=0.002, forward_time=0.129, loss_att=85.990, acc=0.911, loss=85.990, backward_time=0.191, grad_norm=82.774, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=7.824 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:47:17,556 (trainer:732) INFO: 26epoch:train:144-286batch: iter_time=3.025e-04, forward_time=0.130, loss_att=91.522, acc=0.908, loss=91.522, backward_time=0.194, grad_norm=86.068, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.538 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:49:29,026 (trainer:732) INFO: 26epoch:train:287-429batch: iter_time=3.143e-04, forward_time=0.127, loss_att=75.834, acc=0.915, loss=75.834, backward_time=0.188, grad_norm=81.961, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:51:40,866 (trainer:732) INFO: 26epoch:train:430-572batch: iter_time=3.224e-04, forward_time=0.127, loss_att=80.095, acc=0.910, loss=80.095, backward_time=0.189, grad_norm=80.474, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.679 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:53:48,110 (trainer:732) INFO: 26epoch:train:573-715batch: iter_time=3.117e-04, forward_time=0.129, loss_att=86.257, acc=0.911, loss=86.257, backward_time=0.192, grad_norm=89.769, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.568 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:55:59,925 (trainer:732) INFO: 26epoch:train:716-858batch: iter_time=3.041e-04, forward_time=0.128, loss_att=77.181, acc=0.914, loss=77.181, backward_time=0.189, grad_norm=75.681, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 14:58:11,927 (trainer:732) INFO: 26epoch:train:859-1001batch: iter_time=3.515e-04, forward_time=0.128, loss_att=81.826, acc=0.911, loss=81.826, backward_time=0.189, grad_norm=77.449, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:00:25,263 (trainer:732) INFO: 26epoch:train:1002-1144batch: iter_time=3.123e-04, forward_time=0.130, loss_att=88.988, acc=0.908, loss=88.988, backward_time=0.193, grad_norm=80.276, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:02:31,042 (trainer:732) INFO: 26epoch:train:1145-1287batch: iter_time=2.964e-04, forward_time=0.127, loss_att=80.258, acc=0.912, loss=80.258, backward_time=0.188, grad_norm=77.614, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.527 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:04:44,702 (trainer:732) INFO: 26epoch:train:1288-1430batch: iter_time=3.190e-04, forward_time=0.129, loss_att=94.493, acc=0.903, loss=94.493, backward_time=0.193, grad_norm=75.127, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.737 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:06:56,235 (trainer:732) INFO: 26epoch:train:1431-1573batch: iter_time=3.013e-04, forward_time=0.127, loss_att=81.671, acc=0.912, loss=81.671, backward_time=0.189, grad_norm=83.146, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.675 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:09:03,288 (trainer:732) INFO: 26epoch:train:1574-1716batch: iter_time=3.168e-04, forward_time=0.129, loss_att=90.452, acc=0.907, loss=90.452, backward_time=0.192, grad_norm=86.345, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.547 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:11:14,722 (trainer:732) INFO: 26epoch:train:1717-1859batch: iter_time=3.134e-04, forward_time=0.127, loss_att=85.852, acc=0.909, loss=85.852, backward_time=0.189, grad_norm=85.535, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:13:27,197 (trainer:732) INFO: 26epoch:train:1860-2002batch: iter_time=3.073e-04, forward_time=0.128, loss_att=84.373, acc=0.911, loss=84.373, backward_time=0.190, grad_norm=77.546, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:15:40,400 (trainer:732) INFO: 26epoch:train:2003-2145batch: iter_time=3.185e-04, forward_time=0.127, loss_att=78.349, acc=0.913, loss=78.349, backward_time=0.188, grad_norm=82.154, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:17:47,287 (trainer:732) INFO: 26epoch:train:2146-2288batch: iter_time=3.249e-04, forward_time=0.129, loss_att=83.030, acc=0.911, loss=83.030, backward_time=0.192, grad_norm=85.250, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.538 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:20:00,749 (trainer:732) INFO: 26epoch:train:2289-2431batch: iter_time=2.952e-04, forward_time=0.131, loss_att=88.582, acc=0.909, loss=88.582, backward_time=0.195, grad_norm=84.196, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.745 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:22:12,863 (trainer:732) INFO: 26epoch:train:2432-2574batch: iter_time=3.185e-04, forward_time=0.129, loss_att=83.478, acc=0.912, loss=83.478, backward_time=0.192, grad_norm=88.788, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:24:20,466 (trainer:732) INFO: 26epoch:train:2575-2717batch: iter_time=3.250e-04, forward_time=0.130, loss_att=90.309, acc=0.907, loss=90.309, backward_time=0.194, grad_norm=83.117, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.568 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:26:32,186 (trainer:732) INFO: 26epoch:train:2718-2860batch: iter_time=2.973e-04, forward_time=0.127, loss_att=78.801, acc=0.914, loss=78.801, backward_time=0.189, grad_norm=85.194, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.671 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:35:20,117 (trainer:338) INFO: 26epoch results: [train] iter_time=4.124e-04, forward_time=0.128, loss_att=84.180, acc=0.910, loss=84.180, backward_time=0.191, grad_norm=82.362, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.855, time=46 minutes and 16.22 seconds, total_count=74854, gpu_max_cached_mem_GB=23.359, [valid] loss_att=26.711, acc=0.954, cer=0.057, wer=0.196, loss=26.711, time=4 minutes and 0.54 seconds, total_count=2288, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 28.55 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:35:24,469 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:35:24,478 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/16epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:35:24,478 (trainer:272) INFO: 27/60epoch started. Estimated time to finish: 1 day, 5 hours and 20 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:39:59,196 (trainer:732) INFO: 27epoch:train:1-143batch: iter_time=0.003, forward_time=0.129, loss_att=77.477, acc=0.916, loss=77.477, backward_time=0.191, grad_norm=76.736, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=7.785 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:42:05,151 (trainer:732) INFO: 27epoch:train:144-286batch: iter_time=2.797e-04, forward_time=0.128, loss_att=78.976, acc=0.917, loss=78.976, backward_time=0.191, grad_norm=73.619, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.516 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:44:17,731 (trainer:732) INFO: 27epoch:train:287-429batch: iter_time=3.009e-04, forward_time=0.129, loss_att=84.533, acc=0.911, loss=84.533, backward_time=0.191, grad_norm=86.227, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:46:28,680 (trainer:732) INFO: 27epoch:train:430-572batch: iter_time=2.926e-04, forward_time=0.127, loss_att=76.529, acc=0.916, loss=76.529, backward_time=0.189, grad_norm=73.671, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.655 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:48:35,498 (trainer:732) INFO: 27epoch:train:573-715batch: iter_time=2.805e-04, forward_time=0.128, loss_att=82.893, acc=0.913, loss=82.893, backward_time=0.191, grad_norm=85.979, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.555 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:50:49,466 (trainer:732) INFO: 27epoch:train:716-858batch: iter_time=2.896e-04, forward_time=0.130, loss_att=88.327, acc=0.909, loss=88.327, backward_time=0.195, grad_norm=86.244, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.747 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:53:04,067 (trainer:732) INFO: 27epoch:train:859-1001batch: iter_time=2.967e-04, forward_time=0.129, loss_att=89.097, acc=0.913, loss=89.097, backward_time=0.194, grad_norm=91.686, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.763 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:55:16,768 (trainer:732) INFO: 27epoch:train:1002-1144batch: iter_time=3.006e-04, forward_time=0.129, loss_att=87.762, acc=0.909, loss=87.762, backward_time=0.192, grad_norm=87.293, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:57:22,183 (trainer:732) INFO: 27epoch:train:1145-1287batch: iter_time=3.023e-04, forward_time=0.127, loss_att=81.607, acc=0.912, loss=81.607, backward_time=0.189, grad_norm=76.398, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.516 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 15:59:33,951 (trainer:732) INFO: 27epoch:train:1288-1430batch: iter_time=2.954e-04, forward_time=0.128, loss_att=83.368, acc=0.912, loss=83.368, backward_time=0.190, grad_norm=80.540, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.686 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:01:43,573 (trainer:732) INFO: 27epoch:train:1431-1573batch: iter_time=2.813e-04, forward_time=0.126, loss_att=71.892, acc=0.917, loss=71.892, backward_time=0.186, grad_norm=77.289, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.622 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:03:50,216 (trainer:732) INFO: 27epoch:train:1574-1716batch: iter_time=2.960e-04, forward_time=0.128, loss_att=82.847, acc=0.911, loss=82.847, backward_time=0.189, grad_norm=80.181, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.535 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:06:02,468 (trainer:732) INFO: 27epoch:train:1717-1859batch: iter_time=3.003e-04, forward_time=0.128, loss_att=81.225, acc=0.913, loss=81.225, backward_time=0.190, grad_norm=87.333, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:08:14,912 (trainer:732) INFO: 27epoch:train:1860-2002batch: iter_time=3.058e-04, forward_time=0.128, loss_att=77.625, acc=0.917, loss=77.625, backward_time=0.191, grad_norm=83.028, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:10:27,252 (trainer:732) INFO: 27epoch:train:2003-2145batch: iter_time=2.862e-04, forward_time=0.128, loss_att=84.768, acc=0.909, loss=84.768, backward_time=0.190, grad_norm=83.870, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:12:34,725 (trainer:732) INFO: 27epoch:train:2146-2288batch: iter_time=2.980e-04, forward_time=0.128, loss_att=84.914, acc=0.911, loss=84.914, backward_time=0.191, grad_norm=73.464, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.558 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:14:46,298 (trainer:732) INFO: 27epoch:train:2289-2431batch: iter_time=2.769e-04, forward_time=0.127, loss_att=78.896, acc=0.915, loss=78.896, backward_time=0.189, grad_norm=78.991, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.690 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:16:58,663 (trainer:732) INFO: 27epoch:train:2432-2574batch: iter_time=2.816e-04, forward_time=0.127, loss_att=78.383, acc=0.916, loss=78.383, backward_time=0.189, grad_norm=75.675, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:19:07,819 (trainer:732) INFO: 27epoch:train:2575-2717batch: iter_time=2.932e-04, forward_time=0.131, loss_att=93.848, acc=0.905, loss=93.848, backward_time=0.196, grad_norm=80.584, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.617 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:21:20,528 (trainer:732) INFO: 27epoch:train:2718-2860batch: iter_time=2.808e-04, forward_time=0.128, loss_att=79.969, acc=0.912, loss=79.969, backward_time=0.190, grad_norm=82.225, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:30:09,996 (trainer:338) INFO: 27epoch results: [train] iter_time=4.370e-04, forward_time=0.128, loss_att=82.090, acc=0.913, loss=82.090, backward_time=0.191, grad_norm=81.107, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.853, time=46 minutes and 15.2 seconds, total_count=77733, gpu_max_cached_mem_GB=23.359, [valid] loss_att=26.617, acc=0.954, cer=0.056, wer=0.195, loss=26.617, time=4 minutes and 0.97 seconds, total_count=2376, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 29.35 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:30:14,219 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:30:14,227 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/17epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:30:14,228 (trainer:272) INFO: 28/60epoch started. Estimated time to finish: 1 day, 4 hours and 32 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:34:49,906 (trainer:732) INFO: 28epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=78.963, acc=0.915, loss=78.963, backward_time=0.192, grad_norm=82.545, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=7.805 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:36:57,479 (trainer:732) INFO: 28epoch:train:144-286batch: iter_time=3.051e-04, forward_time=0.129, loss_att=82.998, acc=0.916, loss=82.998, backward_time=0.192, grad_norm=83.991, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.567 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:39:10,420 (trainer:732) INFO: 28epoch:train:287-429batch: iter_time=2.871e-04, forward_time=0.131, loss_att=81.602, acc=0.916, loss=81.602, backward_time=0.195, grad_norm=89.197, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.723 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:41:22,936 (trainer:732) INFO: 28epoch:train:430-572batch: iter_time=3.210e-04, forward_time=0.128, loss_att=80.395, acc=0.914, loss=80.395, backward_time=0.190, grad_norm=75.336, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:43:33,883 (trainer:732) INFO: 28epoch:train:573-715batch: iter_time=3.069e-04, forward_time=0.127, loss_att=73.954, acc=0.918, loss=73.954, backward_time=0.188, grad_norm=77.732, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.680 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:45:40,486 (trainer:732) INFO: 28epoch:train:716-858batch: iter_time=2.910e-04, forward_time=0.128, loss_att=80.743, acc=0.915, loss=80.743, backward_time=0.190, grad_norm=84.875, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.536 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:47:52,025 (trainer:732) INFO: 28epoch:train:859-1001batch: iter_time=3.128e-04, forward_time=0.127, loss_att=75.156, acc=0.919, loss=75.156, backward_time=0.189, grad_norm=74.187, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.681 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:50:04,603 (trainer:732) INFO: 28epoch:train:1002-1144batch: iter_time=2.993e-04, forward_time=0.129, loss_att=82.160, acc=0.913, loss=82.160, backward_time=0.192, grad_norm=82.600, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:52:11,503 (trainer:732) INFO: 28epoch:train:1145-1287batch: iter_time=3.001e-04, forward_time=0.129, loss_att=75.434, acc=0.918, loss=75.434, backward_time=0.192, grad_norm=88.046, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.557 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:54:24,389 (trainer:732) INFO: 28epoch:train:1288-1430batch: iter_time=2.972e-04, forward_time=0.129, loss_att=85.469, acc=0.911, loss=85.469, backward_time=0.193, grad_norm=83.226, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:56:36,248 (trainer:732) INFO: 28epoch:train:1431-1573batch: iter_time=2.947e-04, forward_time=0.128, loss_att=78.426, acc=0.916, loss=78.426, backward_time=0.191, grad_norm=80.646, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.688 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 16:58:50,103 (trainer:732) INFO: 28epoch:train:1574-1716batch: iter_time=3.177e-04, forward_time=0.129, loss_att=79.292, acc=0.917, loss=79.292, backward_time=0.191, grad_norm=77.541, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:00:56,523 (trainer:732) INFO: 28epoch:train:1717-1859batch: iter_time=3.495e-04, forward_time=0.127, loss_att=79.274, acc=0.916, loss=79.274, backward_time=0.189, grad_norm=78.151, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.548 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:03:10,052 (trainer:732) INFO: 28epoch:train:1860-2002batch: iter_time=2.939e-04, forward_time=0.129, loss_att=86.513, acc=0.913, loss=86.513, backward_time=0.193, grad_norm=85.087, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.726 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:05:23,648 (trainer:732) INFO: 28epoch:train:2003-2145batch: iter_time=3.067e-04, forward_time=0.129, loss_att=84.154, acc=0.911, loss=84.154, backward_time=0.192, grad_norm=98.648, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.739 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:07:29,156 (trainer:732) INFO: 28epoch:train:2146-2288batch: iter_time=3.179e-04, forward_time=0.126, loss_att=70.840, acc=0.921, loss=70.840, backward_time=0.186, grad_norm=73.002, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.502 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:09:40,391 (trainer:732) INFO: 28epoch:train:2289-2431batch: iter_time=3.346e-04, forward_time=0.129, loss_att=79.924, acc=0.916, loss=79.924, backward_time=0.192, grad_norm=89.527, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.685 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:11:52,754 (trainer:732) INFO: 28epoch:train:2432-2574batch: iter_time=3.143e-04, forward_time=0.127, loss_att=78.525, acc=0.913, loss=78.525, backward_time=0.188, grad_norm=76.767, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:14:04,764 (trainer:732) INFO: 28epoch:train:2575-2717batch: iter_time=3.168e-04, forward_time=0.126, loss_att=76.701, acc=0.917, loss=76.701, backward_time=0.188, grad_norm=82.149, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:16:11,963 (trainer:732) INFO: 28epoch:train:2718-2860batch: iter_time=2.828e-04, forward_time=0.128, loss_att=80.803, acc=0.915, loss=80.803, backward_time=0.190, grad_norm=85.575, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.551 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:24:57,031 (trainer:338) INFO: 28epoch results: [train] iter_time=4.378e-04, forward_time=0.128, loss_att=79.500, acc=0.916, loss=79.500, backward_time=0.191, grad_norm=82.404, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.856, time=46 minutes and 17.02 seconds, total_count=80612, gpu_max_cached_mem_GB=23.359, [valid] loss_att=25.766, acc=0.956, cer=0.055, wer=0.189, loss=25.766, time=4 minutes and 0.69 seconds, total_count=2464, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.09 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:25:01,074 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:25:01,083 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/18epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:25:01,083 (trainer:272) INFO: 29/60epoch started. Estimated time to finish: 1 day, 3 hours and 43 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:29:38,891 (trainer:732) INFO: 29epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=79.759, acc=0.918, loss=79.759, backward_time=0.194, grad_norm=82.331, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=7.864 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:31:51,316 (trainer:732) INFO: 29epoch:train:144-286batch: iter_time=3.174e-04, forward_time=0.129, loss_att=75.721, acc=0.920, loss=75.721, backward_time=0.191, grad_norm=78.226, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:33:57,610 (trainer:732) INFO: 29epoch:train:287-429batch: iter_time=3.039e-04, forward_time=0.128, loss_att=75.695, acc=0.919, loss=75.695, backward_time=0.190, grad_norm=83.484, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.536 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:36:10,787 (trainer:732) INFO: 29epoch:train:430-572batch: iter_time=3.295e-04, forward_time=0.129, loss_att=80.876, acc=0.917, loss=80.876, backward_time=0.192, grad_norm=86.046, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:38:23,821 (trainer:732) INFO: 29epoch:train:573-715batch: iter_time=3.216e-04, forward_time=0.129, loss_att=84.233, acc=0.914, loss=84.233, backward_time=0.192, grad_norm=79.442, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.732 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:40:29,950 (trainer:732) INFO: 29epoch:train:716-858batch: iter_time=3.010e-04, forward_time=0.128, loss_att=74.850, acc=0.923, loss=74.850, backward_time=0.190, grad_norm=82.642, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.531 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:42:43,174 (trainer:732) INFO: 29epoch:train:859-1001batch: iter_time=3.711e-04, forward_time=0.129, loss_att=80.199, acc=0.917, loss=80.199, backward_time=0.191, grad_norm=75.916, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.723 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:44:55,774 (trainer:732) INFO: 29epoch:train:1002-1144batch: iter_time=3.034e-04, forward_time=0.128, loss_att=76.208, acc=0.918, loss=76.208, backward_time=0.190, grad_norm=74.637, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:47:08,221 (trainer:732) INFO: 29epoch:train:1145-1287batch: iter_time=3.200e-04, forward_time=0.129, loss_att=77.832, acc=0.917, loss=77.832, backward_time=0.192, grad_norm=80.084, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.713 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:49:13,171 (trainer:732) INFO: 29epoch:train:1288-1430batch: iter_time=3.148e-04, forward_time=0.129, loss_att=76.938, acc=0.918, loss=76.938, backward_time=0.191, grad_norm=78.676, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.494 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:51:24,297 (trainer:732) INFO: 29epoch:train:1431-1573batch: iter_time=3.163e-04, forward_time=0.128, loss_att=73.984, acc=0.920, loss=73.984, backward_time=0.189, grad_norm=80.572, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.672 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:53:37,172 (trainer:732) INFO: 29epoch:train:1574-1716batch: iter_time=3.175e-04, forward_time=0.130, loss_att=79.513, acc=0.918, loss=79.513, backward_time=0.193, grad_norm=80.743, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:55:43,419 (trainer:732) INFO: 29epoch:train:1717-1859batch: iter_time=3.191e-04, forward_time=0.129, loss_att=75.818, acc=0.918, loss=75.818, backward_time=0.192, grad_norm=81.029, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.538 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 17:57:56,412 (trainer:732) INFO: 29epoch:train:1860-2002batch: iter_time=3.045e-04, forward_time=0.129, loss_att=76.726, acc=0.919, loss=76.726, backward_time=0.190, grad_norm=78.462, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.722 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:00:09,520 (trainer:732) INFO: 29epoch:train:2003-2145batch: iter_time=3.172e-04, forward_time=0.129, loss_att=80.100, acc=0.916, loss=80.100, backward_time=0.190, grad_norm=76.228, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.722 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:02:21,359 (trainer:732) INFO: 29epoch:train:2146-2288batch: iter_time=3.206e-04, forward_time=0.128, loss_att=75.150, acc=0.918, loss=75.150, backward_time=0.190, grad_norm=72.161, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:04:27,334 (trainer:732) INFO: 29epoch:train:2289-2431batch: iter_time=3.015e-04, forward_time=0.127, loss_att=71.340, acc=0.922, loss=71.340, backward_time=0.189, grad_norm=81.089, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.528 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:06:39,488 (trainer:732) INFO: 29epoch:train:2432-2574batch: iter_time=3.008e-04, forward_time=0.126, loss_att=74.633, acc=0.919, loss=74.633, backward_time=0.187, grad_norm=82.089, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:08:51,782 (trainer:732) INFO: 29epoch:train:2575-2717batch: iter_time=3.001e-04, forward_time=0.128, loss_att=74.088, acc=0.920, loss=74.088, backward_time=0.191, grad_norm=82.263, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:10:57,694 (trainer:732) INFO: 29epoch:train:2718-2860batch: iter_time=2.869e-04, forward_time=0.128, loss_att=74.780, acc=0.921, loss=74.780, backward_time=0.190, grad_norm=75.358, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.513 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:19:45,292 (trainer:338) INFO: 29epoch results: [train] iter_time=4.679e-04, forward_time=0.128, loss_att=76.799, acc=0.919, loss=76.799, backward_time=0.191, grad_norm=79.487, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.854, time=46 minutes and 15.96 seconds, total_count=83491, gpu_max_cached_mem_GB=23.359, [valid] loss_att=24.217, acc=0.959, cer=0.051, wer=0.179, loss=24.217, time=4 minutes and 2.15 seconds, total_count=2552, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 26.1 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:19:49,123 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:19:49,132 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/19epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:19:49,133 (trainer:272) INFO: 30/60epoch started. Estimated time to finish: 1 day, 2 hours and 54 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:24:20,405 (trainer:732) INFO: 30epoch:train:1-143batch: iter_time=0.003, forward_time=0.127, loss_att=68.660, acc=0.924, loss=68.660, backward_time=0.188, grad_norm=70.598, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=7.683 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:26:33,287 (trainer:732) INFO: 30epoch:train:144-286batch: iter_time=3.151e-04, forward_time=0.128, loss_att=74.045, acc=0.921, loss=74.045, backward_time=0.190, grad_norm=80.180, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:28:40,963 (trainer:732) INFO: 30epoch:train:287-429batch: iter_time=3.131e-04, forward_time=0.128, loss_att=72.961, acc=0.922, loss=72.961, backward_time=0.189, grad_norm=74.720, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.576 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:30:53,749 (trainer:732) INFO: 30epoch:train:430-572batch: iter_time=3.195e-04, forward_time=0.128, loss_att=70.744, acc=0.924, loss=70.744, backward_time=0.191, grad_norm=78.651, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:33:06,443 (trainer:732) INFO: 30epoch:train:573-715batch: iter_time=3.210e-04, forward_time=0.128, loss_att=73.345, acc=0.922, loss=73.345, backward_time=0.190, grad_norm=78.770, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.726 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:35:13,611 (trainer:732) INFO: 30epoch:train:716-858batch: iter_time=3.231e-04, forward_time=0.129, loss_att=76.237, acc=0.922, loss=76.237, backward_time=0.192, grad_norm=80.232, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.558 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:37:25,857 (trainer:732) INFO: 30epoch:train:859-1001batch: iter_time=3.135e-04, forward_time=0.130, loss_att=76.912, acc=0.922, loss=76.912, backward_time=0.194, grad_norm=83.798, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:39:38,859 (trainer:732) INFO: 30epoch:train:1002-1144batch: iter_time=3.117e-04, forward_time=0.129, loss_att=74.428, acc=0.922, loss=74.428, backward_time=0.191, grad_norm=82.036, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:41:50,143 (trainer:732) INFO: 30epoch:train:1145-1287batch: iter_time=3.256e-04, forward_time=0.129, loss_att=73.028, acc=0.922, loss=73.028, backward_time=0.191, grad_norm=80.473, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.681 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:43:58,091 (trainer:732) INFO: 30epoch:train:1288-1430batch: iter_time=3.349e-04, forward_time=0.127, loss_att=73.744, acc=0.922, loss=73.744, backward_time=0.189, grad_norm=78.621, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.576 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:46:11,865 (trainer:732) INFO: 30epoch:train:1431-1573batch: iter_time=3.090e-04, forward_time=0.131, loss_att=77.956, acc=0.920, loss=77.956, backward_time=0.195, grad_norm=85.870, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.744 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:48:23,791 (trainer:732) INFO: 30epoch:train:1574-1716batch: iter_time=3.296e-04, forward_time=0.127, loss_att=68.987, acc=0.925, loss=68.987, backward_time=0.189, grad_norm=71.910, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.681 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:50:30,223 (trainer:732) INFO: 30epoch:train:1717-1859batch: iter_time=3.256e-04, forward_time=0.128, loss_att=75.262, acc=0.921, loss=75.262, backward_time=0.191, grad_norm=84.867, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.553 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:52:42,201 (trainer:732) INFO: 30epoch:train:1860-2002batch: iter_time=3.057e-04, forward_time=0.128, loss_att=71.015, acc=0.924, loss=71.015, backward_time=0.189, grad_norm=71.618, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.680 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:54:52,736 (trainer:732) INFO: 30epoch:train:2003-2145batch: iter_time=3.164e-04, forward_time=0.126, loss_att=68.034, acc=0.924, loss=68.034, backward_time=0.187, grad_norm=73.164, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.652 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:57:06,056 (trainer:732) INFO: 30epoch:train:2146-2288batch: iter_time=3.180e-04, forward_time=0.129, loss_att=74.427, acc=0.922, loss=74.427, backward_time=0.191, grad_norm=86.199, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 18:59:13,836 (trainer:732) INFO: 30epoch:train:2289-2431batch: iter_time=3.036e-04, forward_time=0.130, loss_att=75.066, acc=0.924, loss=75.066, backward_time=0.194, grad_norm=82.375, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.582 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:01:25,339 (trainer:732) INFO: 30epoch:train:2432-2574batch: iter_time=3.127e-04, forward_time=0.128, loss_att=67.318, acc=0.927, loss=67.318, backward_time=0.189, grad_norm=80.724, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.674 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:03:37,665 (trainer:732) INFO: 30epoch:train:2575-2717batch: iter_time=3.288e-04, forward_time=0.128, loss_att=71.618, acc=0.924, loss=71.618, backward_time=0.190, grad_norm=77.799, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.704 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:05:45,343 (trainer:732) INFO: 30epoch:train:2718-2860batch: iter_time=2.970e-04, forward_time=0.130, loss_att=76.951, acc=0.921, loss=76.951, backward_time=0.194, grad_norm=77.727, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.561 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:14:27,789 (trainer:338) INFO: 30epoch results: [train] iter_time=4.265e-04, forward_time=0.128, loss_att=73.021, acc=0.923, loss=73.021, backward_time=0.191, grad_norm=79.034, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.854, time=46 minutes and 15.5 seconds, total_count=86370, gpu_max_cached_mem_GB=23.359, [valid] loss_att=24.305, acc=0.957, cer=0.049, wer=0.175, loss=24.305, time=3 minutes and 57.39 seconds, total_count=2640, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.77 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:14:31,852 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:14:31,865 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/20epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:14:31,865 (trainer:272) INFO: 31/60epoch started. Estimated time to finish: 1 day, 2 hours and 5 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:19:04,520 (trainer:732) INFO: 31epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=68.089, acc=0.927, loss=68.089, backward_time=0.189, grad_norm=73.076, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=7.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:21:17,800 (trainer:732) INFO: 31epoch:train:144-286batch: iter_time=3.217e-04, forward_time=0.130, loss_att=70.152, acc=0.926, loss=70.152, backward_time=0.193, grad_norm=76.919, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.728 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:23:23,622 (trainer:732) INFO: 31epoch:train:287-429batch: iter_time=3.134e-04, forward_time=0.127, loss_att=66.512, acc=0.927, loss=66.512, backward_time=0.188, grad_norm=82.151, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.522 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:25:36,267 (trainer:732) INFO: 31epoch:train:430-572batch: iter_time=3.245e-04, forward_time=0.127, loss_att=66.499, acc=0.931, loss=66.499, backward_time=0.190, grad_norm=76.715, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:27:49,203 (trainer:732) INFO: 31epoch:train:573-715batch: iter_time=3.207e-04, forward_time=0.130, loss_att=76.414, acc=0.922, loss=76.414, backward_time=0.193, grad_norm=86.957, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.731 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:30:00,707 (trainer:732) INFO: 31epoch:train:716-858batch: iter_time=3.136e-04, forward_time=0.128, loss_att=66.764, acc=0.927, loss=66.764, backward_time=0.189, grad_norm=73.749, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.673 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:32:07,427 (trainer:732) INFO: 31epoch:train:859-1001batch: iter_time=3.249e-04, forward_time=0.129, loss_att=68.400, acc=0.926, loss=68.400, backward_time=0.192, grad_norm=78.921, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.547 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:34:20,233 (trainer:732) INFO: 31epoch:train:1002-1144batch: iter_time=3.072e-04, forward_time=0.128, loss_att=63.952, acc=0.930, loss=63.952, backward_time=0.188, grad_norm=73.777, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:36:32,338 (trainer:732) INFO: 31epoch:train:1145-1287batch: iter_time=3.073e-04, forward_time=0.129, loss_att=71.328, acc=0.924, loss=71.328, backward_time=0.193, grad_norm=79.214, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:38:38,048 (trainer:732) INFO: 31epoch:train:1288-1430batch: iter_time=3.254e-04, forward_time=0.128, loss_att=70.112, acc=0.927, loss=70.112, backward_time=0.190, grad_norm=76.801, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.516 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:40:51,433 (trainer:732) INFO: 31epoch:train:1431-1573batch: iter_time=3.156e-04, forward_time=0.129, loss_att=71.302, acc=0.927, loss=71.302, backward_time=0.191, grad_norm=87.615, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:43:03,155 (trainer:732) INFO: 31epoch:train:1574-1716batch: iter_time=3.127e-04, forward_time=0.128, loss_att=66.383, acc=0.929, loss=66.383, backward_time=0.189, grad_norm=76.864, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.675 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:45:15,597 (trainer:732) INFO: 31epoch:train:1717-1859batch: iter_time=3.264e-04, forward_time=0.128, loss_att=73.695, acc=0.923, loss=73.695, backward_time=0.191, grad_norm=83.423, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:47:21,832 (trainer:732) INFO: 31epoch:train:1860-2002batch: iter_time=3.168e-04, forward_time=0.128, loss_att=67.371, acc=0.928, loss=67.371, backward_time=0.189, grad_norm=77.249, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.532 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:49:34,034 (trainer:732) INFO: 31epoch:train:2003-2145batch: iter_time=3.070e-04, forward_time=0.130, loss_att=75.313, acc=0.922, loss=75.313, backward_time=0.193, grad_norm=73.564, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:51:46,574 (trainer:732) INFO: 31epoch:train:2146-2288batch: iter_time=3.257e-04, forward_time=0.128, loss_att=71.855, acc=0.924, loss=71.855, backward_time=0.190, grad_norm=85.050, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:53:54,418 (trainer:732) INFO: 31epoch:train:2289-2431batch: iter_time=3.178e-04, forward_time=0.130, loss_att=73.504, acc=0.924, loss=73.504, backward_time=0.193, grad_norm=74.367, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.585 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:56:06,946 (trainer:732) INFO: 31epoch:train:2432-2574batch: iter_time=3.160e-04, forward_time=0.129, loss_att=73.328, acc=0.924, loss=73.328, backward_time=0.192, grad_norm=78.682, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 19:58:18,546 (trainer:732) INFO: 31epoch:train:2575-2717batch: iter_time=3.147e-04, forward_time=0.128, loss_att=72.605, acc=0.924, loss=72.605, backward_time=0.190, grad_norm=71.963, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:00:30,978 (trainer:732) INFO: 31epoch:train:2718-2860batch: iter_time=3.056e-04, forward_time=0.129, loss_att=71.250, acc=0.926, loss=71.250, backward_time=0.192, grad_norm=77.327, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.695 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 152) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 152) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:09:19,077 (trainer:338) INFO: 31epoch results: [train] iter_time=4.208e-04, forward_time=0.129, loss_att=70.131, acc=0.926, loss=70.131, backward_time=0.191, grad_norm=78.173, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.858, time=46 minutes and 18.08 seconds, total_count=89249, gpu_max_cached_mem_GB=23.359, [valid] loss_att=23.554, acc=0.960, cer=0.049, wer=0.171, loss=23.554, time=4 minutes and 0.36 seconds, total_count=2728, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 28.77 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:09:23,234 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:09:23,244 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/21epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:09:23,244 (trainer:272) INFO: 32/60epoch started. Estimated time to finish: 1 day, 1 hour and 15 minutes +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:13:56,848 (trainer:732) INFO: 32epoch:train:1-143batch: iter_time=0.003, forward_time=0.131, loss_att=69.109, acc=0.930, loss=69.109, backward_time=0.194, grad_norm=83.972, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=7.746 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:16:08,877 (trainer:732) INFO: 32epoch:train:144-286batch: iter_time=2.986e-04, forward_time=0.129, loss_att=65.392, acc=0.930, loss=65.392, backward_time=0.191, grad_norm=75.042, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:18:16,242 (trainer:732) INFO: 32epoch:train:287-429batch: iter_time=2.950e-04, forward_time=0.130, loss_att=71.956, acc=0.925, loss=71.956, backward_time=0.193, grad_norm=74.600, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.568 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:20:28,278 (trainer:732) INFO: 32epoch:train:430-572batch: iter_time=3.249e-04, forward_time=0.129, loss_att=68.268, acc=0.928, loss=68.268, backward_time=0.192, grad_norm=75.230, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.679 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:22:41,294 (trainer:732) INFO: 32epoch:train:573-715batch: iter_time=3.065e-04, forward_time=0.129, loss_att=73.208, acc=0.925, loss=73.208, backward_time=0.192, grad_norm=79.595, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:24:53,316 (trainer:732) INFO: 32epoch:train:716-858batch: iter_time=2.960e-04, forward_time=0.128, loss_att=63.044, acc=0.932, loss=63.044, backward_time=0.189, grad_norm=75.945, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:26:57,662 (trainer:732) INFO: 32epoch:train:859-1001batch: iter_time=3.102e-04, forward_time=0.126, loss_att=59.697, acc=0.935, loss=59.697, backward_time=0.187, grad_norm=80.560, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.481 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:29:10,456 (trainer:732) INFO: 32epoch:train:1002-1144batch: iter_time=2.984e-04, forward_time=0.127, loss_att=64.451, acc=0.931, loss=64.451, backward_time=0.188, grad_norm=79.003, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.704 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:31:24,075 (trainer:732) INFO: 32epoch:train:1145-1287batch: iter_time=2.927e-04, forward_time=0.129, loss_att=72.633, acc=0.926, loss=72.633, backward_time=0.193, grad_norm=79.049, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.745 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:33:34,027 (trainer:732) INFO: 32epoch:train:1288-1430batch: iter_time=3.113e-04, forward_time=0.128, loss_att=65.671, acc=0.930, loss=65.671, backward_time=0.190, grad_norm=75.543, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.658 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:35:41,861 (trainer:732) INFO: 32epoch:train:1431-1573batch: iter_time=2.936e-04, forward_time=0.129, loss_att=68.883, acc=0.927, loss=68.883, backward_time=0.192, grad_norm=73.381, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.556 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:37:54,956 (trainer:732) INFO: 32epoch:train:1574-1716batch: iter_time=3.041e-04, forward_time=0.128, loss_att=64.882, acc=0.930, loss=64.882, backward_time=0.190, grad_norm=72.149, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.713 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:40:06,610 (trainer:732) INFO: 32epoch:train:1717-1859batch: iter_time=3.150e-04, forward_time=0.127, loss_att=61.999, acc=0.933, loss=61.999, backward_time=0.190, grad_norm=75.314, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.689 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:42:13,550 (trainer:732) INFO: 32epoch:train:1860-2002batch: iter_time=2.978e-04, forward_time=0.127, loss_att=63.998, acc=0.932, loss=63.998, backward_time=0.189, grad_norm=82.866, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.555 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:44:26,013 (trainer:732) INFO: 32epoch:train:2003-2145batch: iter_time=2.992e-04, forward_time=0.127, loss_att=63.328, acc=0.933, loss=63.328, backward_time=0.190, grad_norm=78.765, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.706 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 140) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 140) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:46:39,514 (trainer:732) INFO: 32epoch:train:2146-2288batch: iter_time=3.130e-04, forward_time=0.129, loss_att=69.328, acc=0.928, loss=69.328, backward_time=0.192, grad_norm=77.895, clip=100.000, loss_scale=1.000, optim_step_time=0.066, optim0_lr0=0.002, train_time=3.723 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:48:51,502 (trainer:732) INFO: 32epoch:train:2289-2431batch: iter_time=2.921e-04, forward_time=0.128, loss_att=66.388, acc=0.929, loss=66.388, backward_time=0.191, grad_norm=77.402, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:50:57,721 (trainer:732) INFO: 32epoch:train:2432-2574batch: iter_time=3.013e-04, forward_time=0.128, loss_att=67.233, acc=0.929, loss=67.233, backward_time=0.190, grad_norm=74.733, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.533 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:53:10,531 (trainer:732) INFO: 32epoch:train:2575-2717batch: iter_time=3.116e-04, forward_time=0.128, loss_att=67.062, acc=0.931, loss=67.062, backward_time=0.191, grad_norm=77.179, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 20:55:23,998 (trainer:732) INFO: 32epoch:train:2718-2860batch: iter_time=2.828e-04, forward_time=0.128, loss_att=70.167, acc=0.927, loss=70.167, backward_time=0.191, grad_norm=75.646, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.722 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:04:16,101 (trainer:338) INFO: 32epoch results: [train] iter_time=4.537e-04, forward_time=0.128, loss_att=66.762, acc=0.929, loss=66.762, backward_time=0.191, grad_norm=77.259, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.861, time=46 minutes and 20.29 seconds, total_count=92128, gpu_max_cached_mem_GB=23.359, [valid] loss_att=22.600, acc=0.962, cer=0.047, wer=0.165, loss=22.600, time=4 minutes and 3.45 seconds, total_count=2816, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 29.11 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:04:20,146 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:04:20,156 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/22epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:04:20,157 (trainer:272) INFO: 33/60epoch started. Estimated time to finish: 1 day, 25 minutes and 32.75 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:08:50,319 (trainer:732) INFO: 33epoch:train:1-143batch: iter_time=0.004, forward_time=0.131, loss_att=68.838, acc=0.928, loss=68.838, backward_time=0.194, grad_norm=86.184, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=7.648 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:11:04,155 (trainer:732) INFO: 33epoch:train:144-286batch: iter_time=2.789e-04, forward_time=0.130, loss_att=66.957, acc=0.931, loss=66.957, backward_time=0.193, grad_norm=77.244, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.746 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:13:16,527 (trainer:732) INFO: 33epoch:train:287-429batch: iter_time=2.663e-04, forward_time=0.127, loss_att=63.228, acc=0.933, loss=63.228, backward_time=0.189, grad_norm=75.096, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:15:22,817 (trainer:732) INFO: 33epoch:train:430-572batch: iter_time=2.883e-04, forward_time=0.128, loss_att=60.853, acc=0.937, loss=60.853, backward_time=0.190, grad_norm=81.649, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.522 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:17:34,548 (trainer:732) INFO: 33epoch:train:573-715batch: iter_time=2.822e-04, forward_time=0.127, loss_att=64.154, acc=0.931, loss=64.154, backward_time=0.189, grad_norm=70.329, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:19:46,974 (trainer:732) INFO: 33epoch:train:716-858batch: iter_time=2.786e-04, forward_time=0.128, loss_att=63.247, acc=0.932, loss=63.247, backward_time=0.190, grad_norm=78.238, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:21:53,955 (trainer:732) INFO: 33epoch:train:859-1001batch: iter_time=2.916e-04, forward_time=0.128, loss_att=62.457, acc=0.934, loss=62.457, backward_time=0.190, grad_norm=81.190, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.546 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:24:06,195 (trainer:732) INFO: 33epoch:train:1002-1144batch: iter_time=2.628e-04, forward_time=0.127, loss_att=60.814, acc=0.936, loss=60.814, backward_time=0.188, grad_norm=80.639, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.688 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:26:18,094 (trainer:732) INFO: 33epoch:train:1145-1287batch: iter_time=2.763e-04, forward_time=0.129, loss_att=68.974, acc=0.928, loss=68.974, backward_time=0.192, grad_norm=75.084, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:28:31,033 (trainer:732) INFO: 33epoch:train:1288-1430batch: iter_time=2.837e-04, forward_time=0.127, loss_att=60.292, acc=0.933, loss=60.292, backward_time=0.189, grad_norm=81.938, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:30:37,665 (trainer:732) INFO: 33epoch:train:1431-1573batch: iter_time=2.918e-04, forward_time=0.128, loss_att=67.064, acc=0.931, loss=67.064, backward_time=0.191, grad_norm=82.121, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.546 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:32:49,666 (trainer:732) INFO: 33epoch:train:1574-1716batch: iter_time=2.835e-04, forward_time=0.128, loss_att=60.074, acc=0.935, loss=60.074, backward_time=0.189, grad_norm=70.833, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.681 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:35:01,153 (trainer:732) INFO: 33epoch:train:1717-1859batch: iter_time=2.831e-04, forward_time=0.127, loss_att=58.171, acc=0.936, loss=58.171, backward_time=0.189, grad_norm=75.963, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.686 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 138) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:37:07,541 (trainer:732) INFO: 33epoch:train:1860-2002batch: iter_time=2.764e-04, forward_time=0.127, loss_att=66.429, acc=0.930, loss=66.429, backward_time=0.189, grad_norm=75.715, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.542 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:39:18,976 (trainer:732) INFO: 33epoch:train:2003-2145batch: iter_time=2.664e-04, forward_time=0.126, loss_att=60.495, acc=0.935, loss=60.495, backward_time=0.187, grad_norm=74.537, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.672 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:41:31,417 (trainer:732) INFO: 33epoch:train:2146-2288batch: iter_time=2.715e-04, forward_time=0.126, loss_att=63.028, acc=0.932, loss=63.028, backward_time=0.188, grad_norm=71.746, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.694 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:43:43,192 (trainer:732) INFO: 33epoch:train:2289-2431batch: iter_time=2.692e-04, forward_time=0.128, loss_att=66.424, acc=0.931, loss=66.424, backward_time=0.191, grad_norm=80.804, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:45:51,188 (trainer:732) INFO: 33epoch:train:2432-2574batch: iter_time=2.685e-04, forward_time=0.129, loss_att=66.187, acc=0.932, loss=66.187, backward_time=0.193, grad_norm=82.173, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.573 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:48:03,041 (trainer:732) INFO: 33epoch:train:2575-2717batch: iter_time=2.677e-04, forward_time=0.128, loss_att=64.058, acc=0.931, loss=64.058, backward_time=0.191, grad_norm=76.254, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:50:18,086 (trainer:732) INFO: 33epoch:train:2718-2860batch: iter_time=2.492e-04, forward_time=0.131, loss_att=73.396, acc=0.928, loss=73.396, backward_time=0.195, grad_norm=80.370, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.768 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:59:06,826 (trainer:338) INFO: 33epoch results: [train] iter_time=4.409e-04, forward_time=0.128, loss_att=64.108, acc=0.932, loss=64.108, backward_time=0.190, grad_norm=77.848, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.856, time=46 minutes and 17.22 seconds, total_count=95007, gpu_max_cached_mem_GB=23.359, [valid] loss_att=21.881, acc=0.963, cer=0.046, wer=0.162, loss=21.881, time=4 minutes and 2.56 seconds, total_count=2904, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 26.88 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:59:10,825 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:59:10,836 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/23epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 21:59:10,836 (trainer:272) INFO: 34/60epoch started. Estimated time to finish: 23 hours, 35 minutes and 15.2 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:03:38,849 (trainer:732) INFO: 34epoch:train:1-143batch: iter_time=0.003, forward_time=0.126, loss_att=55.140, acc=0.941, loss=55.140, backward_time=0.186, grad_norm=69.645, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=7.586 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:05:51,720 (trainer:732) INFO: 34epoch:train:144-286batch: iter_time=2.609e-04, forward_time=0.128, loss_att=60.928, acc=0.936, loss=60.928, backward_time=0.190, grad_norm=78.210, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:08:05,043 (trainer:732) INFO: 34epoch:train:287-429batch: iter_time=2.622e-04, forward_time=0.128, loss_att=62.524, acc=0.935, loss=62.524, backward_time=0.191, grad_norm=76.119, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:10:12,363 (trainer:732) INFO: 34epoch:train:430-572batch: iter_time=2.621e-04, forward_time=0.128, loss_att=62.848, acc=0.935, loss=62.848, backward_time=0.190, grad_norm=77.315, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.552 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:12:23,695 (trainer:732) INFO: 34epoch:train:573-715batch: iter_time=2.584e-04, forward_time=0.128, loss_att=63.093, acc=0.933, loss=63.093, backward_time=0.191, grad_norm=72.290, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.685 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:14:34,458 (trainer:732) INFO: 34epoch:train:716-858batch: iter_time=2.474e-04, forward_time=0.127, loss_att=57.392, acc=0.937, loss=57.392, backward_time=0.188, grad_norm=70.705, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.654 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:16:40,545 (trainer:732) INFO: 34epoch:train:859-1001batch: iter_time=2.652e-04, forward_time=0.128, loss_att=64.030, acc=0.933, loss=64.030, backward_time=0.190, grad_norm=75.117, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.531 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:18:53,074 (trainer:732) INFO: 34epoch:train:1002-1144batch: iter_time=2.485e-04, forward_time=0.129, loss_att=64.570, acc=0.932, loss=64.570, backward_time=0.193, grad_norm=74.560, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:21:04,460 (trainer:732) INFO: 34epoch:train:1145-1287batch: iter_time=2.481e-04, forward_time=0.129, loss_att=62.619, acc=0.935, loss=62.619, backward_time=0.192, grad_norm=71.594, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.680 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:23:15,188 (trainer:732) INFO: 34epoch:train:1288-1430batch: iter_time=2.772e-04, forward_time=0.127, loss_att=61.356, acc=0.935, loss=61.356, backward_time=0.190, grad_norm=72.423, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.661 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:25:22,046 (trainer:732) INFO: 34epoch:train:1431-1573batch: iter_time=2.674e-04, forward_time=0.128, loss_att=65.685, acc=0.932, loss=65.685, backward_time=0.191, grad_norm=78.937, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.548 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:27:35,059 (trainer:732) INFO: 34epoch:train:1574-1716batch: iter_time=2.644e-04, forward_time=0.129, loss_att=66.284, acc=0.931, loss=66.284, backward_time=0.193, grad_norm=83.358, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:29:48,488 (trainer:732) INFO: 34epoch:train:1717-1859batch: iter_time=2.673e-04, forward_time=0.130, loss_att=64.683, acc=0.935, loss=64.683, backward_time=0.195, grad_norm=78.054, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:32:01,059 (trainer:732) INFO: 34epoch:train:1860-2002batch: iter_time=2.543e-04, forward_time=0.128, loss_att=59.572, acc=0.936, loss=59.572, backward_time=0.190, grad_norm=82.562, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:34:06,902 (trainer:732) INFO: 34epoch:train:2003-2145batch: iter_time=2.609e-04, forward_time=0.127, loss_att=62.525, acc=0.933, loss=62.525, backward_time=0.190, grad_norm=82.968, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.527 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:36:18,943 (trainer:732) INFO: 34epoch:train:2146-2288batch: iter_time=2.781e-04, forward_time=0.126, loss_att=58.131, acc=0.936, loss=58.131, backward_time=0.187, grad_norm=69.718, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.684 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:38:29,513 (trainer:732) INFO: 34epoch:train:2289-2431batch: iter_time=2.298e-04, forward_time=0.127, loss_att=59.638, acc=0.937, loss=59.638, backward_time=0.189, grad_norm=74.253, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.664 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:40:35,345 (trainer:732) INFO: 34epoch:train:2432-2574batch: iter_time=2.483e-04, forward_time=0.127, loss_att=60.422, acc=0.935, loss=60.422, backward_time=0.189, grad_norm=73.796, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.517 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:42:47,641 (trainer:732) INFO: 34epoch:train:2575-2717batch: iter_time=2.593e-04, forward_time=0.127, loss_att=58.534, acc=0.937, loss=58.534, backward_time=0.189, grad_norm=75.637, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:45:01,588 (trainer:732) INFO: 34epoch:train:2718-2860batch: iter_time=2.463e-04, forward_time=0.130, loss_att=68.247, acc=0.931, loss=68.247, backward_time=0.194, grad_norm=79.536, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.737 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:53:50,849 (trainer:338) INFO: 34epoch results: [train] iter_time=4.134e-04, forward_time=0.128, loss_att=61.734, acc=0.935, loss=61.734, backward_time=0.190, grad_norm=75.813, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.846, time=46 minutes and 9.78 seconds, total_count=97886, gpu_max_cached_mem_GB=23.359, [valid] loss_att=21.861, acc=0.964, cer=0.045, wer=0.158, loss=21.861, time=4 minutes and 4.41 seconds, total_count=2992, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.81 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:53:54,962 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:53:54,972 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/24epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:53:54,972 (trainer:272) INFO: 35/60epoch started. Estimated time to finish: 22 hours, 44 minutes and 36.59 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 22:58:21,313 (trainer:732) INFO: 35epoch:train:1-143batch: iter_time=0.003, forward_time=0.126, loss_att=54.883, acc=0.941, loss=54.883, backward_time=0.187, grad_norm=74.437, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=7.538 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:00:34,717 (trainer:732) INFO: 35epoch:train:144-286batch: iter_time=2.523e-04, forward_time=0.130, loss_att=62.327, acc=0.935, loss=62.327, backward_time=0.194, grad_norm=82.462, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:02:46,833 (trainer:732) INFO: 35epoch:train:287-429batch: iter_time=2.572e-04, forward_time=0.128, loss_att=62.883, acc=0.935, loss=62.883, backward_time=0.191, grad_norm=83.518, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:04:53,796 (trainer:732) INFO: 35epoch:train:430-572batch: iter_time=2.499e-04, forward_time=0.127, loss_att=53.273, acc=0.942, loss=53.273, backward_time=0.189, grad_norm=75.000, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.545 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:07:05,739 (trainer:732) INFO: 35epoch:train:573-715batch: iter_time=2.465e-04, forward_time=0.128, loss_att=60.097, acc=0.937, loss=60.097, backward_time=0.191, grad_norm=71.509, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:09:17,936 (trainer:732) INFO: 35epoch:train:716-858batch: iter_time=2.572e-04, forward_time=0.126, loss_att=56.869, acc=0.940, loss=56.869, backward_time=0.188, grad_norm=76.752, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.704 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:11:30,225 (trainer:732) INFO: 35epoch:train:859-1001batch: iter_time=2.651e-04, forward_time=0.127, loss_att=55.538, acc=0.941, loss=55.538, backward_time=0.189, grad_norm=76.675, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:13:35,871 (trainer:732) INFO: 35epoch:train:1002-1144batch: iter_time=2.438e-04, forward_time=0.126, loss_att=54.206, acc=0.941, loss=54.206, backward_time=0.187, grad_norm=69.401, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.505 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:15:49,642 (trainer:732) INFO: 35epoch:train:1145-1287batch: iter_time=2.492e-04, forward_time=0.130, loss_att=66.440, acc=0.933, loss=66.440, backward_time=0.194, grad_norm=78.178, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.751 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:18:02,057 (trainer:732) INFO: 35epoch:train:1288-1430batch: iter_time=2.642e-04, forward_time=0.128, loss_att=55.326, acc=0.941, loss=55.326, backward_time=0.191, grad_norm=75.056, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:20:08,871 (trainer:732) INFO: 35epoch:train:1431-1573batch: iter_time=2.437e-04, forward_time=0.128, loss_att=59.700, acc=0.938, loss=59.700, backward_time=0.192, grad_norm=69.290, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.547 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:22:21,235 (trainer:732) INFO: 35epoch:train:1574-1716batch: iter_time=2.689e-04, forward_time=0.127, loss_att=59.736, acc=0.936, loss=59.736, backward_time=0.188, grad_norm=79.354, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:24:33,844 (trainer:732) INFO: 35epoch:train:1717-1859batch: iter_time=2.742e-04, forward_time=0.128, loss_att=62.357, acc=0.935, loss=62.357, backward_time=0.190, grad_norm=79.845, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:26:45,866 (trainer:732) INFO: 35epoch:train:1860-2002batch: iter_time=2.487e-04, forward_time=0.127, loss_att=57.107, acc=0.938, loss=57.107, backward_time=0.189, grad_norm=75.078, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:28:53,016 (trainer:732) INFO: 35epoch:train:2003-2145batch: iter_time=2.538e-04, forward_time=0.128, loss_att=63.034, acc=0.935, loss=63.034, backward_time=0.192, grad_norm=76.478, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.558 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:31:06,106 (trainer:732) INFO: 35epoch:train:2146-2288batch: iter_time=2.682e-04, forward_time=0.128, loss_att=59.381, acc=0.937, loss=59.381, backward_time=0.191, grad_norm=79.557, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:33:18,384 (trainer:732) INFO: 35epoch:train:2289-2431batch: iter_time=2.505e-04, forward_time=0.128, loss_att=62.505, acc=0.933, loss=62.505, backward_time=0.191, grad_norm=79.422, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:35:26,811 (trainer:732) INFO: 35epoch:train:2432-2574batch: iter_time=2.478e-04, forward_time=0.129, loss_att=58.863, acc=0.938, loss=58.863, backward_time=0.192, grad_norm=81.171, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.611 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:37:37,289 (trainer:732) INFO: 35epoch:train:2575-2717batch: iter_time=2.692e-04, forward_time=0.127, loss_att=52.234, acc=0.944, loss=52.234, backward_time=0.188, grad_norm=71.142, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.635 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:39:50,519 (trainer:732) INFO: 35epoch:train:2718-2860batch: iter_time=2.340e-04, forward_time=0.128, loss_att=66.233, acc=0.932, loss=66.233, backward_time=0.191, grad_norm=73.176, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:48:42,466 (trainer:338) INFO: 35epoch results: [train] iter_time=4.111e-04, forward_time=0.128, loss_att=59.059, acc=0.938, loss=59.059, backward_time=0.190, grad_norm=76.311, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.853, time=46 minutes and 14.86 seconds, total_count=100765, gpu_max_cached_mem_GB=23.359, [valid] loss_att=21.738, acc=0.964, cer=0.043, wer=0.156, loss=21.738, time=4 minutes and 6.22 seconds, total_count=3080, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 26.41 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:48:46,361 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:48:46,377 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/25epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:48:46,377 (trainer:272) INFO: 36/60epoch started. Estimated time to finish: 21 hours, 53 minutes and 49.14 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:53:11,967 (trainer:732) INFO: 36epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=50.742, acc=0.942, loss=50.742, backward_time=0.188, grad_norm=73.675, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=7.517 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:55:23,933 (trainer:732) INFO: 36epoch:train:144-286batch: iter_time=2.796e-04, forward_time=0.127, loss_att=58.345, acc=0.939, loss=58.345, backward_time=0.190, grad_norm=70.833, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:57:35,394 (trainer:732) INFO: 36epoch:train:287-429batch: iter_time=2.727e-04, forward_time=0.126, loss_att=55.567, acc=0.940, loss=55.567, backward_time=0.188, grad_norm=67.890, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.678 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-21 23:59:49,278 (trainer:732) INFO: 36epoch:train:430-572batch: iter_time=2.871e-04, forward_time=0.129, loss_att=59.296, acc=0.940, loss=59.296, backward_time=0.193, grad_norm=74.943, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.734 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:01:54,145 (trainer:732) INFO: 36epoch:train:573-715batch: iter_time=2.852e-04, forward_time=0.126, loss_att=54.196, acc=0.941, loss=54.196, backward_time=0.187, grad_norm=69.523, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.493 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:04:05,226 (trainer:732) INFO: 36epoch:train:716-858batch: iter_time=2.749e-04, forward_time=0.126, loss_att=50.627, acc=0.942, loss=50.627, backward_time=0.187, grad_norm=70.448, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.674 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:06:18,961 (trainer:732) INFO: 36epoch:train:859-1001batch: iter_time=2.900e-04, forward_time=0.130, loss_att=57.909, acc=0.939, loss=57.909, backward_time=0.193, grad_norm=78.596, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.739 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:08:26,330 (trainer:732) INFO: 36epoch:train:1002-1144batch: iter_time=2.668e-04, forward_time=0.128, loss_att=62.319, acc=0.935, loss=62.319, backward_time=0.191, grad_norm=69.666, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.553 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:10:37,125 (trainer:732) INFO: 36epoch:train:1145-1287batch: iter_time=2.783e-04, forward_time=0.127, loss_att=56.118, acc=0.940, loss=56.118, backward_time=0.189, grad_norm=76.746, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.672 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:12:51,227 (trainer:732) INFO: 36epoch:train:1288-1430batch: iter_time=2.722e-04, forward_time=0.128, loss_att=65.554, acc=0.935, loss=65.554, backward_time=0.191, grad_norm=72.758, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.745 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:15:03,449 (trainer:732) INFO: 36epoch:train:1431-1573batch: iter_time=2.610e-04, forward_time=0.127, loss_att=55.920, acc=0.940, loss=55.920, backward_time=0.189, grad_norm=72.650, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:17:10,618 (trainer:732) INFO: 36epoch:train:1574-1716batch: iter_time=2.796e-04, forward_time=0.129, loss_att=60.158, acc=0.939, loss=60.158, backward_time=0.193, grad_norm=77.260, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.545 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:19:23,387 (trainer:732) INFO: 36epoch:train:1717-1859batch: iter_time=2.819e-04, forward_time=0.129, loss_att=62.141, acc=0.937, loss=62.141, backward_time=0.193, grad_norm=75.872, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:21:35,674 (trainer:732) INFO: 36epoch:train:1860-2002batch: iter_time=2.738e-04, forward_time=0.128, loss_att=56.442, acc=0.940, loss=56.442, backward_time=0.190, grad_norm=77.350, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:23:41,604 (trainer:732) INFO: 36epoch:train:2003-2145batch: iter_time=2.763e-04, forward_time=0.128, loss_att=54.837, acc=0.942, loss=54.837, backward_time=0.191, grad_norm=79.373, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.521 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:25:53,569 (trainer:732) INFO: 36epoch:train:2146-2288batch: iter_time=2.929e-04, forward_time=0.127, loss_att=55.194, acc=0.942, loss=55.194, backward_time=0.189, grad_norm=77.615, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.684 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:28:05,981 (trainer:732) INFO: 36epoch:train:2289-2431batch: iter_time=2.781e-04, forward_time=0.126, loss_att=56.218, acc=0.940, loss=56.218, backward_time=0.188, grad_norm=83.691, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:30:19,286 (trainer:732) INFO: 36epoch:train:2432-2574batch: iter_time=2.738e-04, forward_time=0.130, loss_att=61.037, acc=0.936, loss=61.037, backward_time=0.195, grad_norm=69.791, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:32:26,454 (trainer:732) INFO: 36epoch:train:2575-2717batch: iter_time=2.737e-04, forward_time=0.128, loss_att=56.510, acc=0.942, loss=56.510, backward_time=0.191, grad_norm=74.840, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.558 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:34:39,714 (trainer:732) INFO: 36epoch:train:2718-2860batch: iter_time=2.613e-04, forward_time=0.128, loss_att=56.715, acc=0.940, loss=56.715, backward_time=0.191, grad_norm=79.740, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:43:31,379 (trainer:338) INFO: 36epoch results: [train] iter_time=4.059e-04, forward_time=0.128, loss_att=57.208, acc=0.940, loss=57.208, backward_time=0.190, grad_norm=74.636, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.850, time=46 minutes and 12.97 seconds, total_count=103644, gpu_max_cached_mem_GB=23.359, [valid] loss_att=21.056, acc=0.965, cer=0.043, wer=0.154, loss=21.056, time=4 minutes and 4.4 seconds, total_count=3168, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.63 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:43:35,550 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:43:35,562 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/26epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:43:35,562 (trainer:272) INFO: 37/60epoch started. Estimated time to finish: 21 hours, 2 minutes and 46.65 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:48:01,178 (trainer:732) INFO: 37epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=57.499, acc=0.941, loss=57.499, backward_time=0.193, grad_norm=73.347, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=7.524 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:50:12,545 (trainer:732) INFO: 37epoch:train:144-286batch: iter_time=2.720e-04, forward_time=0.128, loss_att=53.131, acc=0.945, loss=53.131, backward_time=0.191, grad_norm=83.255, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.671 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:52:25,411 (trainer:732) INFO: 37epoch:train:287-429batch: iter_time=2.580e-04, forward_time=0.128, loss_att=56.933, acc=0.942, loss=56.933, backward_time=0.191, grad_norm=83.439, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:54:38,919 (trainer:732) INFO: 37epoch:train:430-572batch: iter_time=2.713e-04, forward_time=0.128, loss_att=53.246, acc=0.943, loss=53.246, backward_time=0.191, grad_norm=73.937, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:56:45,116 (trainer:732) INFO: 37epoch:train:573-715batch: iter_time=2.737e-04, forward_time=0.129, loss_att=56.399, acc=0.940, loss=56.399, backward_time=0.192, grad_norm=71.461, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.533 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 00:58:57,297 (trainer:732) INFO: 37epoch:train:716-858batch: iter_time=2.492e-04, forward_time=0.126, loss_att=55.130, acc=0.940, loss=55.130, backward_time=0.188, grad_norm=67.959, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:01:09,001 (trainer:732) INFO: 37epoch:train:859-1001batch: iter_time=2.730e-04, forward_time=0.128, loss_att=54.723, acc=0.942, loss=54.723, backward_time=0.192, grad_norm=80.237, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.681 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:03:14,309 (trainer:732) INFO: 37epoch:train:1002-1144batch: iter_time=2.673e-04, forward_time=0.127, loss_att=52.167, acc=0.943, loss=52.167, backward_time=0.188, grad_norm=71.157, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.498 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:05:26,388 (trainer:732) INFO: 37epoch:train:1145-1287batch: iter_time=2.716e-04, forward_time=0.128, loss_att=54.321, acc=0.943, loss=54.321, backward_time=0.191, grad_norm=77.447, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:07:39,227 (trainer:732) INFO: 37epoch:train:1288-1430batch: iter_time=2.918e-04, forward_time=0.126, loss_att=51.052, acc=0.943, loss=51.052, backward_time=0.187, grad_norm=71.557, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:09:51,384 (trainer:732) INFO: 37epoch:train:1431-1573batch: iter_time=2.582e-04, forward_time=0.128, loss_att=54.624, acc=0.942, loss=54.624, backward_time=0.191, grad_norm=72.954, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:11:57,433 (trainer:732) INFO: 37epoch:train:1574-1716batch: iter_time=2.674e-04, forward_time=0.127, loss_att=55.542, acc=0.941, loss=55.542, backward_time=0.189, grad_norm=66.136, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.514 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:14:09,251 (trainer:732) INFO: 37epoch:train:1717-1859batch: iter_time=2.602e-04, forward_time=0.128, loss_att=55.389, acc=0.943, loss=55.389, backward_time=0.191, grad_norm=73.941, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:16:21,977 (trainer:732) INFO: 37epoch:train:1860-2002batch: iter_time=2.513e-04, forward_time=0.128, loss_att=52.967, acc=0.944, loss=52.967, backward_time=0.190, grad_norm=71.529, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:18:28,209 (trainer:732) INFO: 37epoch:train:2003-2145batch: iter_time=2.692e-04, forward_time=0.128, loss_att=56.184, acc=0.941, loss=56.184, backward_time=0.190, grad_norm=77.189, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.534 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:20:39,887 (trainer:732) INFO: 37epoch:train:2146-2288batch: iter_time=2.547e-04, forward_time=0.126, loss_att=52.901, acc=0.943, loss=52.901, backward_time=0.187, grad_norm=69.492, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.670 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:22:52,094 (trainer:732) INFO: 37epoch:train:2289-2431batch: iter_time=2.602e-04, forward_time=0.128, loss_att=56.453, acc=0.942, loss=56.453, backward_time=0.192, grad_norm=71.047, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:25:04,098 (trainer:732) INFO: 37epoch:train:2432-2574batch: iter_time=2.484e-04, forward_time=0.126, loss_att=53.443, acc=0.942, loss=53.443, backward_time=0.188, grad_norm=71.697, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.690 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:27:11,006 (trainer:732) INFO: 37epoch:train:2575-2717batch: iter_time=2.676e-04, forward_time=0.129, loss_att=59.060, acc=0.939, loss=59.060, backward_time=0.192, grad_norm=79.236, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.556 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:29:24,881 (trainer:732) INFO: 37epoch:train:2718-2860batch: iter_time=2.470e-04, forward_time=0.129, loss_att=56.693, acc=0.942, loss=56.693, backward_time=0.193, grad_norm=81.934, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.729 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:38:09,247 (trainer:338) INFO: 37epoch results: [train] iter_time=4.149e-04, forward_time=0.128, loss_att=54.842, acc=0.942, loss=54.842, backward_time=0.190, grad_norm=74.733, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.844, time=46 minutes and 8.74 seconds, total_count=106523, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.633, acc=0.966, cer=0.042, wer=0.151, loss=20.633, time=3 minutes and 58.78 seconds, total_count=3256, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 26.16 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:38:13,662 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:38:13,675 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/27epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:38:13,676 (trainer:272) INFO: 38/60epoch started. Estimated time to finish: 20 hours, 11 minutes and 25.03 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:42:43,611 (trainer:732) INFO: 38epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=52.913, acc=0.945, loss=52.913, backward_time=0.190, grad_norm=75.149, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=7.646 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:44:49,608 (trainer:732) INFO: 38epoch:train:144-286batch: iter_time=2.812e-04, forward_time=0.127, loss_att=53.170, acc=0.943, loss=53.170, backward_time=0.189, grad_norm=66.542, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.521 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:47:00,854 (trainer:732) INFO: 38epoch:train:287-429batch: iter_time=2.599e-04, forward_time=0.126, loss_att=51.235, acc=0.946, loss=51.235, backward_time=0.188, grad_norm=67.619, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.671 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:49:13,387 (trainer:732) INFO: 38epoch:train:430-572batch: iter_time=2.757e-04, forward_time=0.127, loss_att=51.251, acc=0.946, loss=51.251, backward_time=0.190, grad_norm=74.120, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:51:18,955 (trainer:732) INFO: 38epoch:train:573-715batch: iter_time=2.745e-04, forward_time=0.127, loss_att=52.010, acc=0.944, loss=52.010, backward_time=0.190, grad_norm=71.829, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.522 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:53:31,185 (trainer:732) INFO: 38epoch:train:716-858batch: iter_time=2.722e-04, forward_time=0.127, loss_att=52.066, acc=0.944, loss=52.066, backward_time=0.189, grad_norm=68.893, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:55:43,456 (trainer:732) INFO: 38epoch:train:859-1001batch: iter_time=2.836e-04, forward_time=0.128, loss_att=55.218, acc=0.942, loss=55.218, backward_time=0.190, grad_norm=74.040, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:57:55,268 (trainer:732) INFO: 38epoch:train:1002-1144batch: iter_time=2.818e-04, forward_time=0.127, loss_att=51.299, acc=0.945, loss=51.299, backward_time=0.190, grad_norm=76.102, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 01:59:59,437 (trainer:732) INFO: 38epoch:train:1145-1287batch: iter_time=2.766e-04, forward_time=0.127, loss_att=49.873, acc=0.947, loss=49.873, backward_time=0.189, grad_norm=69.242, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.483 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:02:12,012 (trainer:732) INFO: 38epoch:train:1288-1430batch: iter_time=2.820e-04, forward_time=0.127, loss_att=53.325, acc=0.943, loss=53.325, backward_time=0.189, grad_norm=71.467, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:04:23,941 (trainer:732) INFO: 38epoch:train:1431-1573batch: iter_time=2.640e-04, forward_time=0.127, loss_att=51.906, acc=0.945, loss=51.906, backward_time=0.188, grad_norm=67.534, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:06:34,336 (trainer:732) INFO: 38epoch:train:1574-1716batch: iter_time=2.709e-04, forward_time=0.129, loss_att=57.236, acc=0.942, loss=57.236, backward_time=0.193, grad_norm=81.710, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.636 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:08:44,057 (trainer:732) INFO: 38epoch:train:1717-1859batch: iter_time=2.888e-04, forward_time=0.128, loss_att=51.248, acc=0.946, loss=51.248, backward_time=0.190, grad_norm=66.083, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.635 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:10:56,915 (trainer:732) INFO: 38epoch:train:1860-2002batch: iter_time=2.796e-04, forward_time=0.129, loss_att=55.740, acc=0.943, loss=55.740, backward_time=0.192, grad_norm=71.346, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:13:09,438 (trainer:732) INFO: 38epoch:train:2003-2145batch: iter_time=2.732e-04, forward_time=0.127, loss_att=50.644, acc=0.944, loss=50.644, backward_time=0.189, grad_norm=72.970, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:15:16,791 (trainer:732) INFO: 38epoch:train:2146-2288batch: iter_time=2.795e-04, forward_time=0.129, loss_att=55.639, acc=0.944, loss=55.639, backward_time=0.193, grad_norm=74.222, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.553 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:17:29,211 (trainer:732) INFO: 38epoch:train:2289-2431batch: iter_time=2.683e-04, forward_time=0.128, loss_att=52.846, acc=0.944, loss=52.846, backward_time=0.191, grad_norm=75.929, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:19:41,927 (trainer:732) INFO: 38epoch:train:2432-2574batch: iter_time=2.766e-04, forward_time=0.129, loss_att=53.877, acc=0.944, loss=53.877, backward_time=0.193, grad_norm=72.496, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:21:52,600 (trainer:732) INFO: 38epoch:train:2575-2717batch: iter_time=2.872e-04, forward_time=0.129, loss_att=57.545, acc=0.940, loss=57.545, backward_time=0.193, grad_norm=74.256, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.660 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:24:02,858 (trainer:732) INFO: 38epoch:train:2718-2860batch: iter_time=2.581e-04, forward_time=0.128, loss_att=52.592, acc=0.945, loss=52.592, backward_time=0.191, grad_norm=74.126, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.628 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:32:54,697 (trainer:338) INFO: 38epoch results: [train] iter_time=3.836e-04, forward_time=0.128, loss_att=53.010, acc=0.944, loss=53.010, backward_time=0.190, grad_norm=72.178, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.843, time=46 minutes and 8.28 seconds, total_count=109402, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.816, acc=0.966, cer=0.042, wer=0.150, loss=20.816, time=4 minutes and 3.88 seconds, total_count=3344, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 28.85 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:32:58,702 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:32:58,716 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/28epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:32:58,716 (trainer:272) INFO: 39/60epoch started. Estimated time to finish: 19 hours, 19 minutes and 57.07 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:37:30,013 (trainer:732) INFO: 39epoch:train:1-143batch: iter_time=0.003, forward_time=0.129, loss_att=49.644, acc=0.947, loss=49.644, backward_time=0.192, grad_norm=73.736, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=7.682 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:39:36,372 (trainer:732) INFO: 39epoch:train:144-286batch: iter_time=2.787e-04, forward_time=0.127, loss_att=50.013, acc=0.945, loss=50.013, backward_time=0.189, grad_norm=69.246, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.537 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:41:50,277 (trainer:732) INFO: 39epoch:train:287-429batch: iter_time=2.754e-04, forward_time=0.130, loss_att=53.603, acc=0.945, loss=53.603, backward_time=0.193, grad_norm=79.991, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.741 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:44:04,863 (trainer:732) INFO: 39epoch:train:430-572batch: iter_time=2.793e-04, forward_time=0.132, loss_att=58.405, acc=0.941, loss=58.405, backward_time=0.197, grad_norm=77.828, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.756 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:46:16,593 (trainer:732) INFO: 39epoch:train:573-715batch: iter_time=2.537e-04, forward_time=0.127, loss_att=45.346, acc=0.951, loss=45.346, backward_time=0.188, grad_norm=68.958, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.686 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:48:23,565 (trainer:732) INFO: 39epoch:train:716-858batch: iter_time=2.565e-04, forward_time=0.127, loss_att=51.580, acc=0.946, loss=51.580, backward_time=0.190, grad_norm=80.916, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.561 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:50:36,011 (trainer:732) INFO: 39epoch:train:859-1001batch: iter_time=2.668e-04, forward_time=0.126, loss_att=49.211, acc=0.948, loss=49.211, backward_time=0.188, grad_norm=72.484, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.703 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:52:47,205 (trainer:732) INFO: 39epoch:train:1002-1144batch: iter_time=2.504e-04, forward_time=0.126, loss_att=45.014, acc=0.950, loss=45.014, backward_time=0.187, grad_norm=70.294, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.660 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:54:52,427 (trainer:732) INFO: 39epoch:train:1145-1287batch: iter_time=2.708e-04, forward_time=0.126, loss_att=49.756, acc=0.945, loss=49.756, backward_time=0.186, grad_norm=66.491, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.515 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:57:05,391 (trainer:732) INFO: 39epoch:train:1288-1430batch: iter_time=2.621e-04, forward_time=0.129, loss_att=54.588, acc=0.945, loss=54.588, backward_time=0.193, grad_norm=74.607, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.713 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 02:59:17,956 (trainer:732) INFO: 39epoch:train:1431-1573batch: iter_time=2.611e-04, forward_time=0.128, loss_att=49.827, acc=0.948, loss=49.827, backward_time=0.191, grad_norm=71.988, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:01:30,627 (trainer:732) INFO: 39epoch:train:1574-1716batch: iter_time=2.678e-04, forward_time=0.127, loss_att=47.341, acc=0.951, loss=47.341, backward_time=0.190, grad_norm=69.956, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:03:36,022 (trainer:732) INFO: 39epoch:train:1717-1859batch: iter_time=2.579e-04, forward_time=0.127, loss_att=49.857, acc=0.946, loss=49.857, backward_time=0.189, grad_norm=74.779, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.512 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:05:47,247 (trainer:732) INFO: 39epoch:train:1860-2002batch: iter_time=2.436e-04, forward_time=0.127, loss_att=50.388, acc=0.946, loss=50.388, backward_time=0.189, grad_norm=74.941, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.675 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:07:59,556 (trainer:732) INFO: 39epoch:train:2003-2145batch: iter_time=2.623e-04, forward_time=0.128, loss_att=49.184, acc=0.948, loss=49.184, backward_time=0.190, grad_norm=70.934, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:10:06,717 (trainer:732) INFO: 39epoch:train:2146-2288batch: iter_time=2.812e-04, forward_time=0.128, loss_att=54.918, acc=0.943, loss=54.918, backward_time=0.191, grad_norm=73.702, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.548 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:12:19,479 (trainer:732) INFO: 39epoch:train:2289-2431batch: iter_time=2.496e-04, forward_time=0.129, loss_att=55.398, acc=0.944, loss=55.398, backward_time=0.193, grad_norm=82.337, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.728 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:14:32,623 (trainer:732) INFO: 39epoch:train:2432-2574batch: iter_time=2.737e-04, forward_time=0.128, loss_att=52.099, acc=0.946, loss=52.099, backward_time=0.190, grad_norm=73.258, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:16:45,687 (trainer:732) INFO: 39epoch:train:2575-2717batch: iter_time=2.656e-04, forward_time=0.129, loss_att=54.890, acc=0.944, loss=54.890, backward_time=0.193, grad_norm=72.345, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:18:52,791 (trainer:732) INFO: 39epoch:train:2718-2860batch: iter_time=2.689e-04, forward_time=0.128, loss_att=48.727, acc=0.948, loss=48.727, backward_time=0.190, grad_norm=71.506, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.546 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:27:41,169 (trainer:338) INFO: 39epoch results: [train] iter_time=3.904e-04, forward_time=0.128, loss_att=50.871, acc=0.946, loss=50.871, backward_time=0.190, grad_norm=73.464, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.850, time=46 minutes and 13.22 seconds, total_count=112281, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.867, acc=0.967, cer=0.040, wer=0.145, loss=19.867, time=4 minutes and 2.52 seconds, total_count=3432, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 26.7 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:27:45,023 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:27:45,037 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/30epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:27:45,037 (trainer:272) INFO: 40/60epoch started. Estimated time to finish: 18 hours, 28 minutes and 19.7 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:32:14,959 (trainer:732) INFO: 40epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=49.455, acc=0.948, loss=49.455, backward_time=0.192, grad_norm=75.959, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=7.643 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:34:22,197 (trainer:732) INFO: 40epoch:train:144-286batch: iter_time=2.712e-04, forward_time=0.127, loss_att=48.874, acc=0.948, loss=48.874, backward_time=0.189, grad_norm=74.102, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.564 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:36:34,779 (trainer:732) INFO: 40epoch:train:287-429batch: iter_time=2.576e-04, forward_time=0.129, loss_att=48.997, acc=0.951, loss=48.997, backward_time=0.192, grad_norm=73.073, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.704 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:38:47,441 (trainer:732) INFO: 40epoch:train:430-572batch: iter_time=2.667e-04, forward_time=0.127, loss_att=47.691, acc=0.948, loss=47.691, backward_time=0.189, grad_norm=72.959, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:41:01,700 (trainer:732) INFO: 40epoch:train:573-715batch: iter_time=2.727e-04, forward_time=0.131, loss_att=55.147, acc=0.945, loss=55.147, backward_time=0.196, grad_norm=75.538, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.764 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:43:06,530 (trainer:732) INFO: 40epoch:train:716-858batch: iter_time=2.488e-04, forward_time=0.126, loss_att=49.612, acc=0.947, loss=49.612, backward_time=0.188, grad_norm=68.304, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.495 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:45:18,703 (trainer:732) INFO: 40epoch:train:859-1001batch: iter_time=2.626e-04, forward_time=0.127, loss_att=49.350, acc=0.946, loss=49.350, backward_time=0.190, grad_norm=64.670, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:47:30,866 (trainer:732) INFO: 40epoch:train:1002-1144batch: iter_time=2.610e-04, forward_time=0.127, loss_att=50.047, acc=0.947, loss=50.047, backward_time=0.190, grad_norm=68.703, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:49:36,485 (trainer:732) INFO: 40epoch:train:1145-1287batch: iter_time=2.934e-04, forward_time=0.128, loss_att=46.916, acc=0.950, loss=46.916, backward_time=0.190, grad_norm=71.767, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.517 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:51:48,570 (trainer:732) INFO: 40epoch:train:1288-1430batch: iter_time=2.815e-04, forward_time=0.126, loss_att=45.275, acc=0.950, loss=45.275, backward_time=0.188, grad_norm=69.072, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:54:01,214 (trainer:732) INFO: 40epoch:train:1431-1573batch: iter_time=2.647e-04, forward_time=0.128, loss_att=48.698, acc=0.950, loss=48.698, backward_time=0.192, grad_norm=69.638, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:56:13,494 (trainer:732) INFO: 40epoch:train:1574-1716batch: iter_time=2.668e-04, forward_time=0.126, loss_att=49.156, acc=0.947, loss=49.156, backward_time=0.188, grad_norm=76.097, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 03:58:19,732 (trainer:732) INFO: 40epoch:train:1717-1859batch: iter_time=2.894e-04, forward_time=0.128, loss_att=46.259, acc=0.950, loss=46.259, backward_time=0.191, grad_norm=72.101, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.544 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:00:31,600 (trainer:732) INFO: 40epoch:train:1860-2002batch: iter_time=2.536e-04, forward_time=0.127, loss_att=48.070, acc=0.948, loss=48.070, backward_time=0.188, grad_norm=79.991, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.683 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:02:44,176 (trainer:732) INFO: 40epoch:train:2003-2145batch: iter_time=2.747e-04, forward_time=0.128, loss_att=51.331, acc=0.947, loss=51.331, backward_time=0.192, grad_norm=81.361, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:04:52,347 (trainer:732) INFO: 40epoch:train:2146-2288batch: iter_time=2.616e-04, forward_time=0.129, loss_att=52.949, acc=0.946, loss=52.949, backward_time=0.192, grad_norm=65.878, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.575 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:07:04,236 (trainer:732) INFO: 40epoch:train:2289-2431batch: iter_time=2.481e-04, forward_time=0.129, loss_att=55.836, acc=0.942, loss=55.836, backward_time=0.193, grad_norm=71.450, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:09:16,583 (trainer:732) INFO: 40epoch:train:2432-2574batch: iter_time=2.649e-04, forward_time=0.129, loss_att=49.832, acc=0.949, loss=49.832, backward_time=0.193, grad_norm=76.604, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:11:27,346 (trainer:732) INFO: 40epoch:train:2575-2717batch: iter_time=2.763e-04, forward_time=0.126, loss_att=44.395, acc=0.950, loss=44.395, backward_time=0.187, grad_norm=71.233, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.651 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:13:34,876 (trainer:732) INFO: 40epoch:train:2718-2860batch: iter_time=2.390e-04, forward_time=0.127, loss_att=50.613, acc=0.946, loss=50.613, backward_time=0.190, grad_norm=68.499, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.559 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:22:24,216 (trainer:338) INFO: 40epoch results: [train] iter_time=3.857e-04, forward_time=0.128, loss_att=49.346, acc=0.948, loss=49.346, backward_time=0.190, grad_norm=72.350, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.845, time=46 minutes and 8.92 seconds, total_count=115160, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.451, acc=0.967, cer=0.041, wer=0.146, loss=20.451, time=4 minutes and 4.02 seconds, total_count=3520, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 26.24 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:22:27,972 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:22:27,982 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/29epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:22:27,982 (trainer:272) INFO: 41/60epoch started. Estimated time to finish: 17 hours, 36 minutes and 31.2 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:26:58,302 (trainer:732) INFO: 41epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=45.496, acc=0.952, loss=45.496, backward_time=0.190, grad_norm=70.465, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=7.652 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:29:10,450 (trainer:732) INFO: 41epoch:train:144-286batch: iter_time=2.645e-04, forward_time=0.128, loss_att=45.605, acc=0.951, loss=45.605, backward_time=0.191, grad_norm=67.381, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:31:17,445 (trainer:732) INFO: 41epoch:train:287-429batch: iter_time=2.533e-04, forward_time=0.128, loss_att=48.956, acc=0.949, loss=48.956, backward_time=0.190, grad_norm=70.155, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.552 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:33:30,508 (trainer:732) INFO: 41epoch:train:430-572batch: iter_time=2.634e-04, forward_time=0.128, loss_att=48.405, acc=0.950, loss=48.405, backward_time=0.191, grad_norm=70.587, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:35:42,635 (trainer:732) INFO: 41epoch:train:573-715batch: iter_time=2.846e-04, forward_time=0.128, loss_att=45.624, acc=0.952, loss=45.624, backward_time=0.190, grad_norm=80.247, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:37:48,554 (trainer:732) INFO: 41epoch:train:716-858batch: iter_time=2.686e-04, forward_time=0.126, loss_att=43.303, acc=0.952, loss=43.303, backward_time=0.188, grad_norm=64.688, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.520 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:40:01,578 (trainer:732) INFO: 41epoch:train:859-1001batch: iter_time=2.633e-04, forward_time=0.130, loss_att=49.582, acc=0.950, loss=49.582, backward_time=0.194, grad_norm=81.421, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:42:13,595 (trainer:732) INFO: 41epoch:train:1002-1144batch: iter_time=2.485e-04, forward_time=0.128, loss_att=48.970, acc=0.948, loss=48.970, backward_time=0.190, grad_norm=68.723, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.682 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:44:25,009 (trainer:732) INFO: 41epoch:train:1145-1287batch: iter_time=2.512e-04, forward_time=0.126, loss_att=44.438, acc=0.951, loss=44.438, backward_time=0.188, grad_norm=65.129, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:46:30,445 (trainer:732) INFO: 41epoch:train:1288-1430batch: iter_time=2.650e-04, forward_time=0.126, loss_att=45.120, acc=0.950, loss=45.120, backward_time=0.188, grad_norm=65.524, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.513 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:48:41,431 (trainer:732) INFO: 41epoch:train:1431-1573batch: iter_time=2.524e-04, forward_time=0.126, loss_att=43.599, acc=0.954, loss=43.599, backward_time=0.188, grad_norm=70.840, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.658 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:50:55,264 (trainer:732) INFO: 41epoch:train:1574-1716batch: iter_time=2.691e-04, forward_time=0.129, loss_att=52.117, acc=0.948, loss=52.117, backward_time=0.192, grad_norm=72.386, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.734 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:53:02,010 (trainer:732) INFO: 41epoch:train:1717-1859batch: iter_time=2.606e-04, forward_time=0.129, loss_att=51.972, acc=0.948, loss=51.972, backward_time=0.192, grad_norm=75.721, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.551 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:55:14,386 (trainer:732) INFO: 41epoch:train:1860-2002batch: iter_time=2.636e-04, forward_time=0.128, loss_att=51.750, acc=0.947, loss=51.750, backward_time=0.191, grad_norm=77.859, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.704 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:57:26,591 (trainer:732) INFO: 41epoch:train:2003-2145batch: iter_time=2.571e-04, forward_time=0.126, loss_att=48.373, acc=0.948, loss=48.373, backward_time=0.188, grad_norm=73.328, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 04:59:39,719 (trainer:732) INFO: 41epoch:train:2146-2288batch: iter_time=2.662e-04, forward_time=0.129, loss_att=51.677, acc=0.947, loss=51.677, backward_time=0.192, grad_norm=71.009, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:01:44,937 (trainer:732) INFO: 41epoch:train:2289-2431batch: iter_time=2.403e-04, forward_time=0.127, loss_att=45.079, acc=0.950, loss=45.079, backward_time=0.189, grad_norm=73.871, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.508 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:03:58,172 (trainer:732) INFO: 41epoch:train:2432-2574batch: iter_time=2.599e-04, forward_time=0.128, loss_att=49.215, acc=0.948, loss=49.215, backward_time=0.191, grad_norm=72.801, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.726 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:06:11,051 (trainer:732) INFO: 41epoch:train:2575-2717batch: iter_time=2.508e-04, forward_time=0.129, loss_att=47.314, acc=0.950, loss=47.314, backward_time=0.192, grad_norm=71.082, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:08:18,088 (trainer:732) INFO: 41epoch:train:2718-2860batch: iter_time=2.304e-04, forward_time=0.128, loss_att=48.345, acc=0.949, loss=48.345, backward_time=0.191, grad_norm=75.251, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.545 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:17:02,756 (trainer:338) INFO: 41epoch results: [train] iter_time=3.599e-04, forward_time=0.128, loss_att=47.733, acc=0.950, loss=47.733, backward_time=0.190, grad_norm=71.959, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.845, time=46 minutes and 9.55 seconds, total_count=118039, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.408, acc=0.967, cer=0.040, wer=0.144, loss=20.408, time=3 minutes and 57.99 seconds, total_count=3608, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.23 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:17:06,977 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:17:06,987 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/31epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:17:06,987 (trainer:272) INFO: 42/60epoch started. Estimated time to finish: 16 hours, 44 minutes and 32.36 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:21:35,780 (trainer:732) INFO: 42epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=42.817, acc=0.954, loss=42.817, backward_time=0.189, grad_norm=69.338, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=7.614 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:23:49,818 (trainer:732) INFO: 42epoch:train:144-286batch: iter_time=2.715e-04, forward_time=0.129, loss_att=51.594, acc=0.948, loss=51.594, backward_time=0.193, grad_norm=76.734, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.745 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:25:56,286 (trainer:732) INFO: 42epoch:train:287-429batch: iter_time=2.697e-04, forward_time=0.129, loss_att=45.179, acc=0.953, loss=45.179, backward_time=0.192, grad_norm=74.465, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.537 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:28:08,978 (trainer:732) INFO: 42epoch:train:430-572batch: iter_time=2.863e-04, forward_time=0.127, loss_att=43.984, acc=0.953, loss=43.984, backward_time=0.189, grad_norm=71.843, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.702 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:30:20,598 (trainer:732) INFO: 42epoch:train:573-715batch: iter_time=2.722e-04, forward_time=0.127, loss_att=44.138, acc=0.952, loss=44.138, backward_time=0.189, grad_norm=67.886, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:32:32,626 (trainer:732) INFO: 42epoch:train:716-858batch: iter_time=2.690e-04, forward_time=0.130, loss_att=49.623, acc=0.949, loss=49.623, backward_time=0.193, grad_norm=66.632, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.688 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:34:38,700 (trainer:732) INFO: 42epoch:train:859-1001batch: iter_time=2.712e-04, forward_time=0.128, loss_att=48.248, acc=0.949, loss=48.248, backward_time=0.191, grad_norm=67.565, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.527 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:36:53,680 (trainer:732) INFO: 42epoch:train:1002-1144batch: iter_time=2.727e-04, forward_time=0.130, loss_att=49.686, acc=0.950, loss=49.686, backward_time=0.195, grad_norm=77.536, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.765 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:39:05,128 (trainer:732) INFO: 42epoch:train:1145-1287batch: iter_time=2.607e-04, forward_time=0.126, loss_att=43.137, acc=0.953, loss=43.137, backward_time=0.187, grad_norm=68.782, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:41:10,070 (trainer:732) INFO: 42epoch:train:1288-1430batch: iter_time=2.684e-04, forward_time=0.126, loss_att=43.813, acc=0.952, loss=43.813, backward_time=0.188, grad_norm=66.160, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.491 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:43:21,329 (trainer:732) INFO: 42epoch:train:1431-1573batch: iter_time=2.812e-04, forward_time=0.126, loss_att=42.437, acc=0.954, loss=42.437, backward_time=0.186, grad_norm=63.436, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.670 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:45:34,247 (trainer:732) INFO: 42epoch:train:1574-1716batch: iter_time=2.747e-04, forward_time=0.128, loss_att=46.940, acc=0.950, loss=46.940, backward_time=0.191, grad_norm=67.991, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:47:46,749 (trainer:732) INFO: 42epoch:train:1717-1859batch: iter_time=2.707e-04, forward_time=0.128, loss_att=48.091, acc=0.950, loss=48.091, backward_time=0.190, grad_norm=68.992, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:49:53,674 (trainer:732) INFO: 42epoch:train:1860-2002batch: iter_time=2.773e-04, forward_time=0.129, loss_att=43.911, acc=0.953, loss=43.911, backward_time=0.191, grad_norm=77.249, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.554 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:52:08,780 (trainer:732) INFO: 42epoch:train:2003-2145batch: iter_time=2.733e-04, forward_time=0.130, loss_att=49.789, acc=0.948, loss=49.789, backward_time=0.194, grad_norm=75.392, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.778 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:54:23,171 (trainer:732) INFO: 42epoch:train:2146-2288batch: iter_time=2.782e-04, forward_time=0.128, loss_att=48.724, acc=0.949, loss=48.724, backward_time=0.192, grad_norm=73.887, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.746 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:56:28,458 (trainer:732) INFO: 42epoch:train:2289-2431batch: iter_time=2.527e-04, forward_time=0.127, loss_att=46.375, acc=0.950, loss=46.375, backward_time=0.189, grad_norm=69.609, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.514 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 05:58:42,000 (trainer:732) INFO: 42epoch:train:2432-2574batch: iter_time=2.546e-04, forward_time=0.128, loss_att=47.654, acc=0.952, loss=47.654, backward_time=0.191, grad_norm=76.495, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:00:52,429 (trainer:732) INFO: 42epoch:train:2575-2717batch: iter_time=2.618e-04, forward_time=0.126, loss_att=43.706, acc=0.952, loss=43.706, backward_time=0.187, grad_norm=65.222, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.647 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:03:03,698 (trainer:732) INFO: 42epoch:train:2718-2860batch: iter_time=2.553e-04, forward_time=0.127, loss_att=44.223, acc=0.953, loss=44.223, backward_time=0.188, grad_norm=69.730, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.661 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:11:52,172 (trainer:338) INFO: 42epoch results: [train] iter_time=3.944e-04, forward_time=0.128, loss_att=46.127, acc=0.951, loss=46.127, backward_time=0.190, grad_norm=70.745, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.854, time=46 minutes and 15.89 seconds, total_count=120918, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.797, acc=0.968, cer=0.039, wer=0.141, loss=19.797, time=4 minutes and 3.4 seconds, total_count=3696, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.9 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:11:56,120 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:11:56,131 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/32epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:11:56,131 (trainer:272) INFO: 43/60epoch started. Estimated time to finish: 15 hours, 52 minutes and 30.23 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:16:28,399 (trainer:732) INFO: 43epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=44.353, acc=0.954, loss=44.353, backward_time=0.189, grad_norm=71.631, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=7.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:18:39,496 (trainer:732) INFO: 43epoch:train:144-286batch: iter_time=2.581e-04, forward_time=0.125, loss_att=37.775, acc=0.956, loss=37.775, backward_time=0.185, grad_norm=59.394, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.663 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:20:44,672 (trainer:732) INFO: 43epoch:train:287-429batch: iter_time=2.477e-04, forward_time=0.127, loss_att=42.388, acc=0.953, loss=42.388, backward_time=0.188, grad_norm=72.102, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.507 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:22:57,696 (trainer:732) INFO: 43epoch:train:430-572batch: iter_time=2.830e-04, forward_time=0.129, loss_att=47.233, acc=0.952, loss=47.233, backward_time=0.193, grad_norm=76.735, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:25:09,806 (trainer:732) INFO: 43epoch:train:573-715batch: iter_time=2.673e-04, forward_time=0.127, loss_att=43.694, acc=0.954, loss=43.694, backward_time=0.190, grad_norm=74.393, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:27:22,202 (trainer:732) INFO: 43epoch:train:716-858batch: iter_time=2.457e-04, forward_time=0.127, loss_att=44.914, acc=0.952, loss=44.914, backward_time=0.189, grad_norm=66.587, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:29:29,128 (trainer:732) INFO: 43epoch:train:859-1001batch: iter_time=2.645e-04, forward_time=0.129, loss_att=46.719, acc=0.953, loss=46.719, backward_time=0.193, grad_norm=79.867, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.553 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:31:42,730 (trainer:732) INFO: 43epoch:train:1002-1144batch: iter_time=2.648e-04, forward_time=0.130, loss_att=47.985, acc=0.951, loss=47.985, backward_time=0.194, grad_norm=70.245, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.726 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:33:53,816 (trainer:732) INFO: 43epoch:train:1145-1287batch: iter_time=2.579e-04, forward_time=0.127, loss_att=43.521, acc=0.953, loss=43.521, backward_time=0.189, grad_norm=71.414, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:35:59,153 (trainer:732) INFO: 43epoch:train:1288-1430batch: iter_time=2.567e-04, forward_time=0.126, loss_att=42.586, acc=0.953, loss=42.586, backward_time=0.188, grad_norm=63.446, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.511 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:38:12,243 (trainer:732) INFO: 43epoch:train:1431-1573batch: iter_time=2.591e-04, forward_time=0.129, loss_att=49.472, acc=0.949, loss=49.472, backward_time=0.193, grad_norm=71.438, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:40:25,962 (trainer:732) INFO: 43epoch:train:1574-1716batch: iter_time=2.637e-04, forward_time=0.129, loss_att=50.722, acc=0.949, loss=50.722, backward_time=0.193, grad_norm=69.685, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.732 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:42:38,030 (trainer:732) INFO: 43epoch:train:1717-1859batch: iter_time=2.644e-04, forward_time=0.129, loss_att=45.859, acc=0.952, loss=45.859, backward_time=0.192, grad_norm=69.542, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:44:45,874 (trainer:732) INFO: 43epoch:train:1860-2002batch: iter_time=2.662e-04, forward_time=0.129, loss_att=46.022, acc=0.953, loss=46.022, backward_time=0.192, grad_norm=70.349, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.568 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:46:56,863 (trainer:732) INFO: 43epoch:train:2003-2145batch: iter_time=2.776e-04, forward_time=0.126, loss_att=41.903, acc=0.953, loss=41.903, backward_time=0.187, grad_norm=63.535, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.663 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:49:10,195 (trainer:732) INFO: 43epoch:train:2146-2288batch: iter_time=2.685e-04, forward_time=0.129, loss_att=46.499, acc=0.953, loss=46.499, backward_time=0.193, grad_norm=80.468, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.719 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:51:15,439 (trainer:732) INFO: 43epoch:train:2289-2431batch: iter_time=2.364e-04, forward_time=0.125, loss_att=37.155, acc=0.958, loss=37.155, backward_time=0.186, grad_norm=65.821, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.506 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:53:28,622 (trainer:732) INFO: 43epoch:train:2432-2574batch: iter_time=2.574e-04, forward_time=0.127, loss_att=44.144, acc=0.953, loss=44.144, backward_time=0.189, grad_norm=66.055, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.734 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:55:42,439 (trainer:732) INFO: 43epoch:train:2575-2717batch: iter_time=2.488e-04, forward_time=0.129, loss_att=48.695, acc=0.950, loss=48.695, backward_time=0.192, grad_norm=72.934, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 06:57:55,972 (trainer:732) INFO: 43epoch:train:2718-2860batch: iter_time=2.434e-04, forward_time=0.128, loss_att=42.663, acc=0.955, loss=42.663, backward_time=0.192, grad_norm=74.261, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:06:44,728 (trainer:338) INFO: 43epoch results: [train] iter_time=3.689e-04, forward_time=0.128, loss_att=44.649, acc=0.953, loss=44.649, backward_time=0.190, grad_norm=70.588, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.859, time=46 minutes and 19.16 seconds, total_count=123797, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.183, acc=0.967, cer=0.040, wer=0.143, loss=20.183, time=4 minutes and 3.5 seconds, total_count=3784, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.93 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:06:48,791 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:06:48,802 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/33epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:06:48,802 (trainer:272) INFO: 44/60epoch started. Estimated time to finish: 15 hours and 21.74 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:11:22,470 (trainer:732) INFO: 44epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=42.342, acc=0.955, loss=42.342, backward_time=0.190, grad_norm=65.688, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=7.744 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:13:34,991 (trainer:732) INFO: 44epoch:train:144-286batch: iter_time=2.638e-04, forward_time=0.127, loss_att=40.117, acc=0.956, loss=40.117, backward_time=0.189, grad_norm=68.286, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:15:47,733 (trainer:732) INFO: 44epoch:train:287-429batch: iter_time=2.616e-04, forward_time=0.128, loss_att=44.857, acc=0.952, loss=44.857, backward_time=0.190, grad_norm=64.978, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:17:55,067 (trainer:732) INFO: 44epoch:train:430-572batch: iter_time=2.637e-04, forward_time=0.127, loss_att=41.277, acc=0.957, loss=41.277, backward_time=0.189, grad_norm=72.750, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.553 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:20:06,809 (trainer:732) INFO: 44epoch:train:573-715batch: iter_time=2.647e-04, forward_time=0.128, loss_att=43.471, acc=0.953, loss=43.471, backward_time=0.191, grad_norm=69.878, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:22:18,533 (trainer:732) INFO: 44epoch:train:716-858batch: iter_time=2.670e-04, forward_time=0.126, loss_att=41.540, acc=0.955, loss=41.540, backward_time=0.187, grad_norm=64.127, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.683 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:24:26,177 (trainer:732) INFO: 44epoch:train:859-1001batch: iter_time=2.780e-04, forward_time=0.129, loss_att=46.674, acc=0.954, loss=46.674, backward_time=0.192, grad_norm=71.438, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.566 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:26:38,941 (trainer:732) INFO: 44epoch:train:1002-1144batch: iter_time=2.518e-04, forward_time=0.129, loss_att=41.485, acc=0.956, loss=41.485, backward_time=0.192, grad_norm=72.134, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:28:50,855 (trainer:732) INFO: 44epoch:train:1145-1287batch: iter_time=2.705e-04, forward_time=0.128, loss_att=42.555, acc=0.955, loss=42.555, backward_time=0.190, grad_norm=74.559, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:31:02,981 (trainer:732) INFO: 44epoch:train:1288-1430batch: iter_time=2.642e-04, forward_time=0.129, loss_att=45.397, acc=0.952, loss=45.397, backward_time=0.193, grad_norm=75.497, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:33:08,557 (trainer:732) INFO: 44epoch:train:1431-1573batch: iter_time=2.618e-04, forward_time=0.127, loss_att=42.534, acc=0.955, loss=42.534, backward_time=0.189, grad_norm=73.470, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.512 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:35:21,411 (trainer:732) INFO: 44epoch:train:1574-1716batch: iter_time=2.743e-04, forward_time=0.128, loss_att=44.181, acc=0.954, loss=44.181, backward_time=0.191, grad_norm=67.850, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:37:33,611 (trainer:732) INFO: 44epoch:train:1717-1859batch: iter_time=2.726e-04, forward_time=0.128, loss_att=42.694, acc=0.954, loss=42.694, backward_time=0.190, grad_norm=65.879, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:39:40,100 (trainer:732) INFO: 44epoch:train:1860-2002batch: iter_time=2.597e-04, forward_time=0.128, loss_att=47.093, acc=0.951, loss=47.093, backward_time=0.191, grad_norm=67.111, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.538 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:41:51,467 (trainer:732) INFO: 44epoch:train:2003-2145batch: iter_time=2.663e-04, forward_time=0.128, loss_att=42.251, acc=0.956, loss=42.251, backward_time=0.191, grad_norm=68.250, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:44:02,335 (trainer:732) INFO: 44epoch:train:2146-2288batch: iter_time=2.688e-04, forward_time=0.126, loss_att=39.931, acc=0.956, loss=39.931, backward_time=0.187, grad_norm=66.230, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.652 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:46:15,010 (trainer:732) INFO: 44epoch:train:2289-2431batch: iter_time=2.608e-04, forward_time=0.129, loss_att=46.880, acc=0.952, loss=46.880, backward_time=0.192, grad_norm=74.708, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:48:20,189 (trainer:732) INFO: 44epoch:train:2432-2574batch: iter_time=2.610e-04, forward_time=0.128, loss_att=41.729, acc=0.955, loss=41.729, backward_time=0.190, grad_norm=62.263, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.505 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:50:33,028 (trainer:732) INFO: 44epoch:train:2575-2717batch: iter_time=2.761e-04, forward_time=0.128, loss_att=44.844, acc=0.955, loss=44.844, backward_time=0.191, grad_norm=74.517, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 07:52:46,657 (trainer:732) INFO: 44epoch:train:2718-2860batch: iter_time=2.436e-04, forward_time=0.130, loss_att=47.570, acc=0.951, loss=47.570, backward_time=0.193, grad_norm=78.328, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.728 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:01:36,180 (trainer:338) INFO: 44epoch results: [train] iter_time=3.744e-04, forward_time=0.128, loss_att=43.359, acc=0.954, loss=43.359, backward_time=0.190, grad_norm=69.886, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.856, time=46 minutes and 16.96 seconds, total_count=126676, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.980, acc=0.968, cer=0.040, wer=0.141, loss=19.980, time=4 minutes and 3.08 seconds, total_count=3872, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.33 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:01:40,036 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:01:40,047 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/34epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:01:40,047 (trainer:272) INFO: 45/60epoch started. Estimated time to finish: 14 hours, 8 minutes and 5.26 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:06:13,228 (trainer:732) INFO: 45epoch:train:1-143batch: iter_time=0.004, forward_time=0.127, loss_att=38.531, acc=0.959, loss=38.531, backward_time=0.189, grad_norm=65.907, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=7.734 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:08:23,261 (trainer:732) INFO: 45epoch:train:144-286batch: iter_time=2.685e-04, forward_time=0.125, loss_att=35.451, acc=0.959, loss=35.451, backward_time=0.185, grad_norm=67.273, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.641 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:10:36,531 (trainer:732) INFO: 45epoch:train:287-429batch: iter_time=2.669e-04, forward_time=0.128, loss_att=41.821, acc=0.957, loss=41.821, backward_time=0.192, grad_norm=71.401, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.726 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:12:42,753 (trainer:732) INFO: 45epoch:train:430-572batch: iter_time=2.726e-04, forward_time=0.128, loss_att=42.897, acc=0.955, loss=42.897, backward_time=0.191, grad_norm=70.203, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.520 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:14:54,343 (trainer:732) INFO: 45epoch:train:573-715batch: iter_time=2.615e-04, forward_time=0.128, loss_att=44.052, acc=0.954, loss=44.052, backward_time=0.191, grad_norm=78.952, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.692 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:17:07,075 (trainer:732) INFO: 45epoch:train:716-858batch: iter_time=2.622e-04, forward_time=0.128, loss_att=43.272, acc=0.955, loss=43.272, backward_time=0.191, grad_norm=66.801, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:19:17,035 (trainer:732) INFO: 45epoch:train:859-1001batch: iter_time=2.721e-04, forward_time=0.129, loss_att=41.907, acc=0.956, loss=41.907, backward_time=0.192, grad_norm=68.175, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.640 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:21:27,558 (trainer:732) INFO: 45epoch:train:1002-1144batch: iter_time=2.653e-04, forward_time=0.129, loss_att=43.038, acc=0.954, loss=43.038, backward_time=0.192, grad_norm=76.834, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.635 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:23:39,268 (trainer:732) INFO: 45epoch:train:1145-1287batch: iter_time=2.586e-04, forward_time=0.126, loss_att=41.302, acc=0.955, loss=41.302, backward_time=0.188, grad_norm=67.014, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:25:51,229 (trainer:732) INFO: 45epoch:train:1288-1430batch: iter_time=2.889e-04, forward_time=0.127, loss_att=40.984, acc=0.956, loss=40.984, backward_time=0.189, grad_norm=67.762, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.686 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:27:56,111 (trainer:732) INFO: 45epoch:train:1431-1573batch: iter_time=2.667e-04, forward_time=0.127, loss_att=35.408, acc=0.960, loss=35.408, backward_time=0.188, grad_norm=60.022, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.496 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:30:08,748 (trainer:732) INFO: 45epoch:train:1574-1716batch: iter_time=2.712e-04, forward_time=0.127, loss_att=41.789, acc=0.956, loss=41.789, backward_time=0.190, grad_norm=65.414, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:32:22,664 (trainer:732) INFO: 45epoch:train:1717-1859batch: iter_time=2.889e-04, forward_time=0.129, loss_att=48.536, acc=0.951, loss=48.536, backward_time=0.193, grad_norm=82.212, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.761 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:34:34,943 (trainer:732) INFO: 45epoch:train:1860-2002batch: iter_time=2.756e-04, forward_time=0.128, loss_att=43.806, acc=0.955, loss=43.806, backward_time=0.191, grad_norm=70.225, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.713 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:36:43,060 (trainer:732) INFO: 45epoch:train:2003-2145batch: iter_time=2.672e-04, forward_time=0.127, loss_att=39.766, acc=0.957, loss=39.766, backward_time=0.189, grad_norm=68.848, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.566 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:38:57,274 (trainer:732) INFO: 45epoch:train:2146-2288batch: iter_time=2.724e-04, forward_time=0.128, loss_att=45.580, acc=0.954, loss=45.580, backward_time=0.192, grad_norm=75.395, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.744 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:41:09,927 (trainer:732) INFO: 45epoch:train:2289-2431batch: iter_time=2.480e-04, forward_time=0.129, loss_att=45.081, acc=0.953, loss=45.081, backward_time=0.192, grad_norm=69.179, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:43:17,323 (trainer:732) INFO: 45epoch:train:2432-2574batch: iter_time=2.623e-04, forward_time=0.129, loss_att=45.563, acc=0.953, loss=45.563, backward_time=0.193, grad_norm=72.151, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.561 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:45:28,426 (trainer:732) INFO: 45epoch:train:2575-2717batch: iter_time=2.800e-04, forward_time=0.127, loss_att=41.060, acc=0.957, loss=41.060, backward_time=0.189, grad_norm=61.790, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.668 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:47:40,512 (trainer:732) INFO: 45epoch:train:2718-2860batch: iter_time=2.589e-04, forward_time=0.128, loss_att=40.438, acc=0.957, loss=40.438, backward_time=0.192, grad_norm=66.910, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.683 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:56:24,739 (trainer:338) INFO: 45epoch results: [train] iter_time=4.554e-04, forward_time=0.128, loss_att=41.901, acc=0.956, loss=41.901, backward_time=0.190, grad_norm=69.693, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.860, time=46 minutes and 20.1 seconds, total_count=129555, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.476, acc=0.967, cer=0.040, wer=0.142, loss=20.476, time=4 minutes and 0.72 seconds, total_count=3960, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 23.87 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:56:28,562 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:56:28,573 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/35epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 08:56:28,573 (trainer:272) INFO: 46/60epoch started. Estimated time to finish: 13 hours, 15 minutes and 41 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:00:55,741 (trainer:732) INFO: 46epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=37.735, acc=0.959, loss=37.735, backward_time=0.190, grad_norm=66.772, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=7.569 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:03:08,461 (trainer:732) INFO: 46epoch:train:144-286batch: iter_time=2.859e-04, forward_time=0.128, loss_att=38.819, acc=0.958, loss=38.819, backward_time=0.191, grad_norm=69.911, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:05:21,056 (trainer:732) INFO: 46epoch:train:287-429batch: iter_time=2.737e-04, forward_time=0.129, loss_att=39.466, acc=0.959, loss=39.466, backward_time=0.192, grad_norm=66.029, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:07:28,534 (trainer:732) INFO: 46epoch:train:430-572batch: iter_time=2.840e-04, forward_time=0.128, loss_att=39.185, acc=0.958, loss=39.185, backward_time=0.190, grad_norm=65.367, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.559 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:09:40,307 (trainer:732) INFO: 46epoch:train:573-715batch: iter_time=2.663e-04, forward_time=0.127, loss_att=38.576, acc=0.959, loss=38.576, backward_time=0.189, grad_norm=71.486, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:11:53,583 (trainer:732) INFO: 46epoch:train:716-858batch: iter_time=2.889e-04, forward_time=0.128, loss_att=37.422, acc=0.960, loss=37.422, backward_time=0.191, grad_norm=66.470, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.729 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:14:06,897 (trainer:732) INFO: 46epoch:train:859-1001batch: iter_time=2.706e-04, forward_time=0.127, loss_att=43.857, acc=0.954, loss=43.857, backward_time=0.190, grad_norm=67.618, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.727 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:16:14,657 (trainer:732) INFO: 46epoch:train:1002-1144batch: iter_time=2.716e-04, forward_time=0.128, loss_att=42.538, acc=0.956, loss=42.538, backward_time=0.191, grad_norm=79.516, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.564 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:18:26,829 (trainer:732) INFO: 46epoch:train:1145-1287batch: iter_time=2.615e-04, forward_time=0.127, loss_att=38.298, acc=0.959, loss=38.298, backward_time=0.189, grad_norm=67.335, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:20:40,346 (trainer:732) INFO: 46epoch:train:1288-1430batch: iter_time=2.621e-04, forward_time=0.130, loss_att=43.709, acc=0.957, loss=43.709, backward_time=0.194, grad_norm=69.465, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:22:45,483 (trainer:732) INFO: 46epoch:train:1431-1573batch: iter_time=2.607e-04, forward_time=0.126, loss_att=39.787, acc=0.957, loss=39.787, backward_time=0.187, grad_norm=62.081, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.501 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:24:58,689 (trainer:732) INFO: 46epoch:train:1574-1716batch: iter_time=2.231e-04, forward_time=0.128, loss_att=43.845, acc=0.954, loss=43.845, backward_time=0.192, grad_norm=81.967, clip=100.000, loss_scale=1.000, optim_step_time=0.055, optim0_lr0=0.002, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:27:10,854 (trainer:732) INFO: 46epoch:train:1717-1859batch: iter_time=2.101e-04, forward_time=0.126, loss_att=42.741, acc=0.954, loss=42.741, backward_time=0.191, grad_norm=69.447, clip=100.000, loss_scale=1.000, optim_step_time=0.056, optim0_lr0=0.002, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:29:23,824 (trainer:732) INFO: 46epoch:train:1860-2002batch: iter_time=2.073e-04, forward_time=0.126, loss_att=44.131, acc=0.954, loss=44.131, backward_time=0.191, grad_norm=71.711, clip=100.000, loss_scale=1.000, optim_step_time=0.056, optim0_lr0=0.002, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:31:28,843 (trainer:732) INFO: 46epoch:train:2003-2145batch: iter_time=2.125e-04, forward_time=0.125, loss_att=39.930, acc=0.956, loss=39.930, backward_time=0.188, grad_norm=60.068, clip=100.000, loss_scale=1.000, optim_step_time=0.055, optim0_lr0=0.002, train_time=3.497 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:33:42,227 (trainer:732) INFO: 46epoch:train:2146-2288batch: iter_time=2.120e-04, forward_time=0.126, loss_att=45.104, acc=0.955, loss=45.104, backward_time=0.190, grad_norm=76.489, clip=100.000, loss_scale=1.000, optim_step_time=0.054, optim0_lr0=0.002, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:35:53,093 (trainer:732) INFO: 46epoch:train:2289-2431batch: iter_time=2.143e-04, forward_time=0.124, loss_att=37.523, acc=0.959, loss=37.523, backward_time=0.187, grad_norm=67.765, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.674 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:38:00,202 (trainer:732) INFO: 46epoch:train:2432-2574batch: iter_time=1.994e-04, forward_time=0.126, loss_att=41.526, acc=0.957, loss=41.526, backward_time=0.191, grad_norm=71.727, clip=100.000, loss_scale=1.000, optim_step_time=0.056, optim0_lr0=0.002, train_time=3.550 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:40:12,880 (trainer:732) INFO: 46epoch:train:2575-2717batch: iter_time=2.424e-04, forward_time=0.127, loss_att=41.536, acc=0.957, loss=41.536, backward_time=0.191, grad_norm=70.195, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:42:26,145 (trainer:732) INFO: 46epoch:train:2718-2860batch: iter_time=2.496e-04, forward_time=0.127, loss_att=41.497, acc=0.955, loss=41.497, backward_time=0.190, grad_norm=66.783, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:51:14,257 (trainer:338) INFO: 46epoch results: [train] iter_time=3.967e-04, forward_time=0.127, loss_att=40.751, acc=0.957, loss=40.751, backward_time=0.190, grad_norm=69.389, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.856, time=46 minutes and 16.61 seconds, total_count=132434, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.437, acc=0.969, cer=0.039, wer=0.138, loss=19.437, time=4 minutes and 3.22 seconds, total_count=4048, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.85 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:51:18,279 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:51:18,292 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/36epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:51:18,292 (trainer:272) INFO: 47/60epoch started. Estimated time to finish: 12 hours, 23 minutes and 10.82 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:55:45,709 (trainer:732) INFO: 47epoch:train:1-143batch: iter_time=0.003, forward_time=0.129, loss_att=36.424, acc=0.961, loss=36.424, backward_time=0.191, grad_norm=70.879, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=7.572 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 09:57:57,576 (trainer:732) INFO: 47epoch:train:144-286batch: iter_time=2.614e-04, forward_time=0.128, loss_att=36.819, acc=0.960, loss=36.819, backward_time=0.190, grad_norm=68.258, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.689 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:00:11,252 (trainer:732) INFO: 47epoch:train:287-429batch: iter_time=2.529e-04, forward_time=0.128, loss_att=40.164, acc=0.958, loss=40.164, backward_time=0.191, grad_norm=68.718, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:02:19,895 (trainer:732) INFO: 47epoch:train:430-572batch: iter_time=2.643e-04, forward_time=0.128, loss_att=40.246, acc=0.958, loss=40.246, backward_time=0.191, grad_norm=74.092, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.592 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:04:33,062 (trainer:732) INFO: 47epoch:train:573-715batch: iter_time=2.650e-04, forward_time=0.128, loss_att=39.929, acc=0.958, loss=39.929, backward_time=0.191, grad_norm=76.177, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:06:45,553 (trainer:732) INFO: 47epoch:train:716-858batch: iter_time=2.472e-04, forward_time=0.127, loss_att=37.784, acc=0.959, loss=37.784, backward_time=0.189, grad_norm=62.403, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:08:59,886 (trainer:732) INFO: 47epoch:train:859-1001batch: iter_time=2.593e-04, forward_time=0.129, loss_att=42.727, acc=0.955, loss=42.727, backward_time=0.193, grad_norm=71.230, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.759 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:11:09,528 (trainer:732) INFO: 47epoch:train:1002-1144batch: iter_time=2.511e-04, forward_time=0.130, loss_att=42.734, acc=0.957, loss=42.734, backward_time=0.194, grad_norm=66.767, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.618 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:13:20,621 (trainer:732) INFO: 47epoch:train:1145-1287batch: iter_time=2.474e-04, forward_time=0.127, loss_att=38.845, acc=0.958, loss=38.845, backward_time=0.189, grad_norm=66.176, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.675 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:15:33,578 (trainer:732) INFO: 47epoch:train:1288-1430batch: iter_time=2.629e-04, forward_time=0.128, loss_att=39.488, acc=0.959, loss=39.488, backward_time=0.191, grad_norm=77.774, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.715 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:17:45,302 (trainer:732) INFO: 47epoch:train:1431-1573batch: iter_time=2.336e-04, forward_time=0.126, loss_att=40.257, acc=0.958, loss=40.257, backward_time=0.188, grad_norm=65.835, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:19:55,831 (trainer:732) INFO: 47epoch:train:1574-1716batch: iter_time=2.526e-04, forward_time=0.128, loss_att=40.666, acc=0.957, loss=40.666, backward_time=0.191, grad_norm=66.067, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.636 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:22:08,047 (trainer:732) INFO: 47epoch:train:1717-1859batch: iter_time=2.680e-04, forward_time=0.128, loss_att=42.034, acc=0.957, loss=42.034, backward_time=0.190, grad_norm=73.739, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:24:21,762 (trainer:732) INFO: 47epoch:train:1860-2002batch: iter_time=2.477e-04, forward_time=0.127, loss_att=39.642, acc=0.958, loss=39.642, backward_time=0.190, grad_norm=70.012, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.741 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:26:28,506 (trainer:732) INFO: 47epoch:train:2003-2145batch: iter_time=2.513e-04, forward_time=0.127, loss_att=38.287, acc=0.959, loss=38.287, backward_time=0.189, grad_norm=68.812, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.549 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:28:42,696 (trainer:732) INFO: 47epoch:train:2146-2288batch: iter_time=2.598e-04, forward_time=0.127, loss_att=38.357, acc=0.960, loss=38.357, backward_time=0.190, grad_norm=67.746, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.741 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:30:54,446 (trainer:732) INFO: 47epoch:train:2289-2431batch: iter_time=2.418e-04, forward_time=0.127, loss_att=37.773, acc=0.960, loss=37.773, backward_time=0.189, grad_norm=69.185, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.693 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:33:02,286 (trainer:732) INFO: 47epoch:train:2432-2574batch: iter_time=2.472e-04, forward_time=0.128, loss_att=38.089, acc=0.960, loss=38.089, backward_time=0.190, grad_norm=66.289, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.593 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:35:15,386 (trainer:732) INFO: 47epoch:train:2575-2717batch: iter_time=2.569e-04, forward_time=0.128, loss_att=37.114, acc=0.959, loss=37.114, backward_time=0.191, grad_norm=63.254, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:37:29,142 (trainer:732) INFO: 47epoch:train:2718-2860batch: iter_time=2.384e-04, forward_time=0.128, loss_att=40.523, acc=0.958, loss=40.523, backward_time=0.190, grad_norm=72.219, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:46:20,811 (trainer:338) INFO: 47epoch results: [train] iter_time=3.754e-04, forward_time=0.128, loss_att=39.302, acc=0.958, loss=39.302, backward_time=0.190, grad_norm=69.191, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.873, time=46 minutes and 29.86 seconds, total_count=135313, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.682, acc=0.969, cer=0.039, wer=0.138, loss=19.682, time=4 minutes and 4.85 seconds, total_count=4136, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.81 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:46:24,896 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:46:24,907 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/37epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:46:24,908 (trainer:272) INFO: 48/60epoch started. Estimated time to finish: 11 hours, 30 minutes and 39.39 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:50:52,522 (trainer:732) INFO: 48epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=35.885, acc=0.961, loss=35.885, backward_time=0.192, grad_norm=68.152, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=7.581 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 152) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 152) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:53:06,049 (trainer:732) INFO: 48epoch:train:144-286batch: iter_time=2.783e-04, forward_time=0.128, loss_att=40.089, acc=0.958, loss=40.089, backward_time=0.191, grad_norm=64.546, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.732 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:55:19,700 (trainer:732) INFO: 48epoch:train:287-429batch: iter_time=2.615e-04, forward_time=0.130, loss_att=38.026, acc=0.959, loss=38.026, backward_time=0.193, grad_norm=65.889, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.732 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:57:28,132 (trainer:732) INFO: 48epoch:train:430-572batch: iter_time=2.750e-04, forward_time=0.127, loss_att=39.358, acc=0.959, loss=39.358, backward_time=0.189, grad_norm=72.262, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.587 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 10:59:40,191 (trainer:732) INFO: 48epoch:train:573-715batch: iter_time=2.718e-04, forward_time=0.127, loss_att=34.535, acc=0.962, loss=34.535, backward_time=0.188, grad_norm=68.890, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:01:53,828 (trainer:732) INFO: 48epoch:train:716-858batch: iter_time=2.600e-04, forward_time=0.127, loss_att=39.417, acc=0.958, loss=39.417, backward_time=0.188, grad_norm=73.054, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.733 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:04:06,987 (trainer:732) INFO: 48epoch:train:859-1001batch: iter_time=2.698e-04, forward_time=0.128, loss_att=37.694, acc=0.960, loss=37.694, backward_time=0.190, grad_norm=65.484, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:06:14,737 (trainer:732) INFO: 48epoch:train:1002-1144batch: iter_time=2.553e-04, forward_time=0.127, loss_att=39.452, acc=0.959, loss=39.452, backward_time=0.190, grad_norm=62.268, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.565 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:08:27,694 (trainer:732) INFO: 48epoch:train:1145-1287batch: iter_time=2.497e-04, forward_time=0.129, loss_att=40.214, acc=0.958, loss=40.214, backward_time=0.192, grad_norm=67.493, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.728 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:10:40,535 (trainer:732) INFO: 48epoch:train:1288-1430batch: iter_time=2.798e-04, forward_time=0.127, loss_att=35.240, acc=0.962, loss=35.240, backward_time=0.190, grad_norm=64.926, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:12:47,583 (trainer:732) INFO: 48epoch:train:1431-1573batch: iter_time=2.511e-04, forward_time=0.125, loss_att=33.828, acc=0.962, loss=33.828, backward_time=0.186, grad_norm=63.700, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.550 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:15:00,506 (trainer:732) INFO: 48epoch:train:1574-1716batch: iter_time=2.688e-04, forward_time=0.127, loss_att=36.832, acc=0.961, loss=36.832, backward_time=0.189, grad_norm=66.765, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:17:12,269 (trainer:732) INFO: 48epoch:train:1717-1859batch: iter_time=2.623e-04, forward_time=0.126, loss_att=36.043, acc=0.961, loss=36.043, backward_time=0.187, grad_norm=67.728, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.701 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:19:25,567 (trainer:732) INFO: 48epoch:train:1860-2002batch: iter_time=2.612e-04, forward_time=0.128, loss_att=36.101, acc=0.962, loss=36.101, backward_time=0.190, grad_norm=68.552, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.002, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:21:33,634 (trainer:732) INFO: 48epoch:train:2003-2145batch: iter_time=2.616e-04, forward_time=0.128, loss_att=39.818, acc=0.959, loss=39.818, backward_time=0.191, grad_norm=78.958, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.579 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:23:49,035 (trainer:732) INFO: 48epoch:train:2146-2288batch: iter_time=2.645e-04, forward_time=0.131, loss_att=42.686, acc=0.958, loss=42.686, backward_time=0.196, grad_norm=70.235, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.777 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:26:02,092 (trainer:732) INFO: 48epoch:train:2289-2431batch: iter_time=2.647e-04, forward_time=0.129, loss_att=38.505, acc=0.960, loss=38.505, backward_time=0.193, grad_norm=75.749, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.735 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:28:10,289 (trainer:732) INFO: 48epoch:train:2432-2574batch: iter_time=2.678e-04, forward_time=0.128, loss_att=39.068, acc=0.958, loss=39.068, backward_time=0.190, grad_norm=70.170, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.583 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:30:24,536 (trainer:732) INFO: 48epoch:train:2575-2717batch: iter_time=2.751e-04, forward_time=0.129, loss_att=42.216, acc=0.957, loss=42.216, backward_time=0.193, grad_norm=70.260, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.757 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:32:37,980 (trainer:732) INFO: 48epoch:train:2718-2860batch: iter_time=2.398e-04, forward_time=0.128, loss_att=40.678, acc=0.957, loss=40.678, backward_time=0.190, grad_norm=64.916, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.721 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 81) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146343:3146535 [1] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 81) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 142) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146344:3146538 [2] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 142) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:41:25,499 (trainer:338) INFO: 48epoch results: [train] iter_time=4.092e-04, forward_time=0.128, loss_att=38.219, acc=0.960, loss=38.219, backward_time=0.190, grad_norm=68.427, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.877, time=46 minutes and 32.61 seconds, total_count=138192, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.044, acc=0.968, cer=0.039, wer=0.139, loss=20.044, time=4 minutes and 2.02 seconds, total_count=4224, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.96 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:41:29,316 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:41:29,328 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/38epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:41:29,328 (trainer:272) INFO: 49/60epoch started. Estimated time to finish: 10 hours, 38 minutes and 0.94 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:45:55,996 (trainer:732) INFO: 49epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=35.935, acc=0.962, loss=35.935, backward_time=0.189, grad_norm=68.177, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=7.552 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:48:09,048 (trainer:732) INFO: 49epoch:train:144-286batch: iter_time=2.675e-04, forward_time=0.126, loss_att=35.807, acc=0.962, loss=35.807, backward_time=0.188, grad_norm=67.908, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:50:22,194 (trainer:732) INFO: 49epoch:train:287-429batch: iter_time=2.643e-04, forward_time=0.126, loss_att=34.462, acc=0.962, loss=34.462, backward_time=0.188, grad_norm=65.679, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.728 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:52:29,578 (trainer:732) INFO: 49epoch:train:430-572batch: iter_time=2.744e-04, forward_time=0.127, loss_att=35.362, acc=0.962, loss=35.362, backward_time=0.189, grad_norm=69.956, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.552 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:54:41,856 (trainer:732) INFO: 49epoch:train:573-715batch: iter_time=3.074e-04, forward_time=0.129, loss_att=36.408, acc=0.961, loss=36.408, backward_time=0.189, grad_norm=65.083, clip=100.000, loss_scale=1.000, optim_step_time=0.071, optim0_lr0=0.002, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:56:54,979 (trainer:732) INFO: 49epoch:train:716-858batch: iter_time=3.195e-04, forward_time=0.131, loss_att=37.854, acc=0.961, loss=37.854, backward_time=0.191, grad_norm=64.915, clip=100.000, loss_scale=1.000, optim_step_time=0.070, optim0_lr0=0.002, train_time=3.722 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 11:59:07,643 (trainer:732) INFO: 49epoch:train:859-1001batch: iter_time=2.912e-04, forward_time=0.129, loss_att=39.472, acc=0.960, loss=39.472, backward_time=0.191, grad_norm=72.990, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:01:14,407 (trainer:732) INFO: 49epoch:train:1002-1144batch: iter_time=2.716e-04, forward_time=0.128, loss_att=37.084, acc=0.960, loss=37.084, backward_time=0.190, grad_norm=60.823, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.537 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:03:27,589 (trainer:732) INFO: 49epoch:train:1145-1287batch: iter_time=2.689e-04, forward_time=0.129, loss_att=38.639, acc=0.959, loss=38.639, backward_time=0.192, grad_norm=68.229, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.738 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:05:41,742 (trainer:732) INFO: 49epoch:train:1288-1430batch: iter_time=2.807e-04, forward_time=0.130, loss_att=40.607, acc=0.959, loss=40.607, backward_time=0.194, grad_norm=79.535, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.748 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:07:49,139 (trainer:732) INFO: 49epoch:train:1431-1573batch: iter_time=2.705e-04, forward_time=0.127, loss_att=35.950, acc=0.961, loss=35.950, backward_time=0.188, grad_norm=62.671, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.563 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:10:02,695 (trainer:732) INFO: 49epoch:train:1574-1716batch: iter_time=2.611e-04, forward_time=0.128, loss_att=37.294, acc=0.959, loss=37.294, backward_time=0.190, grad_norm=66.526, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.728 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:12:16,617 (trainer:732) INFO: 49epoch:train:1717-1859batch: iter_time=2.960e-04, forward_time=0.131, loss_att=41.392, acc=0.958, loss=41.392, backward_time=0.195, grad_norm=73.934, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.759 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:14:29,517 (trainer:732) INFO: 49epoch:train:1860-2002batch: iter_time=2.614e-04, forward_time=0.129, loss_att=37.543, acc=0.960, loss=37.543, backward_time=0.192, grad_norm=67.913, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.002, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:16:36,858 (trainer:732) INFO: 49epoch:train:2003-2145batch: iter_time=2.985e-04, forward_time=0.128, loss_att=38.768, acc=0.958, loss=38.768, backward_time=0.189, grad_norm=68.187, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.564 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:18:49,530 (trainer:732) INFO: 49epoch:train:2146-2288batch: iter_time=2.949e-04, forward_time=0.127, loss_att=33.531, acc=0.964, loss=33.531, backward_time=0.189, grad_norm=69.841, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:21:01,426 (trainer:732) INFO: 49epoch:train:2289-2431batch: iter_time=2.744e-04, forward_time=0.127, loss_att=34.559, acc=0.961, loss=34.559, backward_time=0.188, grad_norm=68.565, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.703 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:23:09,306 (trainer:732) INFO: 49epoch:train:2432-2574batch: iter_time=2.790e-04, forward_time=0.129, loss_att=37.322, acc=0.963, loss=37.322, backward_time=0.192, grad_norm=69.327, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.574 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:25:21,045 (trainer:732) INFO: 49epoch:train:2575-2717batch: iter_time=2.672e-04, forward_time=0.128, loss_att=39.165, acc=0.959, loss=39.165, backward_time=0.191, grad_norm=72.854, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.684 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:27:35,533 (trainer:732) INFO: 49epoch:train:2718-2860batch: iter_time=2.526e-04, forward_time=0.129, loss_att=40.463, acc=0.959, loss=40.463, backward_time=0.193, grad_norm=64.378, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.751 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 101) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146345:3146536 [3] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 0, fd 101) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:36:57,392 (trainer:338) INFO: 49epoch results: [train] iter_time=3.824e-04, forward_time=0.128, loss_att=37.343, acc=0.960, loss=37.343, backward_time=0.190, grad_norm=68.359, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.868, time=46 minutes and 26.7 seconds, total_count=141071, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.057, acc=0.969, cer=0.039, wer=0.138, loss=20.057, time=4 minutes and 16.15 seconds, total_count=4312, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 45.21 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:37:01,879 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:37:01,892 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/40epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:37:01,893 (trainer:272) INFO: 50/60epoch started. Estimated time to finish: 9 hours, 45 minutes and 22.84 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:41:54,908 (trainer:732) INFO: 50epoch:train:1-143batch: iter_time=0.004, forward_time=0.129, loss_att=33.097, acc=0.964, loss=33.097, backward_time=0.190, grad_norm=63.017, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.002, train_time=8.300 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:44:07,093 (trainer:732) INFO: 50epoch:train:144-286batch: iter_time=2.712e-04, forward_time=0.127, loss_att=33.772, acc=0.963, loss=33.772, backward_time=0.188, grad_norm=58.433, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.002, train_time=3.696 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:46:20,965 (trainer:732) INFO: 50epoch:train:287-429batch: iter_time=2.679e-04, forward_time=0.130, loss_att=37.333, acc=0.962, loss=37.333, backward_time=0.194, grad_norm=63.574, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.002, train_time=3.745 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:48:29,239 (trainer:732) INFO: 50epoch:train:430-572batch: iter_time=2.727e-04, forward_time=0.127, loss_att=33.152, acc=0.964, loss=33.152, backward_time=0.189, grad_norm=64.795, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.581 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:50:41,249 (trainer:732) INFO: 50epoch:train:573-715batch: iter_time=2.732e-04, forward_time=0.127, loss_att=34.335, acc=0.963, loss=34.335, backward_time=0.189, grad_norm=72.277, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.002, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:52:54,165 (trainer:732) INFO: 50epoch:train:716-858batch: iter_time=2.655e-04, forward_time=0.128, loss_att=34.478, acc=0.963, loss=34.478, backward_time=0.190, grad_norm=65.475, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:55:02,609 (trainer:732) INFO: 50epoch:train:859-1001batch: iter_time=2.769e-04, forward_time=0.130, loss_att=38.540, acc=0.960, loss=38.540, backward_time=0.193, grad_norm=71.118, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.595 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:57:15,160 (trainer:732) INFO: 50epoch:train:1002-1144batch: iter_time=2.708e-04, forward_time=0.127, loss_att=37.984, acc=0.960, loss=37.984, backward_time=0.188, grad_norm=63.024, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.002, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 12:59:27,754 (trainer:732) INFO: 50epoch:train:1145-1287batch: iter_time=2.585e-04, forward_time=0.129, loss_att=38.150, acc=0.960, loss=38.150, backward_time=0.191, grad_norm=73.807, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.002, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:01:40,985 (trainer:732) INFO: 50epoch:train:1288-1430batch: iter_time=2.636e-04, forward_time=0.129, loss_att=38.144, acc=0.960, loss=38.144, backward_time=0.192, grad_norm=68.811, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:03:49,246 (trainer:732) INFO: 50epoch:train:1431-1573batch: iter_time=2.569e-04, forward_time=0.128, loss_att=37.384, acc=0.961, loss=37.384, backward_time=0.191, grad_norm=67.637, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.583 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:06:03,250 (trainer:732) INFO: 50epoch:train:1574-1716batch: iter_time=2.539e-04, forward_time=0.128, loss_att=38.260, acc=0.959, loss=38.260, backward_time=0.191, grad_norm=69.104, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.740 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:08:17,204 (trainer:732) INFO: 50epoch:train:1717-1859batch: iter_time=2.598e-04, forward_time=0.129, loss_att=38.658, acc=0.961, loss=38.658, backward_time=0.192, grad_norm=67.279, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.757 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:10:25,383 (trainer:732) INFO: 50epoch:train:1860-2002batch: iter_time=2.356e-04, forward_time=0.127, loss_att=37.388, acc=0.961, loss=37.388, backward_time=0.189, grad_norm=67.127, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.606 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:12:36,743 (trainer:732) INFO: 50epoch:train:2003-2145batch: iter_time=2.641e-04, forward_time=0.129, loss_att=37.272, acc=0.961, loss=37.272, backward_time=0.191, grad_norm=71.214, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.654 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:14:50,291 (trainer:732) INFO: 50epoch:train:2146-2288batch: iter_time=2.665e-04, forward_time=0.127, loss_att=35.105, acc=0.963, loss=35.105, backward_time=0.189, grad_norm=60.398, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.727 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:17:04,041 (trainer:732) INFO: 50epoch:train:2289-2431batch: iter_time=2.581e-04, forward_time=0.129, loss_att=38.232, acc=0.960, loss=38.232, backward_time=0.192, grad_norm=73.446, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.755 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:19:10,808 (trainer:732) INFO: 50epoch:train:2432-2574batch: iter_time=2.697e-04, forward_time=0.127, loss_att=33.797, acc=0.962, loss=33.797, backward_time=0.188, grad_norm=60.045, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.540 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:21:23,994 (trainer:732) INFO: 50epoch:train:2575-2717batch: iter_time=2.728e-04, forward_time=0.127, loss_att=36.191, acc=0.961, loss=36.191, backward_time=0.188, grad_norm=63.617, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.729 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:23:38,607 (trainer:732) INFO: 50epoch:train:2718-2860batch: iter_time=2.380e-04, forward_time=0.130, loss_att=39.659, acc=0.960, loss=39.659, backward_time=0.194, grad_norm=72.416, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.754 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:32:35,544 (trainer:338) INFO: 50epoch results: [train] iter_time=4.429e-04, forward_time=0.128, loss_att=36.445, acc=0.961, loss=36.445, backward_time=0.190, grad_norm=66.770, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.910, time=46 minutes and 57.15 seconds, total_count=143950, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.013, acc=0.969, cer=0.039, wer=0.137, loss=20.013, time=4 minutes and 11.47 seconds, total_count=4400, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.02 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:32:39,550 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:32:39,563 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/39epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:32:39,563 (trainer:272) INFO: 51/60epoch started. Estimated time to finish: 8 hours, 52 minutes and 38.8 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:37:12,098 (trainer:732) INFO: 51epoch:train:1-143batch: iter_time=0.002, forward_time=0.132, loss_att=36.156, acc=0.964, loss=36.156, backward_time=0.195, grad_norm=68.044, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=7.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:39:25,079 (trainer:732) INFO: 51epoch:train:144-286batch: iter_time=3.131e-04, forward_time=0.129, loss_att=32.208, acc=0.966, loss=32.208, backward_time=0.190, grad_norm=70.800, clip=100.000, loss_scale=1.000, optim_step_time=0.069, optim0_lr0=0.001, train_time=3.714 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:41:36,367 (trainer:732) INFO: 51epoch:train:287-429batch: iter_time=3.379e-04, forward_time=0.128, loss_att=32.024, acc=0.964, loss=32.024, backward_time=0.187, grad_norm=57.641, clip=100.000, loss_scale=1.000, optim_step_time=0.071, optim0_lr0=0.001, train_time=3.673 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:43:44,061 (trainer:732) INFO: 51epoch:train:430-572batch: iter_time=3.305e-04, forward_time=0.129, loss_att=35.755, acc=0.962, loss=35.755, backward_time=0.190, grad_norm=73.277, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.560 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:45:55,747 (trainer:732) INFO: 51epoch:train:573-715batch: iter_time=3.006e-04, forward_time=0.128, loss_att=31.863, acc=0.965, loss=31.863, backward_time=0.188, grad_norm=63.458, clip=100.000, loss_scale=1.000, optim_step_time=0.064, optim0_lr0=0.001, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:48:08,971 (trainer:732) INFO: 51epoch:train:716-858batch: iter_time=2.636e-04, forward_time=0.128, loss_att=35.350, acc=0.962, loss=35.350, backward_time=0.190, grad_norm=72.900, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.724 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:50:17,016 (trainer:732) INFO: 51epoch:train:859-1001batch: iter_time=2.808e-04, forward_time=0.129, loss_att=37.354, acc=0.961, loss=37.354, backward_time=0.191, grad_norm=64.270, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.580 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:52:29,241 (trainer:732) INFO: 51epoch:train:1002-1144batch: iter_time=2.662e-04, forward_time=0.127, loss_att=31.662, acc=0.965, loss=31.662, backward_time=0.188, grad_norm=58.766, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.688 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:54:41,160 (trainer:732) INFO: 51epoch:train:1145-1287batch: iter_time=2.740e-04, forward_time=0.128, loss_att=35.112, acc=0.963, loss=35.112, backward_time=0.191, grad_norm=69.324, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.704 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:56:54,541 (trainer:732) INFO: 51epoch:train:1288-1430batch: iter_time=2.958e-04, forward_time=0.128, loss_att=33.498, acc=0.964, loss=33.498, backward_time=0.189, grad_norm=77.572, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.726 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 13:59:02,433 (trainer:732) INFO: 51epoch:train:1431-1573batch: iter_time=2.833e-04, forward_time=0.130, loss_att=37.244, acc=0.961, loss=37.244, backward_time=0.193, grad_norm=68.610, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.576 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:01:13,930 (trainer:732) INFO: 51epoch:train:1574-1716batch: iter_time=2.922e-04, forward_time=0.126, loss_att=32.694, acc=0.964, loss=32.694, backward_time=0.186, grad_norm=60.783, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.670 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:03:27,042 (trainer:732) INFO: 51epoch:train:1717-1859batch: iter_time=2.863e-04, forward_time=0.129, loss_att=36.209, acc=0.963, loss=36.209, backward_time=0.192, grad_norm=71.573, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.735 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:05:36,356 (trainer:732) INFO: 51epoch:train:1860-2002batch: iter_time=2.779e-04, forward_time=0.130, loss_att=38.391, acc=0.961, loss=38.391, backward_time=0.193, grad_norm=72.336, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.616 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 2, fd 152) + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/alloc.h:48 NCCL WARN Cuda failure 'out of memory' +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:231 -> 1 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 153) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:07:49,527 (trainer:732) INFO: 51epoch:train:2003-2145batch: iter_time=2.722e-04, forward_time=0.128, loss_att=37.516, acc=0.961, loss=37.516, backward_time=0.191, grad_norm=67.595, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:10:03,459 (trainer:732) INFO: 51epoch:train:2146-2288batch: iter_time=2.817e-04, forward_time=0.129, loss_att=38.094, acc=0.960, loss=38.094, backward_time=0.192, grad_norm=71.854, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:12:16,507 (trainer:732) INFO: 51epoch:train:2289-2431batch: iter_time=2.554e-04, forward_time=0.127, loss_att=36.139, acc=0.962, loss=36.139, backward_time=0.189, grad_norm=68.939, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.731 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:14:24,359 (trainer:732) INFO: 51epoch:train:2432-2574batch: iter_time=2.655e-04, forward_time=0.128, loss_att=36.906, acc=0.962, loss=36.906, backward_time=0.191, grad_norm=64.709, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.577 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:16:36,955 (trainer:732) INFO: 51epoch:train:2575-2717batch: iter_time=2.683e-04, forward_time=0.126, loss_att=32.272, acc=0.964, loss=32.272, backward_time=0.188, grad_norm=66.025, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:18:49,916 (trainer:732) INFO: 51epoch:train:2718-2860batch: iter_time=2.581e-04, forward_time=0.129, loss_att=35.289, acc=0.964, loss=35.289, backward_time=0.193, grad_norm=63.602, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:27:47,972 (trainer:338) INFO: 51epoch results: [train] iter_time=3.754e-04, forward_time=0.128, loss_att=35.050, acc=0.963, loss=35.050, backward_time=0.190, grad_norm=67.590, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.873, time=46 minutes and 29.77 seconds, total_count=146829, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.616, acc=0.969, cer=0.038, wer=0.136, loss=19.616, time=4 minutes and 13.3 seconds, total_count=4488, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.33 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:27:51,866 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:27:51,877 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/41epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:27:51,877 (trainer:272) INFO: 52/60epoch started. Estimated time to finish: 7 hours, 59 minutes and 43.46 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:32:23,560 (trainer:732) INFO: 52epoch:train:1-143batch: iter_time=0.003, forward_time=0.127, loss_att=35.344, acc=0.963, loss=35.344, backward_time=0.188, grad_norm=60.573, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=7.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:34:39,498 (trainer:732) INFO: 52epoch:train:144-286batch: iter_time=2.583e-04, forward_time=0.131, loss_att=33.631, acc=0.966, loss=33.631, backward_time=0.196, grad_norm=67.305, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.802 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:36:52,232 (trainer:732) INFO: 52epoch:train:287-429batch: iter_time=2.503e-04, forward_time=0.127, loss_att=33.977, acc=0.964, loss=33.977, backward_time=0.190, grad_norm=65.624, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:39:01,661 (trainer:732) INFO: 52epoch:train:430-572batch: iter_time=2.775e-04, forward_time=0.129, loss_att=36.538, acc=0.963, loss=36.538, backward_time=0.192, grad_norm=70.719, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.611 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:41:14,898 (trainer:732) INFO: 52epoch:train:573-715batch: iter_time=2.538e-04, forward_time=0.128, loss_att=34.105, acc=0.963, loss=34.105, backward_time=0.192, grad_norm=66.848, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.737 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:43:27,533 (trainer:732) INFO: 52epoch:train:716-858batch: iter_time=2.398e-04, forward_time=0.127, loss_att=32.197, acc=0.966, loss=32.197, backward_time=0.189, grad_norm=69.557, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:45:35,417 (trainer:732) INFO: 52epoch:train:859-1001batch: iter_time=2.577e-04, forward_time=0.127, loss_att=35.766, acc=0.960, loss=35.766, backward_time=0.188, grad_norm=59.020, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.582 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:47:49,354 (trainer:732) INFO: 52epoch:train:1002-1144batch: iter_time=2.509e-04, forward_time=0.128, loss_att=34.391, acc=0.963, loss=34.391, backward_time=0.191, grad_norm=64.752, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.734 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:50:03,778 (trainer:732) INFO: 52epoch:train:1145-1287batch: iter_time=2.513e-04, forward_time=0.129, loss_att=35.016, acc=0.964, loss=35.016, backward_time=0.193, grad_norm=65.773, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.001, train_time=3.771 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:52:16,984 (trainer:732) INFO: 52epoch:train:1288-1430batch: iter_time=3.131e-04, forward_time=0.127, loss_att=33.189, acc=0.965, loss=33.189, backward_time=0.190, grad_norm=66.763, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.739 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:54:24,508 (trainer:732) INFO: 52epoch:train:1431-1573batch: iter_time=2.652e-04, forward_time=0.127, loss_att=35.127, acc=0.963, loss=35.127, backward_time=0.190, grad_norm=68.561, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.553 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:56:37,301 (trainer:732) INFO: 52epoch:train:1574-1716batch: iter_time=2.565e-04, forward_time=0.126, loss_att=33.217, acc=0.964, loss=33.217, backward_time=0.188, grad_norm=63.078, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 14:58:50,487 (trainer:732) INFO: 52epoch:train:1717-1859batch: iter_time=2.660e-04, forward_time=0.129, loss_att=38.121, acc=0.961, loss=38.121, backward_time=0.193, grad_norm=64.844, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.739 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:00:58,519 (trainer:732) INFO: 52epoch:train:1860-2002batch: iter_time=2.498e-04, forward_time=0.128, loss_att=36.270, acc=0.962, loss=36.270, backward_time=0.190, grad_norm=64.493, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.578 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:03:10,572 (trainer:732) INFO: 52epoch:train:2003-2145batch: iter_time=2.582e-04, forward_time=0.127, loss_att=32.787, acc=0.963, loss=32.787, backward_time=0.188, grad_norm=62.593, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:05:23,794 (trainer:732) INFO: 52epoch:train:2146-2288batch: iter_time=2.692e-04, forward_time=0.126, loss_att=34.104, acc=0.963, loss=34.104, backward_time=0.188, grad_norm=62.942, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.713 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:07:33,129 (trainer:732) INFO: 52epoch:train:2289-2431batch: iter_time=2.423e-04, forward_time=0.127, loss_att=31.650, acc=0.966, loss=31.650, backward_time=0.189, grad_norm=67.884, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.659 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:09:43,883 (trainer:732) INFO: 52epoch:train:2432-2574batch: iter_time=2.624e-04, forward_time=0.130, loss_att=38.377, acc=0.960, loss=38.377, backward_time=0.194, grad_norm=62.008, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.626 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:11:56,647 (trainer:732) INFO: 52epoch:train:2575-2717batch: iter_time=2.699e-04, forward_time=0.128, loss_att=32.070, acc=0.965, loss=32.070, backward_time=0.190, grad_norm=68.800, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.713 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:14:09,115 (trainer:732) INFO: 52epoch:train:2718-2860batch: iter_time=2.474e-04, forward_time=0.126, loss_att=32.774, acc=0.965, loss=32.774, backward_time=0.188, grad_norm=66.287, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.694 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:23:03,085 (trainer:338) INFO: 52epoch results: [train] iter_time=4.139e-04, forward_time=0.128, loss_att=34.381, acc=0.963, loss=34.381, backward_time=0.190, grad_norm=65.528, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.883, time=46 minutes and 36.69 seconds, total_count=149708, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.894, acc=0.969, cer=0.038, wer=0.136, loss=19.894, time=4 minutes and 7.3 seconds, total_count=4576, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.21 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:23:07,164 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:23:07,176 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/45epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:23:07,176 (trainer:272) INFO: 53/60epoch started. Estimated time to finish: 7 hours, 6 minutes and 43.32 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:27:40,094 (trainer:732) INFO: 53epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=32.668, acc=0.965, loss=32.668, backward_time=0.190, grad_norm=65.808, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=7.728 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:29:53,278 (trainer:732) INFO: 53epoch:train:144-286batch: iter_time=2.644e-04, forward_time=0.128, loss_att=31.267, acc=0.966, loss=31.267, backward_time=0.190, grad_norm=60.517, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:32:02,383 (trainer:732) INFO: 53epoch:train:287-429batch: iter_time=2.495e-04, forward_time=0.130, loss_att=36.448, acc=0.963, loss=36.448, backward_time=0.194, grad_norm=64.980, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.611 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:34:16,456 (trainer:732) INFO: 53epoch:train:430-572batch: iter_time=2.636e-04, forward_time=0.129, loss_att=33.203, acc=0.966, loss=33.203, backward_time=0.193, grad_norm=67.503, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.739 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:36:28,212 (trainer:732) INFO: 53epoch:train:573-715batch: iter_time=2.628e-04, forward_time=0.126, loss_att=30.988, acc=0.967, loss=30.988, backward_time=0.187, grad_norm=64.402, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.694 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:38:39,697 (trainer:732) INFO: 53epoch:train:716-858batch: iter_time=2.443e-04, forward_time=0.126, loss_att=31.916, acc=0.965, loss=31.916, backward_time=0.186, grad_norm=58.220, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.678 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:40:46,735 (trainer:732) INFO: 53epoch:train:859-1001batch: iter_time=2.704e-04, forward_time=0.127, loss_att=32.916, acc=0.965, loss=32.916, backward_time=0.189, grad_norm=63.999, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.556 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:42:59,797 (trainer:732) INFO: 53epoch:train:1002-1144batch: iter_time=2.600e-04, forward_time=0.126, loss_att=31.942, acc=0.966, loss=31.942, backward_time=0.187, grad_norm=61.041, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:45:14,769 (trainer:732) INFO: 53epoch:train:1145-1287batch: iter_time=2.570e-04, forward_time=0.130, loss_att=38.411, acc=0.962, loss=38.411, backward_time=0.195, grad_norm=66.201, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.788 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:47:21,411 (trainer:732) INFO: 53epoch:train:1288-1430batch: iter_time=2.536e-04, forward_time=0.128, loss_att=29.278, acc=0.969, loss=29.278, backward_time=0.190, grad_norm=62.395, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.545 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:49:32,961 (trainer:732) INFO: 53epoch:train:1431-1573batch: iter_time=2.595e-04, forward_time=0.127, loss_att=33.271, acc=0.965, loss=33.271, backward_time=0.188, grad_norm=64.189, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:51:47,640 (trainer:732) INFO: 53epoch:train:1574-1716batch: iter_time=2.646e-04, forward_time=0.129, loss_att=36.060, acc=0.963, loss=36.060, backward_time=0.193, grad_norm=67.518, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.758 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:54:01,148 (trainer:732) INFO: 53epoch:train:1717-1859batch: iter_time=2.573e-04, forward_time=0.128, loss_att=34.290, acc=0.964, loss=34.290, backward_time=0.190, grad_norm=64.313, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:56:08,876 (trainer:732) INFO: 53epoch:train:1860-2002batch: iter_time=2.521e-04, forward_time=0.128, loss_att=34.509, acc=0.963, loss=34.509, backward_time=0.190, grad_norm=69.036, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.578 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 15:58:21,138 (trainer:732) INFO: 53epoch:train:2003-2145batch: iter_time=2.505e-04, forward_time=0.127, loss_att=31.561, acc=0.965, loss=31.561, backward_time=0.188, grad_norm=64.806, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:00:35,195 (trainer:732) INFO: 53epoch:train:2146-2288batch: iter_time=2.589e-04, forward_time=0.127, loss_att=32.973, acc=0.965, loss=32.973, backward_time=0.190, grad_norm=62.465, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.738 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:02:42,693 (trainer:732) INFO: 53epoch:train:2289-2431batch: iter_time=2.478e-04, forward_time=0.128, loss_att=37.658, acc=0.960, loss=37.658, backward_time=0.191, grad_norm=78.219, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.574 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:04:54,530 (trainer:732) INFO: 53epoch:train:2432-2574batch: iter_time=2.712e-04, forward_time=0.126, loss_att=29.383, acc=0.968, loss=29.383, backward_time=0.188, grad_norm=60.649, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:07:07,098 (trainer:732) INFO: 53epoch:train:2575-2717batch: iter_time=2.886e-04, forward_time=0.127, loss_att=30.991, acc=0.965, loss=30.991, backward_time=0.189, grad_norm=68.887, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:09:23,001 (trainer:732) INFO: 53epoch:train:2718-2860batch: iter_time=2.513e-04, forward_time=0.132, loss_att=39.715, acc=0.962, loss=39.715, backward_time=0.197, grad_norm=73.741, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.788 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:18:17,376 (trainer:338) INFO: 53epoch results: [train] iter_time=4.088e-04, forward_time=0.128, loss_att=33.389, acc=0.965, loss=33.389, backward_time=0.190, grad_norm=65.445, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.881, time=46 minutes and 35.01 seconds, total_count=152587, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.978, acc=0.969, cer=0.038, wer=0.135, loss=19.978, time=4 minutes and 7.74 seconds, total_count=4664, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.45 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:18:21,343 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:18:21,354 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/43epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:18:21,355 (trainer:272) INFO: 54/60epoch started. Estimated time to finish: 6 hours, 13 minutes and 37.93 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:22:53,643 (trainer:732) INFO: 54epoch:train:1-143batch: iter_time=0.003, forward_time=0.130, loss_att=30.648, acc=0.968, loss=30.648, backward_time=0.192, grad_norm=72.484, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=7.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:25:07,476 (trainer:732) INFO: 54epoch:train:144-286batch: iter_time=2.626e-04, forward_time=0.128, loss_att=29.432, acc=0.969, loss=29.432, backward_time=0.191, grad_norm=64.388, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.737 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:27:15,905 (trainer:732) INFO: 54epoch:train:287-429batch: iter_time=2.527e-04, forward_time=0.129, loss_att=33.020, acc=0.965, loss=33.020, backward_time=0.192, grad_norm=57.786, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.597 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:29:28,897 (trainer:732) INFO: 54epoch:train:430-572batch: iter_time=2.755e-04, forward_time=0.128, loss_att=32.093, acc=0.966, loss=32.093, backward_time=0.190, grad_norm=63.440, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.709 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:31:41,959 (trainer:732) INFO: 54epoch:train:573-715batch: iter_time=2.736e-04, forward_time=0.128, loss_att=31.824, acc=0.966, loss=31.824, backward_time=0.190, grad_norm=62.332, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:33:53,570 (trainer:732) INFO: 54epoch:train:716-858batch: iter_time=2.485e-04, forward_time=0.126, loss_att=30.809, acc=0.966, loss=30.809, backward_time=0.187, grad_norm=61.754, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.680 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:35:59,434 (trainer:732) INFO: 54epoch:train:859-1001batch: iter_time=2.662e-04, forward_time=0.127, loss_att=31.952, acc=0.965, loss=31.952, backward_time=0.189, grad_norm=56.632, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.518 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:38:15,109 (trainer:732) INFO: 54epoch:train:1002-1144batch: iter_time=2.810e-04, forward_time=0.130, loss_att=37.484, acc=0.963, loss=37.484, backward_time=0.194, grad_norm=70.466, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.785 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:40:27,433 (trainer:732) INFO: 54epoch:train:1145-1287batch: iter_time=2.676e-04, forward_time=0.128, loss_att=31.898, acc=0.966, loss=31.898, backward_time=0.190, grad_norm=64.097, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:42:35,746 (trainer:732) INFO: 54epoch:train:1288-1430batch: iter_time=2.685e-04, forward_time=0.129, loss_att=35.175, acc=0.964, loss=35.175, backward_time=0.192, grad_norm=64.514, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.590 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:44:48,354 (trainer:732) INFO: 54epoch:train:1431-1573batch: iter_time=2.649e-04, forward_time=0.128, loss_att=31.952, acc=0.966, loss=31.952, backward_time=0.190, grad_norm=64.448, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:47:00,886 (trainer:732) INFO: 54epoch:train:1574-1716batch: iter_time=2.684e-04, forward_time=0.128, loss_att=30.775, acc=0.966, loss=30.775, backward_time=0.191, grad_norm=74.770, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:49:12,722 (trainer:732) INFO: 54epoch:train:1717-1859batch: iter_time=2.656e-04, forward_time=0.127, loss_att=29.119, acc=0.967, loss=29.119, backward_time=0.189, grad_norm=61.383, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.703 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:51:20,954 (trainer:732) INFO: 54epoch:train:1860-2002batch: iter_time=2.519e-04, forward_time=0.126, loss_att=33.567, acc=0.963, loss=33.567, backward_time=0.188, grad_norm=67.574, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.580 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:53:34,508 (trainer:732) INFO: 54epoch:train:2003-2145batch: iter_time=2.450e-04, forward_time=0.127, loss_att=34.924, acc=0.964, loss=34.924, backward_time=0.189, grad_norm=64.406, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:55:48,089 (trainer:732) INFO: 54epoch:train:2146-2288batch: iter_time=2.561e-04, forward_time=0.128, loss_att=32.707, acc=0.967, loss=32.707, backward_time=0.191, grad_norm=70.380, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.727 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 16:57:55,617 (trainer:732) INFO: 54epoch:train:2289-2431batch: iter_time=2.422e-04, forward_time=0.128, loss_att=32.531, acc=0.965, loss=32.531, backward_time=0.190, grad_norm=67.054, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.575 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:00:10,523 (trainer:732) INFO: 54epoch:train:2432-2574batch: iter_time=2.567e-04, forward_time=0.129, loss_att=35.054, acc=0.963, loss=35.054, backward_time=0.192, grad_norm=61.054, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.770 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:02:24,793 (trainer:732) INFO: 54epoch:train:2575-2717batch: iter_time=2.625e-04, forward_time=0.128, loss_att=37.661, acc=0.962, loss=37.661, backward_time=0.191, grad_norm=70.082, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.757 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:04:37,626 (trainer:732) INFO: 54epoch:train:2718-2860batch: iter_time=2.549e-04, forward_time=0.127, loss_att=29.862, acc=0.967, loss=29.862, backward_time=0.189, grad_norm=60.298, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.706 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:13:28,932 (trainer:338) INFO: 54epoch results: [train] iter_time=3.782e-04, forward_time=0.128, loss_att=32.572, acc=0.965, loss=32.572, backward_time=0.190, grad_norm=64.970, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.881, time=46 minutes and 35.28 seconds, total_count=155466, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.782, acc=0.969, cer=0.038, wer=0.136, loss=19.782, time=4 minutes and 4.33 seconds, total_count=4752, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 27.96 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:13:33,125 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:13:33,137 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/42epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:13:33,137 (trainer:272) INFO: 55/60epoch started. Estimated time to finish: 5 hours, 20 minutes and 27.51 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:18:08,315 (trainer:732) INFO: 55epoch:train:1-143batch: iter_time=0.003, forward_time=0.129, loss_att=32.552, acc=0.967, loss=32.552, backward_time=0.192, grad_norm=67.005, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=7.792 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:20:21,381 (trainer:732) INFO: 55epoch:train:144-286batch: iter_time=2.838e-04, forward_time=0.128, loss_att=30.585, acc=0.968, loss=30.585, backward_time=0.191, grad_norm=65.389, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.719 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:22:29,175 (trainer:732) INFO: 55epoch:train:287-429batch: iter_time=2.712e-04, forward_time=0.129, loss_att=29.304, acc=0.969, loss=29.304, backward_time=0.192, grad_norm=65.398, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.581 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:24:41,766 (trainer:732) INFO: 55epoch:train:430-572batch: iter_time=2.782e-04, forward_time=0.126, loss_att=29.967, acc=0.968, loss=29.967, backward_time=0.187, grad_norm=64.557, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.695 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:26:55,207 (trainer:732) INFO: 55epoch:train:573-715batch: iter_time=2.805e-04, forward_time=0.129, loss_att=32.708, acc=0.967, loss=32.708, backward_time=0.192, grad_norm=70.526, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:29:08,214 (trainer:732) INFO: 55epoch:train:716-858batch: iter_time=2.812e-04, forward_time=0.128, loss_att=31.442, acc=0.966, loss=31.442, backward_time=0.190, grad_norm=62.986, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.719 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:31:16,558 (trainer:732) INFO: 55epoch:train:859-1001batch: iter_time=2.860e-04, forward_time=0.129, loss_att=31.951, acc=0.966, loss=31.951, backward_time=0.192, grad_norm=63.803, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.590 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:33:29,390 (trainer:732) INFO: 55epoch:train:1002-1144batch: iter_time=2.715e-04, forward_time=0.126, loss_att=31.644, acc=0.965, loss=31.644, backward_time=0.187, grad_norm=61.500, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:35:41,817 (trainer:732) INFO: 55epoch:train:1145-1287batch: iter_time=2.653e-04, forward_time=0.128, loss_att=31.414, acc=0.967, loss=31.414, backward_time=0.191, grad_norm=65.473, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.718 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:37:48,910 (trainer:732) INFO: 55epoch:train:1288-1430batch: iter_time=2.907e-04, forward_time=0.128, loss_att=32.524, acc=0.966, loss=32.524, backward_time=0.191, grad_norm=65.979, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.554 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:40:03,080 (trainer:732) INFO: 55epoch:train:1431-1573batch: iter_time=2.654e-04, forward_time=0.130, loss_att=32.370, acc=0.967, loss=32.370, backward_time=0.193, grad_norm=65.446, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.747 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:42:17,273 (trainer:732) INFO: 55epoch:train:1574-1716batch: iter_time=2.773e-04, forward_time=0.129, loss_att=33.344, acc=0.965, loss=33.344, backward_time=0.192, grad_norm=65.671, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.745 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:44:31,611 (trainer:732) INFO: 55epoch:train:1717-1859batch: iter_time=2.856e-04, forward_time=0.129, loss_att=32.292, acc=0.966, loss=32.292, backward_time=0.193, grad_norm=64.961, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.769 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:46:38,839 (trainer:732) INFO: 55epoch:train:1860-2002batch: iter_time=2.609e-04, forward_time=0.128, loss_att=31.526, acc=0.966, loss=31.526, backward_time=0.190, grad_norm=66.701, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.560 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:48:52,614 (trainer:732) INFO: 55epoch:train:2003-2145batch: iter_time=2.727e-04, forward_time=0.129, loss_att=32.129, acc=0.965, loss=32.129, backward_time=0.192, grad_norm=60.596, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.740 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:51:06,998 (trainer:732) INFO: 55epoch:train:2146-2288batch: iter_time=2.932e-04, forward_time=0.128, loss_att=33.737, acc=0.965, loss=33.737, backward_time=0.192, grad_norm=66.196, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.748 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:53:14,505 (trainer:732) INFO: 55epoch:train:2289-2431batch: iter_time=2.648e-04, forward_time=0.128, loss_att=33.552, acc=0.964, loss=33.552, backward_time=0.190, grad_norm=63.196, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.579 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:55:25,536 (trainer:732) INFO: 55epoch:train:2432-2574batch: iter_time=2.699e-04, forward_time=0.124, loss_att=29.617, acc=0.966, loss=29.617, backward_time=0.184, grad_norm=62.070, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.659 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:57:37,555 (trainer:732) INFO: 55epoch:train:2575-2717batch: iter_time=2.862e-04, forward_time=0.126, loss_att=27.494, acc=0.969, loss=27.494, backward_time=0.186, grad_norm=60.141, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 17:59:51,426 (trainer:732) INFO: 55epoch:train:2718-2860batch: iter_time=2.426e-04, forward_time=0.129, loss_att=33.861, acc=0.964, loss=33.861, backward_time=0.193, grad_norm=71.780, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:08:36,332 (trainer:338) INFO: 55epoch results: [train] iter_time=4.060e-04, forward_time=0.128, loss_att=31.666, acc=0.966, loss=31.666, backward_time=0.190, grad_norm=64.936, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.877, time=46 minutes and 32.28 seconds, total_count=158345, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.418, acc=0.970, cer=0.037, wer=0.133, loss=19.418, time=4 minutes and 3.94 seconds, total_count=4840, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 26.98 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:08:40,450 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:08:40,465 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/44epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:08:40,465 (trainer:272) INFO: 56/60epoch started. Estimated time to finish: 4 hours, 27 minutes and 12.26 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:13:12,335 (trainer:732) INFO: 56epoch:train:1-143batch: iter_time=0.003, forward_time=0.129, loss_att=29.423, acc=0.968, loss=29.423, backward_time=0.190, grad_norm=65.022, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=7.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:15:25,516 (trainer:732) INFO: 56epoch:train:144-286batch: iter_time=2.749e-04, forward_time=0.129, loss_att=31.455, acc=0.967, loss=31.455, backward_time=0.191, grad_norm=67.147, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.721 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:17:34,164 (trainer:732) INFO: 56epoch:train:287-429batch: iter_time=2.847e-04, forward_time=0.129, loss_att=31.151, acc=0.967, loss=31.151, backward_time=0.192, grad_norm=65.314, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.602 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:19:46,514 (trainer:732) INFO: 56epoch:train:430-572batch: iter_time=2.818e-04, forward_time=0.126, loss_att=27.481, acc=0.969, loss=27.481, backward_time=0.187, grad_norm=63.442, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.690 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:21:59,390 (trainer:732) INFO: 56epoch:train:573-715batch: iter_time=2.808e-04, forward_time=0.129, loss_att=30.527, acc=0.968, loss=30.527, backward_time=0.192, grad_norm=63.716, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:24:12,802 (trainer:732) INFO: 56epoch:train:716-858batch: iter_time=2.785e-04, forward_time=0.128, loss_att=29.984, acc=0.968, loss=29.984, backward_time=0.191, grad_norm=59.911, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.723 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:26:21,234 (trainer:732) INFO: 56epoch:train:859-1001batch: iter_time=2.895e-04, forward_time=0.129, loss_att=30.663, acc=0.968, loss=30.663, backward_time=0.192, grad_norm=64.087, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.592 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:28:34,218 (trainer:732) INFO: 56epoch:train:1002-1144batch: iter_time=2.711e-04, forward_time=0.128, loss_att=29.543, acc=0.968, loss=29.543, backward_time=0.190, grad_norm=60.756, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.710 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:30:47,570 (trainer:732) INFO: 56epoch:train:1145-1287batch: iter_time=2.798e-04, forward_time=0.129, loss_att=33.553, acc=0.965, loss=33.553, backward_time=0.192, grad_norm=65.606, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.740 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:32:56,862 (trainer:732) INFO: 56epoch:train:1288-1430batch: iter_time=2.791e-04, forward_time=0.129, loss_att=30.914, acc=0.967, loss=30.914, backward_time=0.193, grad_norm=69.132, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.614 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:35:08,138 (trainer:732) INFO: 56epoch:train:1431-1573batch: iter_time=2.656e-04, forward_time=0.127, loss_att=30.314, acc=0.967, loss=30.314, backward_time=0.188, grad_norm=64.413, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.676 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:37:23,196 (trainer:732) INFO: 56epoch:train:1574-1716batch: iter_time=2.795e-04, forward_time=0.130, loss_att=33.319, acc=0.965, loss=33.319, backward_time=0.193, grad_norm=71.705, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.766 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:39:32,145 (trainer:732) INFO: 56epoch:train:1717-1859batch: iter_time=2.904e-04, forward_time=0.127, loss_att=29.638, acc=0.968, loss=29.638, backward_time=0.188, grad_norm=67.901, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.649 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:41:43,144 (trainer:732) INFO: 56epoch:train:1860-2002batch: iter_time=2.616e-04, forward_time=0.129, loss_att=31.984, acc=0.967, loss=31.984, backward_time=0.192, grad_norm=64.644, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.627 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:43:56,585 (trainer:732) INFO: 56epoch:train:2003-2145batch: iter_time=2.876e-04, forward_time=0.128, loss_att=31.044, acc=0.966, loss=31.044, backward_time=0.190, grad_norm=69.023, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.738 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:46:10,396 (trainer:732) INFO: 56epoch:train:2146-2288batch: iter_time=2.933e-04, forward_time=0.127, loss_att=31.899, acc=0.966, loss=31.899, backward_time=0.189, grad_norm=73.234, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:48:17,370 (trainer:732) INFO: 56epoch:train:2289-2431batch: iter_time=2.664e-04, forward_time=0.128, loss_att=32.186, acc=0.966, loss=32.186, backward_time=0.190, grad_norm=67.688, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.558 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:50:30,686 (trainer:732) INFO: 56epoch:train:2432-2574batch: iter_time=2.766e-04, forward_time=0.128, loss_att=30.727, acc=0.967, loss=30.727, backward_time=0.190, grad_norm=58.342, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.731 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:52:45,933 (trainer:732) INFO: 56epoch:train:2575-2717batch: iter_time=2.812e-04, forward_time=0.130, loss_att=34.023, acc=0.965, loss=34.023, backward_time=0.195, grad_norm=68.164, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.778 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 18:54:52,712 (trainer:732) INFO: 56epoch:train:2718-2860batch: iter_time=2.670e-04, forward_time=0.125, loss_att=27.980, acc=0.969, loss=27.980, backward_time=0.185, grad_norm=59.198, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.539 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:03:33,911 (trainer:338) INFO: 56epoch results: [train] iter_time=4.178e-04, forward_time=0.128, loss_att=30.856, acc=0.967, loss=30.856, backward_time=0.190, grad_norm=65.418, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.877, time=46 minutes and 32.1 seconds, total_count=161224, gpu_max_cached_mem_GB=23.359, [valid] loss_att=20.229, acc=0.969, cer=0.038, wer=0.135, loss=20.229, time=3 minutes and 56.13 seconds, total_count=4928, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.22 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:03:37,655 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:03:37,668 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/48epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:03:37,668 (trainer:272) INFO: 57/60epoch started. Estimated time to finish: 3 hours, 33 minutes and 52.29 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:08:09,476 (trainer:732) INFO: 57epoch:train:1-143batch: iter_time=0.003, forward_time=0.128, loss_att=27.987, acc=0.970, loss=27.987, backward_time=0.189, grad_norm=66.960, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=7.694 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:10:23,344 (trainer:732) INFO: 57epoch:train:144-286batch: iter_time=2.745e-04, forward_time=0.129, loss_att=31.404, acc=0.968, loss=31.404, backward_time=0.193, grad_norm=64.768, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.747 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:12:29,978 (trainer:732) INFO: 57epoch:train:287-429batch: iter_time=2.745e-04, forward_time=0.129, loss_att=30.054, acc=0.969, loss=30.054, backward_time=0.192, grad_norm=63.700, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.540 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:14:42,955 (trainer:732) INFO: 57epoch:train:430-572batch: iter_time=2.768e-04, forward_time=0.128, loss_att=29.150, acc=0.968, loss=29.150, backward_time=0.190, grad_norm=64.831, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.711 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:16:55,455 (trainer:732) INFO: 57epoch:train:573-715batch: iter_time=2.812e-04, forward_time=0.127, loss_att=30.354, acc=0.968, loss=30.354, backward_time=0.190, grad_norm=64.938, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.713 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:19:09,050 (trainer:732) INFO: 57epoch:train:716-858batch: iter_time=2.733e-04, forward_time=0.128, loss_att=30.237, acc=0.968, loss=30.237, backward_time=0.191, grad_norm=64.276, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:21:15,816 (trainer:732) INFO: 57epoch:train:859-1001batch: iter_time=2.891e-04, forward_time=0.128, loss_att=28.472, acc=0.969, loss=28.472, backward_time=0.190, grad_norm=57.035, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.545 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:23:29,317 (trainer:732) INFO: 57epoch:train:1002-1144batch: iter_time=2.773e-04, forward_time=0.128, loss_att=29.320, acc=0.968, loss=29.320, backward_time=0.190, grad_norm=64.020, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.722 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:25:43,479 (trainer:732) INFO: 57epoch:train:1145-1287batch: iter_time=2.645e-04, forward_time=0.130, loss_att=32.130, acc=0.968, loss=32.130, backward_time=0.194, grad_norm=68.115, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.767 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:27:51,814 (trainer:732) INFO: 57epoch:train:1288-1430batch: iter_time=2.844e-04, forward_time=0.128, loss_att=31.168, acc=0.967, loss=31.168, backward_time=0.191, grad_norm=67.399, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.588 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:30:05,645 (trainer:732) INFO: 57epoch:train:1431-1573batch: iter_time=2.753e-04, forward_time=0.128, loss_att=32.036, acc=0.967, loss=32.036, backward_time=0.191, grad_norm=64.625, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.743 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:32:19,026 (trainer:732) INFO: 57epoch:train:1574-1716batch: iter_time=2.710e-04, forward_time=0.128, loss_att=30.978, acc=0.967, loss=30.978, backward_time=0.191, grad_norm=60.312, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.720 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:34:32,313 (trainer:732) INFO: 57epoch:train:1717-1859batch: iter_time=2.883e-04, forward_time=0.129, loss_att=32.016, acc=0.966, loss=32.016, backward_time=0.191, grad_norm=63.128, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.738 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:36:40,338 (trainer:732) INFO: 57epoch:train:1860-2002batch: iter_time=2.515e-04, forward_time=0.128, loss_att=28.908, acc=0.969, loss=28.908, backward_time=0.190, grad_norm=63.881, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.584 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:38:54,072 (trainer:732) INFO: 57epoch:train:2003-2145batch: iter_time=2.657e-04, forward_time=0.127, loss_att=31.964, acc=0.966, loss=31.964, backward_time=0.190, grad_norm=64.545, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.737 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:41:06,831 (trainer:732) INFO: 57epoch:train:2146-2288batch: iter_time=2.881e-04, forward_time=0.127, loss_att=28.421, acc=0.969, loss=28.421, backward_time=0.189, grad_norm=61.951, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:43:13,934 (trainer:732) INFO: 57epoch:train:2289-2431batch: iter_time=2.680e-04, forward_time=0.127, loss_att=29.508, acc=0.968, loss=29.508, backward_time=0.188, grad_norm=62.188, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.565 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:45:26,281 (trainer:732) INFO: 57epoch:train:2432-2574batch: iter_time=2.706e-04, forward_time=0.127, loss_att=29.810, acc=0.968, loss=29.810, backward_time=0.188, grad_norm=60.752, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.699 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:47:38,800 (trainer:732) INFO: 57epoch:train:2575-2717batch: iter_time=2.708e-04, forward_time=0.128, loss_att=31.902, acc=0.966, loss=31.902, backward_time=0.191, grad_norm=62.941, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.708 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:49:51,532 (trainer:732) INFO: 57epoch:train:2718-2860batch: iter_time=2.591e-04, forward_time=0.127, loss_att=28.974, acc=0.968, loss=28.974, backward_time=0.189, grad_norm=61.631, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.700 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:58:37,702 (trainer:338) INFO: 57epoch results: [train] iter_time=3.976e-04, forward_time=0.128, loss_att=30.206, acc=0.968, loss=30.206, backward_time=0.190, grad_norm=63.550, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.878, time=46 minutes and 33.46 seconds, total_count=164103, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.615, acc=0.969, cer=0.038, wer=0.134, loss=19.615, time=4 minutes and 0.76 seconds, total_count=5016, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.81 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:58:41,395 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:58:41,409 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/49epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 19:58:41,410 (trainer:272) INFO: 58/60epoch started. Estimated time to finish: 2 hours, 40 minutes and 29.25 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:03:12,447 (trainer:732) INFO: 58epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=27.089, acc=0.971, loss=27.089, backward_time=0.189, grad_norm=67.069, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=7.674 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:05:24,325 (trainer:732) INFO: 58epoch:train:144-286batch: iter_time=2.631e-04, forward_time=0.127, loss_att=27.892, acc=0.970, loss=27.892, backward_time=0.189, grad_norm=62.255, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.690 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:07:32,381 (trainer:732) INFO: 58epoch:train:287-429batch: iter_time=2.567e-04, forward_time=0.127, loss_att=28.275, acc=0.970, loss=28.275, backward_time=0.190, grad_norm=58.828, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.582 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:09:46,542 (trainer:732) INFO: 58epoch:train:430-572batch: iter_time=2.818e-04, forward_time=0.128, loss_att=28.887, acc=0.969, loss=28.887, backward_time=0.191, grad_norm=64.136, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.742 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:11:58,853 (trainer:732) INFO: 58epoch:train:573-715batch: iter_time=2.536e-04, forward_time=0.128, loss_att=28.982, acc=0.968, loss=28.982, backward_time=0.190, grad_norm=59.231, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.712 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:14:13,712 (trainer:732) INFO: 58epoch:train:716-858batch: iter_time=2.513e-04, forward_time=0.129, loss_att=31.844, acc=0.967, loss=31.844, backward_time=0.193, grad_norm=70.257, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.771 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:16:21,118 (trainer:732) INFO: 58epoch:train:859-1001batch: iter_time=2.726e-04, forward_time=0.127, loss_att=28.852, acc=0.969, loss=28.852, backward_time=0.190, grad_norm=64.495, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.563 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:18:34,534 (trainer:732) INFO: 58epoch:train:1002-1144batch: iter_time=2.716e-04, forward_time=0.129, loss_att=31.067, acc=0.967, loss=31.067, backward_time=0.192, grad_norm=62.431, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.722 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:20:46,012 (trainer:732) INFO: 58epoch:train:1145-1287batch: iter_time=2.490e-04, forward_time=0.128, loss_att=28.199, acc=0.970, loss=28.199, backward_time=0.191, grad_norm=72.138, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.685 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:22:53,673 (trainer:732) INFO: 58epoch:train:1288-1430batch: iter_time=2.491e-04, forward_time=0.128, loss_att=28.441, acc=0.970, loss=28.441, backward_time=0.191, grad_norm=64.637, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.572 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:25:05,834 (trainer:732) INFO: 58epoch:train:1431-1573batch: iter_time=2.501e-04, forward_time=0.127, loss_att=27.321, acc=0.970, loss=27.321, backward_time=0.189, grad_norm=63.549, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.697 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:27:20,121 (trainer:732) INFO: 58epoch:train:1574-1716batch: iter_time=2.531e-04, forward_time=0.129, loss_att=33.501, acc=0.966, loss=33.501, backward_time=0.192, grad_norm=68.352, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.746 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:29:31,650 (trainer:732) INFO: 58epoch:train:1717-1859batch: iter_time=2.730e-04, forward_time=0.125, loss_att=27.525, acc=0.969, loss=27.525, backward_time=0.186, grad_norm=61.377, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.686 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:31:38,591 (trainer:732) INFO: 58epoch:train:1860-2002batch: iter_time=2.696e-04, forward_time=0.127, loss_att=27.769, acc=0.969, loss=27.769, backward_time=0.188, grad_norm=62.471, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.556 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:33:52,044 (trainer:732) INFO: 58epoch:train:2003-2145batch: iter_time=2.680e-04, forward_time=0.127, loss_att=28.559, acc=0.969, loss=28.559, backward_time=0.189, grad_norm=61.202, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.732 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:36:06,489 (trainer:732) INFO: 58epoch:train:2146-2288batch: iter_time=2.786e-04, forward_time=0.129, loss_att=31.749, acc=0.968, loss=31.749, backward_time=0.193, grad_norm=73.841, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=3.749 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:38:14,787 (trainer:732) INFO: 58epoch:train:2289-2431batch: iter_time=2.523e-04, forward_time=0.129, loss_att=34.532, acc=0.965, loss=34.532, backward_time=0.193, grad_norm=69.687, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.595 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:40:28,304 (trainer:732) INFO: 58epoch:train:2432-2574batch: iter_time=2.583e-04, forward_time=0.129, loss_att=30.259, acc=0.969, loss=30.259, backward_time=0.193, grad_norm=64.891, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:42:42,078 (trainer:732) INFO: 58epoch:train:2575-2717batch: iter_time=2.674e-04, forward_time=0.128, loss_att=30.946, acc=0.967, loss=30.946, backward_time=0.191, grad_norm=64.600, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.738 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:44:54,855 (trainer:732) INFO: 58epoch:train:2718-2860batch: iter_time=2.364e-04, forward_time=0.127, loss_att=30.420, acc=0.968, loss=30.420, backward_time=0.190, grad_norm=67.934, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.001, train_time=3.705 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:53:45,820 (trainer:338) INFO: 58epoch results: [train] iter_time=3.686e-04, forward_time=0.128, loss_att=29.498, acc=0.969, loss=29.498, backward_time=0.190, grad_norm=65.106, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.876, time=46 minutes and 30.49 seconds, total_count=166982, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.579, acc=0.970, cer=0.037, wer=0.132, loss=19.579, time=4 minutes and 5.03 seconds, total_count=5104, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 28.89 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:53:49,805 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:53:49,817 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/47epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:53:49,818 (trainer:272) INFO: 59/60epoch started. Estimated time to finish: 1 hour, 47 minutes and 2.9 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 20:58:22,400 (trainer:732) INFO: 59epoch:train:1-143batch: iter_time=0.002, forward_time=0.128, loss_att=26.396, acc=0.971, loss=26.396, backward_time=0.190, grad_norm=59.327, clip=100.000, loss_scale=1.000, optim_step_time=0.063, optim0_lr0=0.001, train_time=7.719 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:00:37,114 (trainer:732) INFO: 59epoch:train:144-286batch: iter_time=2.646e-04, forward_time=0.130, loss_att=30.687, acc=0.970, loss=30.687, backward_time=0.194, grad_norm=66.526, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.764 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:02:46,230 (trainer:732) INFO: 59epoch:train:287-429batch: iter_time=2.722e-04, forward_time=0.130, loss_att=29.230, acc=0.970, loss=29.230, backward_time=0.194, grad_norm=62.658, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.613 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:04:59,897 (trainer:732) INFO: 59epoch:train:430-572batch: iter_time=2.817e-04, forward_time=0.127, loss_att=27.574, acc=0.970, loss=27.574, backward_time=0.190, grad_norm=61.279, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.730 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:07:12,387 (trainer:732) INFO: 59epoch:train:573-715batch: iter_time=2.830e-04, forward_time=0.127, loss_att=28.613, acc=0.968, loss=28.613, backward_time=0.188, grad_norm=57.908, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:09:25,577 (trainer:732) INFO: 59epoch:train:716-858batch: iter_time=2.640e-04, forward_time=0.129, loss_att=30.528, acc=0.968, loss=30.528, backward_time=0.192, grad_norm=63.649, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.001, train_time=3.744 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:11:33,602 (trainer:732) INFO: 59epoch:train:859-1001batch: iter_time=2.688e-04, forward_time=0.127, loss_att=27.734, acc=0.969, loss=27.734, backward_time=0.190, grad_norm=62.642, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.563 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:13:47,575 (trainer:732) INFO: 59epoch:train:1002-1144batch: iter_time=2.705e-04, forward_time=0.128, loss_att=29.948, acc=0.969, loss=29.948, backward_time=0.191, grad_norm=60.550, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:15:59,733 (trainer:732) INFO: 59epoch:train:1145-1287batch: iter_time=2.591e-04, forward_time=0.128, loss_att=29.004, acc=0.969, loss=29.004, backward_time=0.190, grad_norm=63.398, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.707 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:18:06,836 (trainer:732) INFO: 59epoch:train:1288-1430batch: iter_time=2.756e-04, forward_time=0.129, loss_att=28.473, acc=0.969, loss=28.473, backward_time=0.192, grad_norm=61.617, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.556 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:20:19,143 (trainer:732) INFO: 59epoch:train:1431-1573batch: iter_time=2.617e-04, forward_time=0.126, loss_att=28.167, acc=0.969, loss=28.167, backward_time=0.187, grad_norm=62.204, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.698 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:22:32,534 (trainer:732) INFO: 59epoch:train:1574-1716batch: iter_time=2.650e-04, forward_time=0.126, loss_att=29.388, acc=0.969, loss=29.388, backward_time=0.188, grad_norm=62.023, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.723 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:24:42,160 (trainer:732) INFO: 59epoch:train:1717-1859batch: iter_time=2.712e-04, forward_time=0.130, loss_att=29.327, acc=0.970, loss=29.327, backward_time=0.194, grad_norm=66.775, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.670 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:26:53,426 (trainer:732) INFO: 59epoch:train:1860-2002batch: iter_time=2.763e-04, forward_time=0.127, loss_att=25.978, acc=0.972, loss=25.978, backward_time=0.189, grad_norm=61.289, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.637 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:29:07,271 (trainer:732) INFO: 59epoch:train:2003-2145batch: iter_time=2.872e-04, forward_time=0.129, loss_att=31.009, acc=0.968, loss=31.009, backward_time=0.191, grad_norm=62.983, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.745 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:31:20,989 (trainer:732) INFO: 59epoch:train:2146-2288batch: iter_time=2.678e-04, forward_time=0.128, loss_att=29.189, acc=0.969, loss=29.189, backward_time=0.191, grad_norm=71.977, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.001, train_time=3.728 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:33:27,398 (trainer:732) INFO: 59epoch:train:2289-2431batch: iter_time=2.724e-04, forward_time=0.127, loss_att=26.761, acc=0.970, loss=26.761, backward_time=0.189, grad_norm=59.267, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.544 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:35:39,784 (trainer:732) INFO: 59epoch:train:2432-2574batch: iter_time=2.771e-04, forward_time=0.126, loss_att=28.659, acc=0.969, loss=28.659, backward_time=0.188, grad_norm=67.125, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.701 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] include/socket.h:423 NCCL WARN Net : Connection closed by remote peer 10.38.17.248<48138> +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:445 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO include/socket.h:457 -> 2 +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] NCCL INFO bootstrap.cc:229 -> 2 + +de-74279-k2-train-3-0115153910-566d497669-rr6jb:3146342:3146537 [0] bootstrap.cc:279 NCCL WARN [Rem Allocator] Allocation failed (segment 3, fd 153) +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:37:53,938 (trainer:732) INFO: 59epoch:train:2575-2717batch: iter_time=2.770e-04, forward_time=0.128, loss_att=30.371, acc=0.968, loss=30.371, backward_time=0.191, grad_norm=62.720, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.752 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:40:02,583 (trainer:732) INFO: 59epoch:train:2718-2860batch: iter_time=2.588e-04, forward_time=0.128, loss_att=29.061, acc=0.970, loss=29.061, backward_time=0.192, grad_norm=66.489, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.589 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:48:52,044 (trainer:338) INFO: 59epoch results: [train] iter_time=3.782e-04, forward_time=0.128, loss_att=28.779, acc=0.969, loss=28.779, backward_time=0.190, grad_norm=63.146, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.876, time=46 minutes and 32.15 seconds, total_count=169861, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.639, acc=0.970, cer=0.037, wer=0.133, loss=19.639, time=4 minutes and 4.09 seconds, total_count=5192, gpu_max_cached_mem_GB=23.359, [att_plot] time=4 minutes and 25.99 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:48:55,923 (trainer:384) INFO: There are no improvements in this epoch +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:48:55,938 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/46epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:48:55,938 (trainer:272) INFO: 60/60epoch started. Estimated time to finish: 53 minutes and 33.06 seconds +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:53:28,470 (trainer:732) INFO: 60epoch:train:1-143batch: iter_time=0.004, forward_time=0.128, loss_att=26.632, acc=0.971, loss=26.632, backward_time=0.189, grad_norm=57.826, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=7.716 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:55:42,029 (trainer:732) INFO: 60epoch:train:144-286batch: iter_time=3.136e-04, forward_time=0.129, loss_att=27.204, acc=0.971, loss=27.204, backward_time=0.191, grad_norm=66.109, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.736 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 21:57:48,364 (trainer:732) INFO: 60epoch:train:287-429batch: iter_time=2.752e-04, forward_time=0.127, loss_att=27.249, acc=0.971, loss=27.249, backward_time=0.190, grad_norm=61.080, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.533 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:00:01,577 (trainer:732) INFO: 60epoch:train:430-572batch: iter_time=2.849e-04, forward_time=0.128, loss_att=29.151, acc=0.970, loss=29.151, backward_time=0.191, grad_norm=63.892, clip=100.000, loss_scale=1.000, optim_step_time=0.058, optim0_lr0=0.001, train_time=3.717 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:02:15,340 (trainer:732) INFO: 60epoch:train:573-715batch: iter_time=2.824e-04, forward_time=0.128, loss_att=28.556, acc=0.970, loss=28.556, backward_time=0.191, grad_norm=58.848, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.758 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:04:29,149 (trainer:732) INFO: 60epoch:train:716-858batch: iter_time=2.717e-04, forward_time=0.129, loss_att=27.796, acc=0.970, loss=27.796, backward_time=0.191, grad_norm=64.664, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.738 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:06:34,552 (trainer:732) INFO: 60epoch:train:859-1001batch: iter_time=2.814e-04, forward_time=0.126, loss_att=25.858, acc=0.971, loss=25.858, backward_time=0.188, grad_norm=64.843, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.510 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:08:48,446 (trainer:732) INFO: 60epoch:train:1002-1144batch: iter_time=2.643e-04, forward_time=0.128, loss_att=28.550, acc=0.970, loss=28.550, backward_time=0.190, grad_norm=59.027, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.732 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:11:02,679 (trainer:732) INFO: 60epoch:train:1145-1287batch: iter_time=2.863e-04, forward_time=0.129, loss_att=28.833, acc=0.970, loss=28.833, backward_time=0.193, grad_norm=70.012, clip=100.000, loss_scale=1.000, optim_step_time=0.062, optim0_lr0=0.001, train_time=3.767 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:13:11,624 (trainer:732) INFO: 60epoch:train:1288-1430batch: iter_time=2.911e-04, forward_time=0.128, loss_att=28.896, acc=0.970, loss=28.896, backward_time=0.192, grad_norm=68.340, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.607 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:15:25,916 (trainer:732) INFO: 60epoch:train:1431-1573batch: iter_time=2.655e-04, forward_time=0.129, loss_att=27.947, acc=0.971, loss=27.947, backward_time=0.192, grad_norm=64.485, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.754 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:17:38,052 (trainer:732) INFO: 60epoch:train:1574-1716batch: iter_time=2.767e-04, forward_time=0.127, loss_att=26.118, acc=0.971, loss=26.118, backward_time=0.188, grad_norm=60.068, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.687 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:19:50,236 (trainer:732) INFO: 60epoch:train:1717-1859batch: iter_time=2.859e-04, forward_time=0.127, loss_att=27.587, acc=0.970, loss=27.587, backward_time=0.189, grad_norm=59.719, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.725 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:21:58,766 (trainer:732) INFO: 60epoch:train:1860-2002batch: iter_time=2.721e-04, forward_time=0.129, loss_att=28.893, acc=0.970, loss=28.893, backward_time=0.192, grad_norm=69.667, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.582 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:24:11,435 (trainer:732) INFO: 60epoch:train:2003-2145batch: iter_time=2.862e-04, forward_time=0.127, loss_att=28.272, acc=0.969, loss=28.272, backward_time=0.189, grad_norm=58.217, clip=100.000, loss_scale=1.000, optim_step_time=0.061, optim0_lr0=0.001, train_time=3.704 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:26:25,803 (trainer:732) INFO: 60epoch:train:2146-2288batch: iter_time=2.855e-04, forward_time=0.128, loss_att=28.975, acc=0.969, loss=28.975, backward_time=0.191, grad_norm=58.998, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.751 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:28:33,922 (trainer:732) INFO: 60epoch:train:2289-2431batch: iter_time=2.580e-04, forward_time=0.128, loss_att=30.593, acc=0.968, loss=30.593, backward_time=0.191, grad_norm=71.340, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.591 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:30:45,880 (trainer:732) INFO: 60epoch:train:2432-2574batch: iter_time=2.706e-04, forward_time=0.128, loss_att=27.855, acc=0.969, loss=27.855, backward_time=0.190, grad_norm=61.918, clip=100.000, loss_scale=1.000, optim_step_time=0.057, optim0_lr0=0.001, train_time=3.691 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:33:01,083 (trainer:732) INFO: 60epoch:train:2575-2717batch: iter_time=2.677e-04, forward_time=0.129, loss_att=30.476, acc=0.969, loss=30.476, backward_time=0.192, grad_norm=65.470, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.780 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:35:08,512 (trainer:732) INFO: 60epoch:train:2718-2860batch: iter_time=2.641e-04, forward_time=0.127, loss_att=27.504, acc=0.970, loss=27.504, backward_time=0.189, grad_norm=61.953, clip=100.000, loss_scale=1.000, optim_step_time=0.059, optim0_lr0=0.001, train_time=3.556 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:38,952 (trainer:338) INFO: 60epoch results: [train] iter_time=4.494e-04, forward_time=0.128, loss_att=28.099, acc=0.970, loss=28.099, backward_time=0.190, grad_norm=63.307, clip=100.000, loss_scale=1.000, optim_step_time=0.060, optim0_lr0=0.001, train_time=3.876, time=46 minutes and 31.62 seconds, total_count=172740, gpu_max_cached_mem_GB=23.359, [valid] loss_att=19.405, acc=0.970, cer=0.037, wer=0.131, loss=19.405, time=3 minutes and 56.89 seconds, total_count=5280, gpu_max_cached_mem_GB=23.359, [att_plot] time=3 minutes and 14.5 seconds, total_count=0, gpu_max_cached_mem_GB=23.359 +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:42,938 (trainer:386) INFO: The best model has been updated: valid.acc +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:42,951 (trainer:440) INFO: The model files were removed: exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/50epoch.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:42,951 (trainer:458) INFO: The training was finished at 60 epochs +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:42,979 (average_nbest_models:69) INFO: Averaging 10best models: criterion="valid.acc": exp/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.ave_10best.pth +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,820 (average_nbest_models:96) INFO: Accumulating encoder.encoders.0.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,823 (average_nbest_models:96) INFO: Accumulating encoder.encoders.1.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,825 (average_nbest_models:96) INFO: Accumulating encoder.encoders.2.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,827 (average_nbest_models:96) INFO: Accumulating encoder.encoders.3.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,829 (average_nbest_models:96) INFO: Accumulating encoder.encoders.4.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,831 (average_nbest_models:96) INFO: Accumulating encoder.encoders.5.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,833 (average_nbest_models:96) INFO: Accumulating encoder.encoders.6.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,836 (average_nbest_models:96) INFO: Accumulating encoder.encoders.7.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,838 (average_nbest_models:96) INFO: Accumulating encoder.encoders.8.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,840 (average_nbest_models:96) INFO: Accumulating encoder.encoders.9.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,842 (average_nbest_models:96) INFO: Accumulating encoder.encoders.10.conv_module.norm.num_batches_tracked instead of averaging +[de-74279-k2-train-3-0115153910-566d497669-rr6jb:0/4] 2024-02-22 22:42:52,844 (average_nbest_models:96) INFO: Accumulating encoder.encoders.11.conv_module.norm.num_batches_tracked instead of averaging +# Accounting: time=192843 threads=1 +# Ended (code 0) at Thu Feb 22 22:42:56 CST 2024, elapsed time 192843 seconds diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.ave.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.ave.pth new file mode 100644 index 0000000000000000000000000000000000000000..619655ce85f821973651fd4d31aee3cde366c2dc --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.ave.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:452c46486a6c10cec8bff7f5b35febab878e74ed78f7117af7a56e93c2c61310 +size 172358249 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.ave_10best.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.ave_10best.pth new file mode 100644 index 0000000000000000000000000000000000000000..619655ce85f821973651fd4d31aee3cde366c2dc --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.ave_10best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:452c46486a6c10cec8bff7f5b35febab878e74ed78f7117af7a56e93c2c61310 +size 172358249 diff --git a/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.best.pth b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.best.pth new file mode 100644 index 0000000000000000000000000000000000000000..07cffd1b1c091ad8875aa1b808929430842084db --- /dev/null +++ b/small/asr_train_sot_asr_conformer_raw_en_char_sp_finetune_ls100_45epoch_new_whamr/valid.acc.best.pth @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fc0710e1565fce6d3ce275190a1335a03d912f728a67066516df16d751f26514 +size 172367337