add perf log

perf_log/model_repo_tlg_mbr/rtf-20.txt

@@ -0,0 +1,4 @@
+RTF: 0.0033
+total_duration: 36108.919 seconds
+(10.03 hours)
+processing time: 118.776 seconds (0.03 hours)

perf_log/model_repo_tlg_mbr/rtf-40.txt

@@ -0,0 +1,4 @@
+RTF: 0.0023
+total_duration: 36108.919 seconds
+(10.03 hours)
+processing time: 82.408 seconds (0.02 hours)

perf_log/model_repo_tlg_mbr/rtf-60.txt

@@ -0,0 +1,4 @@
+RTF: 0.0020
+total_duration: 36108.919 seconds
+(10.03 hours)
+processing time: 72.950 seconds (0.02 hours)

perf_log/model_repo_tlg_mbr/rtf-80.txt

@@ -0,0 +1,4 @@
+RTF: 0.0020
+total_duration: 36108.919 seconds
+(10.03 hours)
+processing time: 70.790 seconds (0.02 hours)

perf_log/model_repo_tlg_mbr/stats-20-summary-new.txt

@@ -0,0 +1,52 @@
+model name is attention_rescoring 
+queue 0.01  s, infer 1425.32 s, input 51.35 s, output 12.41 s 
+Batch_size 1 , 7176  times, infer 1425319.75 ms, avg 198.62 ms, 198.62 ms input 51352.69  ms, avg 7.16 ms, output 12410.77  ms, avg 1.73 ms 
+model name is encoder 
+queue 78.94 s, infer 253.77 s, input 1.73  s, output 4.18  s 
+Batch_size 1 , 3029  times, infer 44498.59  ms, avg 14.69 ms, 14.69 ms input 890.41    ms, avg 0.29 ms, output 1012.47   ms, avg 0.33 ms 
+Batch_size 2 , 217   times, infer 5660.02   ms, avg 26.08 ms, 13.04 ms input 34.80     ms, avg 0.16 ms, output 119.71    ms, avg 0.55 ms 
+Batch_size 3 , 168   times, infer 4610.71   ms, avg 27.44 ms, 9.15 ms input 25.79     ms, avg 0.15 ms, output 86.97     ms, avg 0.52 ms 
+Batch_size 4 , 173   times, infer 5293.88   ms, avg 30.60 ms, 7.65 ms input 36.04     ms, avg 0.21 ms, output 119.68    ms, avg 0.69 ms 
+Batch_size 5 , 124   times, infer 4076.08   ms, avg 32.87 ms, 6.57 ms input 16.84     ms, avg 0.14 ms, output 74.42     ms, avg 0.60 ms 
+Batch_size 6 , 78    times, infer 3212.68   ms, avg 41.19 ms, 6.86 ms input 15.36     ms, avg 0.20 ms, output 70.23     ms, avg 0.90 ms 
+Batch_size 7 , 51    times, infer 2474.50   ms, avg 48.52 ms, 6.93 ms input 9.24      ms, avg 0.18 ms, output 38.03     ms, avg 0.75 ms 
+Batch_size 8 , 43    times, infer 2384.90   ms, avg 55.46 ms, 6.93 ms input 9.87      ms, avg 0.23 ms, output 39.27     ms, avg 0.91 ms 
+Batch_size 9 , 26    times, infer 1916.89   ms, avg 73.73 ms, 8.19 ms input 6.30      ms, avg 0.24 ms, output 22.52     ms, avg 0.87 ms 
+Batch_size 10, 26    times, infer 2177.51   ms, avg 83.75 ms, 8.38 ms input 9.85      ms, avg 0.38 ms, output 46.08     ms, avg 1.77 ms 
+Batch_size 11, 11    times, infer 1581.82   ms, avg 143.80 ms, 13.07 ms input 2.65      ms, avg 0.24 ms, output 7.33      ms, avg 0.67 ms 
+Batch_size 12, 7     times, infer 852.35    ms, avg 121.76 ms, 10.15 ms input 3.09      ms, avg 0.44 ms, output 4.84      ms, avg 0.69 ms 
+Batch_size 13, 1     times, infer 660.79    ms, avg 660.79 ms, 50.83 ms input 0.28      ms, avg 0.28 ms, output 0.60      ms, avg 0.60 ms 
+Batch_size 16, 1     times, infer 727.80    ms, avg 727.80 ms, 45.49 ms input 0.35      ms, avg 0.35 ms, output 0.55      ms, avg 0.55 ms 
+model name is feature_extractor 
+queue 23.72 s, infer 54.99 s, input 2.63  s, output 1.44  s 
+Batch_size 1 , 3752  times, infer 13789.44  ms, avg 3.68 ms, 3.68 ms input 570.04    ms, avg 0.15 ms, output 385.18    ms, avg 0.10 ms 
+Batch_size 2 , 260   times, infer 1780.06   ms, avg 6.85 ms, 3.42 ms input 70.05     ms, avg 0.27 ms, output 40.73     ms, avg 0.16 ms 
+Batch_size 3 , 305   times, infer 2951.96   ms, avg 9.68 ms, 3.23 ms input 124.72    ms, avg 0.41 ms, output 71.31     ms, avg 0.23 ms 
+Batch_size 4 , 244   times, infer 2945.23   ms, avg 12.07 ms, 3.02 ms input 134.74    ms, avg 0.55 ms, output 73.80     ms, avg 0.30 ms 
+Batch_size 5 , 101   times, infer 1472.61   ms, avg 14.58 ms, 2.92 ms input 71.15     ms, avg 0.70 ms, output 38.43     ms, avg 0.38 ms 
+Batch_size 6 , 43    times, infer 746.95    ms, avg 17.37 ms, 2.90 ms input 35.55     ms, avg 0.83 ms, output 18.41     ms, avg 0.43 ms 
+Batch_size 7 , 19    times, infer 362.44    ms, avg 19.08 ms, 2.73 ms input 19.59     ms, avg 1.03 ms, output 9.74      ms, avg 0.51 ms 
+Batch_size 8 , 5     times, infer 114.60    ms, avg 22.92 ms, 2.87 ms input 5.75      ms, avg 1.15 ms, output 3.17      ms, avg 0.63 ms 
+Batch_size 9 , 3     times, infer 63.59     ms, avg 21.20 ms, 2.36 ms input 4.17      ms, avg 1.39 ms, output 2.13      ms, avg 0.71 ms 
+Batch_size 10, 1     times, infer 28.75     ms, avg 28.75 ms, 2.87 ms input 1.12      ms, avg 1.12 ms, output 0.72      ms, avg 0.72 ms 
+Batch_size 11, 1     times, infer 19.00     ms, avg 19.00 ms, 1.73 ms input 2.23      ms, avg 2.23 ms, output 0.70      ms, avg 0.70 ms 
+Batch_size 13, 1     times, infer 20.74     ms, avg 20.74 ms, 1.60 ms input 2.09      ms, avg 2.09 ms, output 0.85      ms, avg 0.85 ms 
+Batch_size 16, 1     times, infer 22.95     ms, avg 22.95 ms, 1.43 ms input 9.51      ms, avg 9.51 ms, output 1.28      ms, avg 1.28 ms 
+model name is scoring 
+queue 534.14 s, infer 1116.56 s, input 46.99 s, output 6.79  s 
+Batch_size 1 , 704   times, infer 91951.10  ms, avg 130.61 ms, 130.61 ms input 775.04    ms, avg 1.10 ms, output 135.66    ms, avg 0.19 ms 
+Batch_size 2 , 210   times, infer 27404.23  ms, avg 130.50 ms, 65.25 ms input 471.36    ms, avg 2.24 ms, output 61.07     ms, avg 0.29 ms 
+Batch_size 3 , 137   times, infer 16895.43  ms, avg 123.32 ms, 41.11 ms input 286.70    ms, avg 2.09 ms, output 51.48     ms, avg 0.38 ms 
+Batch_size 4 , 49    times, infer 6572.75   ms, avg 134.14 ms, 33.53 ms input 159.14    ms, avg 3.25 ms, output 22.42     ms, avg 0.46 ms 
+Batch_size 5 , 25    times, infer 4114.44   ms, avg 164.58 ms, 32.92 ms input 108.17    ms, avg 4.33 ms, output 14.95     ms, avg 0.60 ms 
+Batch_size 6 , 39    times, infer 6093.90   ms, avg 156.25 ms, 26.04 ms input 156.85    ms, avg 4.02 ms, output 24.24     ms, avg 0.62 ms 
+Batch_size 7 , 31    times, infer 5235.52   ms, avg 168.89 ms, 24.13 ms input 146.28    ms, avg 4.72 ms, output 21.55     ms, avg 0.70 ms 
+Batch_size 8 , 29    times, infer 4629.00   ms, avg 159.62 ms, 19.95 ms input 166.62    ms, avg 5.75 ms, output 22.23     ms, avg 0.77 ms 
+Batch_size 9 , 20    times, infer 3450.58   ms, avg 172.53 ms, 19.17 ms input 128.25    ms, avg 6.41 ms, output 15.72     ms, avg 0.79 ms 
+Batch_size 10, 17    times, infer 2814.94   ms, avg 165.58 ms, 16.56 ms input 103.38    ms, avg 6.08 ms, output 16.42     ms, avg 0.97 ms 
+Batch_size 11, 31    times, infer 5507.99   ms, avg 177.68 ms, 16.15 ms input 201.17    ms, avg 6.49 ms, output 32.13     ms, avg 1.04 ms 
+Batch_size 12, 25    times, infer 4290.56   ms, avg 171.62 ms, 14.30 ms input 195.57    ms, avg 7.82 ms, output 24.38     ms, avg 0.98 ms 
+Batch_size 13, 23    times, infer 3893.52   ms, avg 169.28 ms, 13.02 ms input 166.72    ms, avg 7.25 ms, output 26.45     ms, avg 1.15 ms 
+Batch_size 14, 14    times, infer 2509.66   ms, avg 179.26 ms, 12.80 ms input 105.40    ms, avg 7.53 ms, output 15.79     ms, avg 1.13 ms 
+Batch_size 15, 33    times, infer 5545.84   ms, avg 168.06 ms, 11.20 ms input 273.86    ms, avg 8.30 ms, output 42.50     ms, avg 1.29 ms 
+Batch_size 16, 166   times, infer 26361.53  ms, avg 158.80 ms, 9.93 ms input 1590.02   ms, avg 9.58 ms, output 224.01    ms, avg 1.35 ms 

perf_log/model_repo_tlg_mbr/stats-20.json

@@ -0,0 +1 @@
+{"model_stats": [{"name": "attention_rescoring", "version": "1", "last_inference": "1680576254578", "inference_count": "7176", "execution_count": "7176", "inference_stats": {"success": {"count": "7176", "ns": "2128360908669"}, "fail": {}, "queue": {"count": "7176", "ns": "12006232"}, "compute_input": {"count": "7176", "ns": "51352685496"}, "compute_infer": {"count": "7176", "ns": "1425319750705"}, "compute_output": {"count": "7176", "ns": "12410765884"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "7176", "ns": "51352685496"}, "compute_infer": {"count": "7176", "ns": "1425319750705"}, "compute_output": {"count": "7176", "ns": "12410765884"}}]}, {"name": "decoder", "version": "1", "inference_stats": {"success": {}, "fail": {}, "queue": {}, "compute_input": {}, "compute_infer": {}, "compute_output": {}, "cache_hit": {}, "cache_miss": {}}}, {"name": "encoder", "version": "1", "last_inference": "1680576254545", "inference_count": "7176", "execution_count": "3955", "inference_stats": {"success": {"count": "7176", "ns": "339517381095"}, "fail": {}, "queue": {"count": "7176", "ns": "78935411720"}, "compute_input": {"count": "7176", "ns": "1732237168"}, "compute_infer": {"count": "7176", "ns": "253773831565"}, "compute_output": {"count": "7176", "ns": "4184221188"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "3029", "ns": "890406108"}, "compute_infer": {"count": "3029", "ns": "44498594833"}, "compute_output": {"count": "3029", "ns": "1012473444"}}, {"batch_size": "2", "compute_input": {"count": "217", "ns": "34802382"}, "compute_infer": {"count": "217", "ns": "5660016686"}, "compute_output": {"count": "217", "ns": "119711453"}}, {"batch_size": "3", "compute_input": {"count": "168", "ns": "25794251"}, "compute_infer": {"count": "168", "ns": "4610713340"}, "compute_output": {"count": "168", "ns": "86974249"}}, {"batch_size": "4", "compute_input": {"count": "173", "ns": "36043270"}, "compute_infer": {"count": "173", "ns": "5293880159"}, "compute_output": {"count": "173", "ns": "119676995"}}, {"batch_size": "5", "compute_input": {"count": "124", "ns": "16835171"}, "compute_infer": {"count": "124", "ns": "4076078370"}, "compute_output": {"count": "124", "ns": "74417250"}}, {"batch_size": "6", "compute_input": {"count": "78", "ns": "15362373"}, "compute_infer": {"count": "78", "ns": "3212684369"}, "compute_output": {"count": "78", "ns": "70230070"}}, {"batch_size": "7", "compute_input": {"count": "51", "ns": "9243550"}, "compute_infer": {"count": "51", "ns": "2474497079"}, "compute_output": {"count": "51", "ns": "38030148"}}, {"batch_size": "8", "compute_input": {"count": "43", "ns": "9874872"}, "compute_infer": {"count": "43", "ns": "2384899380"}, "compute_output": {"count": "43", "ns": "39272436"}}, {"batch_size": "9", "compute_input": {"count": "26", "ns": "6304712"}, "compute_infer": {"count": "26", "ns": "1916886476"}, "compute_output": {"count": "26", "ns": "22522122"}}, {"batch_size": "10", "compute_input": {"count": "26", "ns": "9850794"}, "compute_infer": {"count": "26", "ns": "2177513507"}, "compute_output": {"count": "26", "ns": "46081306"}}, {"batch_size": "11", "compute_input": {"count": "11", "ns": "2651438"}, "compute_infer": {"count": "11", "ns": "1581818786"}, "compute_output": {"count": "11", "ns": "7328349"}}, {"batch_size": "12", "compute_input": {"count": "7", "ns": "3085238"}, "compute_infer": {"count": "7", "ns": "852349841"}, "compute_output": {"count": "7", "ns": "4838612"}}, {"batch_size": "13", "compute_input": {"count": "1", "ns": "280538"}, "compute_infer": {"count": "1", "ns": "660787951"}, "compute_output": {"count": "1", "ns": "602312"}}, {"batch_size": "16", "compute_input": {"count": "1", "ns": "345658"}, "compute_infer": {"count": "1", "ns": "727800537"}, "compute_output": {"count": "1", "ns": "551220"}}]}, {"name": "feature_extractor", "version": "1", "last_inference": "1680576254533", "inference_count": "7176", "execution_count": "4736", "inference_stats": {"success": {"count": "7176", "ns": "83180443850"}, "fail": {}, "queue": {"count": "7176", "ns": "23716848367"}, "compute_input": {"count": "7176", "ns": "2628152441"}, "compute_infer": {"count": "7176", "ns": "54990675491"}, "compute_output": {"count": "7176", "ns": "1437482914"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "3752", "ns": "570043178"}, "compute_infer": {"count": "3752", "ns": "13789444481"}, "compute_output": {"count": "3752", "ns": "385179935"}}, {"batch_size": "2", "compute_input": {"count": "260", "ns": "70050000"}, "compute_infer": {"count": "260", "ns": "1780058751"}, "compute_output": {"count": "260", "ns": "40733094"}}, {"batch_size": "3", "compute_input": {"count": "305", "ns": "124722378"}, "compute_infer": {"count": "305", "ns": "2951955244"}, "compute_output": {"count": "305", "ns": "71314424"}}, {"batch_size": "4", "compute_input": {"count": "244", "ns": "134735791"}, "compute_infer": {"count": "244", "ns": "2945228891"}, "compute_output": {"count": "244", "ns": "73801425"}}, {"batch_size": "5", "compute_input": {"count": "101", "ns": "71153227"}, "compute_infer": {"count": "101", "ns": "1472613081"}, "compute_output": {"count": "101", "ns": "38426117"}}, {"batch_size": "6", "compute_input": {"count": "43", "ns": "35552023"}, "compute_infer": {"count": "43", "ns": "746945333"}, "compute_output": {"count": "43", "ns": "18413395"}}, {"batch_size": "7", "compute_input": {"count": "19", "ns": "19593889"}, "compute_infer": {"count": "19", "ns": "362438402"}, "compute_output": {"count": "19", "ns": "9744117"}}, {"batch_size": "8", "compute_input": {"count": "5", "ns": "5749901"}, "compute_infer": {"count": "5", "ns": "114604953"}, "compute_output": {"count": "5", "ns": "3172885"}}, {"batch_size": "9", "compute_input": {"count": "3", "ns": "4171064"}, "compute_infer": {"count": "3", "ns": "63593200"}, "compute_output": {"count": "3", "ns": "2127795"}}, {"batch_size": "10", "compute_input": {"count": "1", "ns": "1122984"}, "compute_infer": {"count": "1", "ns": "28747253"}, "compute_output": {"count": "1", "ns": "716439"}}, {"batch_size": "11", "compute_input": {"count": "1", "ns": "2234915"}, "compute_infer": {"count": "1", "ns": "19000910"}, "compute_output": {"count": "1", "ns": "700892"}}, {"batch_size": "13", "compute_input": {"count": "1", "ns": "2089644"}, "compute_infer": {"count": "1", "ns": "20744883"}, "compute_output": {"count": "1", "ns": "850720"}}, {"batch_size": "16", "compute_input": {"count": "1", "ns": "9509088"}, "compute_infer": {"count": "1", "ns": "22948847"}, "compute_output": {"count": "1", "ns": "1275078"}}]}, {"name": "scoring", "version": "1", "last_inference": "1680576254578", "inference_count": "7176", "execution_count": "1553", "inference_stats": {"success": {"count": "7176", "ns": "1705743180262"}, "fail": {}, "queue": {"count": "7176", "ns": "534141498121"}, "compute_input": {"count": "7176", "ns": "46992295887"}, "compute_infer": {"count": "7176", "ns": "1116555243649"}, "compute_output": {"count": "7176", "ns": "6789061782"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "704", "ns": "775039620"}, "compute_infer": {"count": "704", "ns": "91951103407"}, "compute_output": {"count": "704", "ns": "135658892"}}, {"batch_size": "2", "compute_input": {"count": "210", "ns": "471363321"}, "compute_infer": {"count": "210", "ns": "27404226303"}, "compute_output": {"count": "210", "ns": "61069313"}}, {"batch_size": "3", "compute_input": {"count": "137", "ns": "286701875"}, "compute_infer": {"count": "137", "ns": "16895433991"}, "compute_output": {"count": "137", "ns": "51484685"}}, {"batch_size": "4", "compute_input": {"count": "49", "ns": "159144945"}, "compute_infer": {"count": "49", "ns": "6572749679"}, "compute_output": {"count": "49", "ns": "22416217"}}, {"batch_size": "5", "compute_input": {"count": "25", "ns": "108172311"}, "compute_infer": {"count": "25", "ns": "4114442124"}, "compute_output": {"count": "25", "ns": "14950742"}}, {"batch_size": "6", "compute_input": {"count": "39", "ns": "156846168"}, "compute_infer": {"count": "39", "ns": "6093903286"}, "compute_output": {"count": "39", "ns": "24243240"}}, {"batch_size": "7", "compute_input": {"count": "31", "ns": "146282567"}, "compute_infer": {"count": "31", "ns": "5235518983"}, "compute_output": {"count": "31", "ns": "21554557"}}, {"batch_size": "8", "compute_input": {"count": "29", "ns": "166616055"}, "compute_infer": {"count": "29", "ns": "4629002092"}, "compute_output": {"count": "29", "ns": "22229924"}}, {"batch_size": "9", "compute_input": {"count": "20", "ns": "128249649"}, "compute_infer": {"count": "20", "ns": "3450578290"}, "compute_output": {"count": "20", "ns": "15717434"}}, {"batch_size": "10", "compute_input": {"count": "17", "ns": "103379887"}, "compute_infer": {"count": "17", "ns": "2814937003"}, "compute_output": {"count": "17", "ns": "16424759"}}, {"batch_size": "11", "compute_input": {"count": "31", "ns": "201174822"}, "compute_infer": {"count": "31", "ns": "5507988312"}, "compute_output": {"count": "31", "ns": "32128174"}}, {"batch_size": "12", "compute_input": {"count": "25", "ns": "195566566"}, "compute_infer": {"count": "25", "ns": "4290556037"}, "compute_output": {"count": "25", "ns": "24376227"}}, {"batch_size": "13", "compute_input": {"count": "23", "ns": "166722401"}, "compute_infer": {"count": "23", "ns": "3893515532"}, "compute_output": {"count": "23", "ns": "26448452"}}, {"batch_size": "14", "compute_input": {"count": "14", "ns": "105400438"}, "compute_infer": {"count": "14", "ns": "2509660750"}, "compute_output": {"count": "14", "ns": "15789160"}}, {"batch_size": "15", "compute_input": {"count": "33", "ns": "273861018"}, "compute_infer": {"count": "33", "ns": "5545835042"}, "compute_output": {"count": "33", "ns": "42499134"}}, {"batch_size": "16", "compute_input": {"count": "166", "ns": "1590019943"}, "compute_infer": {"count": "166", "ns": "26361531902"}, "compute_output": {"count": "166", "ns": "224013540"}}]}]}

perf_log/model_repo_tlg_mbr/stats-40.json

@@ -0,0 +1 @@
+{"model_stats": [{"name": "attention_rescoring", "version": "1", "last_inference": "1680576341426", "inference_count": "14352", "execution_count": "14352", "inference_stats": {"success": {"count": "14352", "ns": "5066779555539"}, "fail": {}, "queue": {"count": "14352", "ns": "23468470"}, "compute_input": {"count": "14352", "ns": "119265712814"}, "compute_infer": {"count": "14352", "ns": "3171715765822"}, "compute_output": {"count": "14352", "ns": "26878725951"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "14352", "ns": "119265712814"}, "compute_infer": {"count": "14352", "ns": "3171715765822"}, "compute_output": {"count": "14352", "ns": "26878725951"}}]}, {"name": "decoder", "version": "1", "inference_stats": {"success": {}, "fail": {}, "queue": {}, "compute_input": {}, "compute_infer": {}, "compute_output": {}, "cache_hit": {}, "cache_miss": {}}}, {"name": "encoder", "version": "1", "last_inference": "1680576341383", "inference_count": "14352", "execution_count": "7116", "inference_stats": {"success": {"count": "14352", "ns": "579163122112"}, "fail": {}, "queue": {"count": "14352", "ns": "122068932806"}, "compute_input": {"count": "14352", "ns": "3164558289"}, "compute_infer": {"count": "14352", "ns": "443449044455"}, "compute_output": {"count": "14352", "ns": "8553018649"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "5136", "ns": "1509448141"}, "compute_infer": {"count": "5136", "ns": "74985494604"}, "compute_output": {"count": "5136", "ns": "1744658488"}}, {"batch_size": "2", "compute_input": {"count": "409", "ns": "63610989"}, "compute_infer": {"count": "409", "ns": "9267039365"}, "compute_output": {"count": "409", "ns": "225460056"}}, {"batch_size": "3", "compute_input": {"count": "320", "ns": "40886865"}, "compute_infer": {"count": "320", "ns": "7777206889"}, "compute_output": {"count": "320", "ns": "167759446"}}, {"batch_size": "4", "compute_input": {"count": "386", "ns": "58666268"}, "compute_infer": {"count": "386", "ns": "10454310434"}, "compute_output": {"count": "386", "ns": "254375902"}}, {"batch_size": "5", "compute_input": {"count": "291", "ns": "41100727"}, "compute_infer": {"count": "291", "ns": "8166778941"}, "compute_output": {"count": "291", "ns": "197039138"}}, {"batch_size": "6", "compute_input": {"count": "171", "ns": "30981485"}, "compute_infer": {"count": "171", "ns": "5557120635"}, "compute_output": {"count": "171", "ns": "150968871"}}, {"batch_size": "7", "compute_input": {"count": "145", "ns": "27603000"}, "compute_infer": {"count": "145", "ns": "4996632566"}, "compute_output": {"count": "145", "ns": "95224475"}}, {"batch_size": "8", "compute_input": {"count": "92", "ns": "18682701"}, "compute_infer": {"count": "92", "ns": "3791870642"}, "compute_output": {"count": "92", "ns": "74775623"}}, {"batch_size": "9", "compute_input": {"count": "77", "ns": "16496676"}, "compute_infer": {"count": "77", "ns": "3510418039"}, "compute_output": {"count": "77", "ns": "75353088"}}, {"batch_size": "10", "compute_input": {"count": "46", "ns": "14142498"}, "compute_infer": {"count": "46", "ns": "2806280698"}, "compute_output": {"count": "46", "ns": "58247334"}}, {"batch_size": "11", "compute_input": {"count": "24", "ns": "5725277"}, "compute_infer": {"count": "24", "ns": "2059091119"}, "compute_output": {"count": "24", "ns": "16368951"}}, {"batch_size": "12", "compute_input": {"count": "12", "ns": "4365533"}, "compute_infer": {"count": "12", "ns": "1601424935"}, "compute_output": {"count": "12", "ns": "8113109"}}, {"batch_size": "13", "compute_input": {"count": "3", "ns": "838477"}, "compute_infer": {"count": "3", "ns": "737707805"}, "compute_output": {"count": "3", "ns": "7642001"}}, {"batch_size": "14", "compute_input": {"count": "1", "ns": "281579"}, "compute_infer": {"count": "1", "ns": "765760602"}, "compute_output": {"count": "1", "ns": "737730"}}, {"batch_size": "16", "compute_input": {"count": "3", "ns": "1024096"}, "compute_infer": {"count": "3", "ns": "1466129772"}, "compute_output": {"count": "3", "ns": "2070734"}}]}, {"name": "feature_extractor", "version": "1", "last_inference": "1680576341369", "inference_count": "14352", "execution_count": "8673", "inference_stats": {"success": {"count": "14352", "ns": "168556670794"}, "fail": {}, "queue": {"count": "14352", "ns": "36289769937"}, "compute_input": {"count": "14352", "ns": "5648929380"}, "compute_infer": {"count": "14352", "ns": "122623942773"}, "compute_output": {"count": "14352", "ns": "3099048474"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "6487", "ns": "974917255"}, "compute_infer": {"count": "6487", "ns": "21613876262"}, "compute_output": {"count": "6487", "ns": "660887433"}}, {"batch_size": "2", "compute_input": {"count": "531", "ns": "139359524"}, "compute_infer": {"count": "531", "ns": "3604011780"}, "compute_output": {"count": "531", "ns": "82293019"}}, {"batch_size": "3", "compute_input": {"count": "684", "ns": "271500718"}, "compute_infer": {"count": "684", "ns": "6730742526"}, "compute_output": {"count": "684", "ns": "155903588"}}, {"batch_size": "4", "compute_input": {"count": "510", "ns": "276841095"}, "compute_infer": {"count": "510", "ns": "6315142538"}, "compute_output": {"count": "510", "ns": "151092792"}}, {"batch_size": "5", "compute_input": {"count": "239", "ns": "162743150"}, "compute_infer": {"count": "239", "ns": "3620154552"}, "compute_output": {"count": "239", "ns": "86528309"}}, {"batch_size": "6", "compute_input": {"count": "138", "ns": "110523715"}, "compute_infer": {"count": "138", "ns": "2380527459"}, "compute_output": {"count": "138", "ns": "57853968"}}, {"batch_size": "7", "compute_input": {"count": "43", "ns": "41892719"}, "compute_infer": {"count": "43", "ns": "871920660"}, "compute_output": {"count": "43", "ns": "20924260"}}, {"batch_size": "8", "compute_input": {"count": "20", "ns": "21974822"}, "compute_infer": {"count": "20", "ns": "443193956"}, "compute_output": {"count": "20", "ns": "11472922"}}, {"batch_size": "9", "compute_input": {"count": "10", "ns": "12947161"}, "compute_infer": {"count": "10", "ns": "221009400"}, "compute_output": {"count": "10", "ns": "6276118"}}, {"batch_size": "10", "compute_input": {"count": "3", "ns": "3841387"}, "compute_infer": {"count": "3", "ns": "84867932"}, "compute_output": {"count": "3", "ns": "1926825"}}, {"batch_size": "11", "compute_input": {"count": "2", "ns": "4042170"}, "compute_infer": {"count": "2", "ns": "49528852"}, "compute_output": {"count": "2", "ns": "1554103"}}, {"batch_size": "13", "compute_input": {"count": "3", "ns": "5458427"}, "compute_infer": {"count": "3", "ns": "88265021"}, "compute_output": {"count": "3", "ns": "2596315"}}, {"batch_size": "14", "compute_input": {"count": "1", "ns": "4681151"}, "compute_infer": {"count": "1", "ns": "53355699"}, "compute_output": {"count": "1", "ns": "1155985"}}, {"batch_size": "16", "compute_input": {"count": "2", "ns": "11976440"}, "compute_infer": {"count": "2", "ns": "64956568"}, "compute_output": {"count": "2", "ns": "2543137"}}]}, {"name": "scoring", "version": "1", "last_inference": "1680576341426", "inference_count": "14352", "execution_count": "2332", "inference_stats": {"success": {"count": "14352", "ns": "4318872110108"}, "fail": {}, "queue": {"count": "14352", "ns": "1584772080218"}, "compute_input": {"count": "14352", "ns": "110452225145"}, "compute_infer": {"count": "14352", "ns": "2605642778594"}, "compute_output": {"count": "14352", "ns": "15226658828"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "890", "ns": "1035908982"}, "compute_infer": {"count": "890", "ns": "126638388515"}, "compute_output": {"count": "890", "ns": "172416139"}}, {"batch_size": "2", "compute_input": {"count": "245", "ns": "558949309"}, "compute_infer": {"count": "245", "ns": "33908119715"}, "compute_output": {"count": "245", "ns": "70897356"}}, {"batch_size": "3", "compute_input": {"count": "151", "ns": "325104082"}, "compute_infer": {"count": "151", "ns": "19249941406"}, "compute_output": {"count": "151", "ns": "56690988"}}, {"batch_size": "4", "compute_input": {"count": "73", "ns": "252583473"}, "compute_infer": {"count": "73", "ns": "10661554437"}, "compute_output": {"count": "73", "ns": "33066501"}}, {"batch_size": "5", "compute_input": {"count": "36", "ns": "168733996"}, "compute_infer": {"count": "36", "ns": "6337444864"}, "compute_output": {"count": "36", "ns": "21600155"}}, {"batch_size": "6", "compute_input": {"count": "50", "ns": "209454579"}, "compute_infer": {"count": "50", "ns": "8250033482"}, "compute_output": {"count": "50", "ns": "30718300"}}, {"batch_size": "7", "compute_input": {"count": "62", "ns": "342010468"}, "compute_infer": {"count": "62", "ns": "11739075083"}, "compute_output": {"count": "62", "ns": "43514623"}}, {"batch_size": "8", "compute_input": {"count": "99", "ns": "479323334"}, "compute_infer": {"count": "99", "ns": "19253314948"}, "compute_output": {"count": "99", "ns": "73184438"}}, {"batch_size": "9", "compute_input": {"count": "41", "ns": "243459619"}, "compute_infer": {"count": "41", "ns": "8083442676"}, "compute_output": {"count": "41", "ns": "34483937"}}, {"batch_size": "10", "compute_input": {"count": "34", "ns": "208845980"}, "compute_infer": {"count": "34", "ns": "6592760985"}, "compute_output": {"count": "34", "ns": "32020599"}}, {"batch_size": "11", "compute_input": {"count": "45", "ns": "296453012"}, "compute_infer": {"count": "45", "ns": "8793002105"}, "compute_output": {"count": "45", "ns": "46598728"}}, {"batch_size": "12", "compute_input": {"count": "39", "ns": "301278383"}, "compute_infer": {"count": "39", "ns": "7556223763"}, "compute_output": {"count": "39", "ns": "38926901"}}, {"batch_size": "13", "compute_input": {"count": "38", "ns": "353379792"}, "compute_infer": {"count": "38", "ns": "7061249245"}, "compute_output": {"count": "38", "ns": "45317801"}}, {"batch_size": "14", "compute_input": {"count": "29", "ns": "293631866"}, "compute_infer": {"count": "29", "ns": "5625196586"}, "compute_output": {"count": "29", "ns": "34006480"}}, {"batch_size": "15", "compute_input": {"count": "51", "ns": "481622890"}, "compute_infer": {"count": "51", "ns": "9223361272"}, "compute_output": {"count": "51", "ns": "64431583"}}, {"batch_size": "16", "compute_input": {"count": "449", "ns": "4431166696"}, "compute_infer": {"count": "449", "ns": "84901807787"}, "compute_output": {"count": "449", "ns": "611609653"}}]}]}

perf_log/model_repo_tlg_mbr/stats-60.json

@@ -0,0 +1 @@
+{"model_stats": [{"name": "attention_rescoring", "version": "1", "last_inference": "1680576418815", "inference_count": "21528", "execution_count": "21528", "inference_stats": {"success": {"count": "21528", "ns": "9004116460631"}, "fail": {}, "queue": {"count": "21528", "ns": "34298126"}, "compute_input": {"count": "21528", "ns": "202743228347"}, "compute_infer": {"count": "21528", "ns": "5193537038227"}, "compute_output": {"count": "21528", "ns": "42562421003"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "21528", "ns": "202743228347"}, "compute_infer": {"count": "21528", "ns": "5193537038227"}, "compute_output": {"count": "21528", "ns": "42562421003"}}]}, {"name": "decoder", "version": "1", "inference_stats": {"success": {}, "fail": {}, "queue": {}, "compute_input": {}, "compute_infer": {}, "compute_output": {}, "cache_hit": {}, "cache_miss": {}}}, {"name": "encoder", "version": "1", "last_inference": "1680576418649", "inference_count": "21528", "execution_count": "9855", "inference_stats": {"success": {"count": "21528", "ns": "841304012374"}, "fail": {}, "queue": {"count": "21528", "ns": "175097723291"}, "compute_input": {"count": "21528", "ns": "4553671930"}, "compute_infer": {"count": "21528", "ns": "645174021998"}, "compute_output": {"count": "21528", "ns": "13402007690"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "6814", "ns": "1972479657"}, "compute_infer": {"count": "6814", "ns": "101671685002"}, "compute_output": {"count": "6814", "ns": "2380442586"}}, {"batch_size": "2", "compute_input": {"count": "583", "ns": "84431708"}, "compute_infer": {"count": "583", "ns": "12669562281"}, "compute_output": {"count": "583", "ns": "327078883"}}, {"batch_size": "3", "compute_input": {"count": "477", "ns": "55664058"}, "compute_infer": {"count": "477", "ns": "11381683569"}, "compute_output": {"count": "477", "ns": "252334933"}}, {"batch_size": "4", "compute_input": {"count": "581", "ns": "82996797"}, "compute_infer": {"count": "581", "ns": "15469767000"}, "compute_output": {"count": "581", "ns": "366992129"}}, {"batch_size": "5", "compute_input": {"count": "438", "ns": "62862916"}, "compute_infer": {"count": "438", "ns": "12020944678"}, "compute_output": {"count": "438", "ns": "282197509"}}, {"batch_size": "6", "compute_input": {"count": "274", "ns": "45604951"}, "compute_infer": {"count": "274", "ns": "8394433048"}, "compute_output": {"count": "274", "ns": "229453108"}}, {"batch_size": "7", "compute_input": {"count": "222", "ns": "41743345"}, "compute_infer": {"count": "222", "ns": "7193680861"}, "compute_output": {"count": "222", "ns": "143391606"}}, {"batch_size": "8", "compute_input": {"count": "163", "ns": "31608781"}, "compute_infer": {"count": "163", "ns": "5896554277"}, "compute_output": {"count": "163", "ns": "124581447"}}, {"batch_size": "9", "compute_input": {"count": "124", "ns": "25805379"}, "compute_infer": {"count": "124", "ns": "4993507504"}, "compute_output": {"count": "124", "ns": "107812075"}}, {"batch_size": "10", "compute_input": {"count": "89", "ns": "23783953"}, "compute_infer": {"count": "89", "ns": "4188000146"}, "compute_output": {"count": "89", "ns": "104528840"}}, {"batch_size": "11", "compute_input": {"count": "41", "ns": "9957621"}, "compute_infer": {"count": "41", "ns": "2668594309"}, "compute_output": {"count": "41", "ns": "29925107"}}, {"batch_size": "12", "compute_input": {"count": "24", "ns": "7522465"}, "compute_infer": {"count": "24", "ns": "2045676742"}, "compute_output": {"count": "24", "ns": "20496600"}}, {"batch_size": "13", "compute_input": {"count": "12", "ns": "3376428"}, "compute_infer": {"count": "12", "ns": "1830286925"}, "compute_output": {"count": "12", "ns": "14857322"}}, {"batch_size": "14", "compute_input": {"count": "4", "ns": "1246349"}, "compute_infer": {"count": "4", "ns": "1527438167"}, "compute_output": {"count": "4", "ns": "2747313"}}, {"batch_size": "16", "compute_input": {"count": "9", "ns": "3068609"}, "compute_infer": {"count": "9", "ns": "1764842509"}, "compute_output": {"count": "9", "ns": "33249687"}}]}, {"name": "feature_extractor", "version": "1", "last_inference": "1680576418633", "inference_count": "21528", "execution_count": "12295", "inference_stats": {"success": {"count": "21528", "ns": "262076725330"}, "fail": {}, "queue": {"count": "21528", "ns": "51227392018"}, "compute_input": {"count": "21528", "ns": "8863457691"}, "compute_infer": {"count": "21528", "ns": "195684233739"}, "compute_output": {"count": "21528", "ns": "4891997029"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "8833", "ns": "1310769727"}, "compute_infer": {"count": "8833", "ns": "28146243759"}, "compute_output": {"count": "8833", "ns": "892379816"}}, {"batch_size": "2", "compute_input": {"count": "805", "ns": "208886865"}, "compute_infer": {"count": "805", "ns": "5222805193"}, "compute_output": {"count": "805", "ns": "126882737"}}, {"batch_size": "3", "compute_input": {"count": "1091", "ns": "425479276"}, "compute_infer": {"count": "1091", "ns": "10657020106"}, "compute_output": {"count": "1091", "ns": "246226015"}}, {"batch_size": "4", "compute_input": {"count": "777", "ns": "419190770"}, "compute_infer": {"count": "777", "ns": "9719174668"}, "compute_output": {"count": "777", "ns": "229287291"}}, {"batch_size": "5", "compute_input": {"count": "397", "ns": "265793875"}, "compute_infer": {"count": "397", "ns": "6065687172"}, "compute_output": {"count": "397", "ns": "140858964"}}, {"batch_size": "6", "compute_input": {"count": "223", "ns": "176241044"}, "compute_infer": {"count": "223", "ns": "3886561085"}, "compute_output": {"count": "223", "ns": "92247816"}}, {"batch_size": "7", "compute_input": {"count": "85", "ns": "82033164"}, "compute_infer": {"count": "85", "ns": "1708841135"}, "compute_output": {"count": "85", "ns": "40556908"}}, {"batch_size": "8", "compute_input": {"count": "35", "ns": "37804642"}, "compute_infer": {"count": "35", "ns": "817862884"}, "compute_output": {"count": "35", "ns": "19698813"}}, {"batch_size": "9", "compute_input": {"count": "28", "ns": "34874936"}, "compute_infer": {"count": "28", "ns": "665151426"}, "compute_output": {"count": "28", "ns": "18612711"}}, {"batch_size": "10", "compute_input": {"count": "7", "ns": "9307280"}, "compute_infer": {"count": "7", "ns": "190327077"}, "compute_output": {"count": "7", "ns": "4386962"}}, {"batch_size": "11", "compute_input": {"count": "5", "ns": "8589760"}, "compute_infer": {"count": "5", "ns": "134743600"}, "compute_output": {"count": "5", "ns": "3959244"}}, {"batch_size": "13", "compute_input": {"count": "3", "ns": "5458427"}, "compute_infer": {"count": "3", "ns": "88265021"}, "compute_output": {"count": "3", "ns": "2596315"}}, {"batch_size": "14", "compute_input": {"count": "2", "ns": "6765507"}, "compute_infer": {"count": "2", "ns": "83819088"}, "compute_output": {"count": "2", "ns": "2206636"}}, {"batch_size": "15", "compute_input": {"count": "2", "ns": "3992942"}, "compute_infer": {"count": "2", "ns": "90666428"}, "compute_output": {"count": "2", "ns": "2032204"}}, {"batch_size": "16", "compute_input": {"count": "2", "ns": "11976440"}, "compute_infer": {"count": "2", "ns": "64956568"}, "compute_output": {"count": "2", "ns": "2543137"}}]}, {"name": "scoring", "version": "1", "last_inference": "1680576418815", "inference_count": "21528", "execution_count": "2916", "inference_stats": {"success": {"count": "21528", "ns": "7899866890865"}, "fail": {}, "queue": {"count": "21528", "ns": "3329172832617"}, "compute_input": {"count": "21528", "ns": "189326098726"}, "compute_infer": {"count": "21528", "ns": "4352678782490"}, "compute_output": {"count": "21528", "ns": "24268416284"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "974", "ns": "1138712083"}, "compute_infer": {"count": "974", "ns": "145216197187"}, "compute_output": {"count": "974", "ns": "188709326"}}, {"batch_size": "2", "compute_input": {"count": "257", "ns": "579001454"}, "compute_infer": {"count": "257", "ns": "35724001338"}, "compute_output": {"count": "257", "ns": "74785724"}}, {"batch_size": "3", "compute_input": {"count": "157", "ns": "338999397"}, "compute_infer": {"count": "157", "ns": "20514579160"}, "compute_output": {"count": "157", "ns": "59104124"}}, {"batch_size": "4", "compute_input": {"count": "83", "ns": "284651055"}, "compute_infer": {"count": "83", "ns": "12628487741"}, "compute_output": {"count": "83", "ns": "37850169"}}, {"batch_size": "5", "compute_input": {"count": "39", "ns": "177688537"}, "compute_infer": {"count": "39", "ns": "7031722287"}, "compute_output": {"count": "39", "ns": "23283725"}}, {"batch_size": "6", "compute_input": {"count": "54", "ns": "222668376"}, "compute_infer": {"count": "54", "ns": "9209066481"}, "compute_output": {"count": "54", "ns": "33260098"}}, {"batch_size": "7", "compute_input": {"count": "67", "ns": "370812077"}, "compute_infer": {"count": "67", "ns": "12849594065"}, "compute_output": {"count": "67", "ns": "46740322"}}, {"batch_size": "8", "compute_input": {"count": "103", "ns": "498784355"}, "compute_infer": {"count": "103", "ns": "20051459767"}, "compute_output": {"count": "103", "ns": "75709421"}}, {"batch_size": "9", "compute_input": {"count": "46", "ns": "268524476"}, "compute_infer": {"count": "46", "ns": "9203850921"}, "compute_output": {"count": "46", "ns": "38408948"}}, {"batch_size": "10", "compute_input": {"count": "37", "ns": "226112111"}, "compute_infer": {"count": "37", "ns": "7218680341"}, "compute_output": {"count": "37", "ns": "34823765"}}, {"batch_size": "11", "compute_input": {"count": "48", "ns": "314741556"}, "compute_infer": {"count": "48", "ns": "9353884861"}, "compute_output": {"count": "48", "ns": "49394657"}}, {"batch_size": "12", "compute_input": {"count": "103", "ns": "1002841067"}, "compute_infer": {"count": "103", "ns": "24511520901"}, "compute_output": {"count": "103", "ns": "107471617"}}, {"batch_size": "13", "compute_input": {"count": "47", "ns": "457708194"}, "compute_infer": {"count": "47", "ns": "9316252531"}, "compute_output": {"count": "47", "ns": "55520032"}}, {"batch_size": "14", "compute_input": {"count": "42", "ns": "424335339"}, "compute_infer": {"count": "42", "ns": "8917329695"}, "compute_output": {"count": "42", "ns": "48615926"}}, {"batch_size": "15", "compute_input": {"count": "66", "ns": "677885002"}, "compute_infer": {"count": "66", "ns": "12719312512"}, "compute_output": {"count": "66", "ns": "83084198"}}, {"batch_size": "16", "compute_input": {"count": "793", "ns": "8364381337"}, "compute_infer": {"count": "793", "ns": "168399313436"}, "compute_output": {"count": "793", "ns": "1073563779"}}]}]}

perf_log/model_repo_tlg_mbr/stats-80.json

@@ -0,0 +1 @@
+{"model_stats": [{"name": "attention_rescoring", "version": "1", "last_inference": "1680576494233", "inference_count": "28704", "execution_count": "28704", "inference_stats": {"success": {"count": "28704", "ns": "14180585082898"}, "fail": {}, "queue": {"count": "28704", "ns": "45081523"}, "compute_input": {"count": "28704", "ns": "294684752251"}, "compute_infer": {"count": "28704", "ns": "7244434416388"}, "compute_output": {"count": "28704", "ns": "58436125750"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "28704", "ns": "294684752251"}, "compute_infer": {"count": "28704", "ns": "7244434416388"}, "compute_output": {"count": "28704", "ns": "58436125750"}}]}, {"name": "decoder", "version": "1", "inference_stats": {"success": {}, "fail": {}, "queue": {}, "compute_input": {}, "compute_infer": {}, "compute_output": {}, "cache_hit": {}, "cache_miss": {}}}, {"name": "encoder", "version": "1", "last_inference": "1680576494109", "inference_count": "28704", "execution_count": "12457", "inference_stats": {"success": {"count": "28704", "ns": "1117044317958"}, "fail": {}, "queue": {"count": "28704", "ns": "230383389484"}, "compute_input": {"count": "28704", "ns": "6060123308"}, "compute_infer": {"count": "28704", "ns": "858107461409"}, "compute_output": {"count": "28704", "ns": "18260829235"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "8356", "ns": "2408756152"}, "compute_infer": {"count": "8356", "ns": "127484996004"}, "compute_output": {"count": "8356", "ns": "2936178225"}}, {"batch_size": "2", "compute_input": {"count": "758", "ns": "108775153"}, "compute_infer": {"count": "758", "ns": "16292879101"}, "compute_output": {"count": "758", "ns": "392053605"}}, {"batch_size": "3", "compute_input": {"count": "593", "ns": "68269226"}, "compute_infer": {"count": "593", "ns": "14058167737"}, "compute_output": {"count": "593", "ns": "325965581"}}, {"batch_size": "4", "compute_input": {"count": "798", "ns": "110967000"}, "compute_infer": {"count": "798", "ns": "21129237568"}, "compute_output": {"count": "798", "ns": "500795999"}}, {"batch_size": "5", "compute_input": {"count": "591", "ns": "84687929"}, "compute_infer": {"count": "591", "ns": "15986607023"}, "compute_output": {"count": "591", "ns": "378831771"}}, {"batch_size": "6", "compute_input": {"count": "390", "ns": "64435808"}, "compute_infer": {"count": "390", "ns": "11627944812"}, "compute_output": {"count": "390", "ns": "332610617"}}, {"batch_size": "7", "compute_input": {"count": "304", "ns": "55069827"}, "compute_infer": {"count": "304", "ns": "9538960368"}, "compute_output": {"count": "304", "ns": "199563079"}}, {"batch_size": "8", "compute_input": {"count": "213", "ns": "43270523"}, "compute_infer": {"count": "213", "ns": "7485667328"}, "compute_output": {"count": "213", "ns": "174319326"}}, {"batch_size": "9", "compute_input": {"count": "180", "ns": "37311547"}, "compute_infer": {"count": "180", "ns": "6754573005"}, "compute_output": {"count": "180", "ns": "164051202"}}, {"batch_size": "10", "compute_input": {"count": "111", "ns": "28833646"}, "compute_infer": {"count": "111", "ns": "4931187403"}, "compute_output": {"count": "111", "ns": "120048316"}}, {"batch_size": "11", "compute_input": {"count": "67", "ns": "17413953"}, "compute_infer": {"count": "67", "ns": "3610850139"}, "compute_output": {"count": "67", "ns": "55607805"}}, {"batch_size": "12", "compute_input": {"count": "44", "ns": "13088058"}, "compute_infer": {"count": "44", "ns": "2748374286"}, "compute_output": {"count": "44", "ns": "37942636"}}, {"batch_size": "13", "compute_input": {"count": "24", "ns": "8442760"}, "compute_infer": {"count": "24", "ns": "2298243284"}, "compute_output": {"count": "24", "ns": "24769982"}}, {"batch_size": "14", "compute_input": {"count": "9", "ns": "2837508"}, "compute_infer": {"count": "9", "ns": "1710482961"}, "compute_output": {"count": "9", "ns": "7220885"}}, {"batch_size": "15", "compute_input": {"count": "3", "ns": "1022792"}, "compute_infer": {"count": "3", "ns": "1620917761"}, "compute_output": {"count": "3", "ns": "1950222"}}, {"batch_size": "16", "compute_input": {"count": "16", "ns": "6616267"}, "compute_infer": {"count": "16", "ns": "2127870246"}, "compute_output": {"count": "16", "ns": "42615462"}}]}, {"name": "feature_extractor", "version": "1", "last_inference": "1680576494098", "inference_count": "28704", "execution_count": "15851", "inference_stats": {"success": {"count": "28704", "ns": "358798870393"}, "fail": {}, "queue": {"count": "28704", "ns": "66980659981"}, "compute_input": {"count": "28704", "ns": "12277803476"}, "compute_infer": {"count": "28704", "ns": "270818788590"}, "compute_output": {"count": "28704", "ns": "6771973482"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "11131", "ns": "1639688644"}, "compute_infer": {"count": "11131", "ns": "34478065612"}, "compute_output": {"count": "11131", "ns": "1121542246"}}, {"batch_size": "2", "compute_input": {"count": "1076", "ns": "277910508"}, "compute_infer": {"count": "1076", "ns": "6814474302"}, "compute_output": {"count": "1076", "ns": "169159725"}}, {"batch_size": "3", "compute_input": {"count": "1451", "ns": "566287030"}, "compute_infer": {"count": "1451", "ns": "13918596568"}, "compute_output": {"count": "1451", "ns": "328345025"}}, {"batch_size": "4", "compute_input": {"count": "1058", "ns": "566804445"}, "compute_infer": {"count": "1058", "ns": "13170358027"}, "compute_output": {"count": "1058", "ns": "313289391"}}, {"batch_size": "5", "compute_input": {"count": "559", "ns": "370782444"}, "compute_infer": {"count": "559", "ns": "8506282439"}, "compute_output": {"count": "559", "ns": "196672328"}}, {"batch_size": "6", "compute_input": {"count": "317", "ns": "250442148"}, "compute_infer": {"count": "317", "ns": "5530947694"}, "compute_output": {"count": "317", "ns": "131252628"}}, {"batch_size": "7", "compute_input": {"count": "134", "ns": "126881597"}, "compute_infer": {"count": "134", "ns": "2721386251"}, "compute_output": {"count": "134", "ns": "63998060"}}, {"batch_size": "8", "compute_input": {"count": "45", "ns": "49296632"}, "compute_infer": {"count": "45", "ns": "1058221825"}, "compute_output": {"count": "45", "ns": "25434433"}}, {"batch_size": "9", "compute_input": {"count": "45", "ns": "56064358"}, "compute_infer": {"count": "45", "ns": "1067528589"}, "compute_output": {"count": "45", "ns": "29214310"}}, {"batch_size": "10", "compute_input": {"count": "11", "ns": "14789462"}, "compute_infer": {"count": "11", "ns": "303824601"}, "compute_output": {"count": "11", "ns": "7294564"}}, {"batch_size": "11", "compute_input": {"count": "7", "ns": "12093622"}, "compute_infer": {"count": "7", "ns": "222205910"}, "compute_output": {"count": "7", "ns": "5453110"}}, {"batch_size": "12", "compute_input": {"count": "2", "ns": "3691459"}, "compute_infer": {"count": "2", "ns": "70138166"}, "compute_output": {"count": "2", "ns": "1669811"}}, {"batch_size": "13", "compute_input": {"count": "3", "ns": "5458427"}, "compute_infer": {"count": "3", "ns": "88265021"}, "compute_output": {"count": "3", "ns": "2596315"}}, {"batch_size": "14", "compute_input": {"count": "2", "ns": "6765507"}, "compute_infer": {"count": "2", "ns": "83819088"}, "compute_output": {"count": "2", "ns": "2206636"}}, {"batch_size": "15", "compute_input": {"count": "2", "ns": "3992942"}, "compute_infer": {"count": "2", "ns": "90666428"}, "compute_output": {"count": "2", "ns": "2032204"}}, {"batch_size": "16", "compute_input": {"count": "8", "ns": "26358352"}, "compute_infer": {"count": "8", "ns": "339321288"}, "compute_output": {"count": "8", "ns": "8784714"}}]}, {"name": "scoring", "version": "1", "last_inference": "1680576494233", "inference_count": "28704", "execution_count": "3474", "inference_stats": {"success": {"count": "28704", "ns": "12702435539191"}, "fail": {}, "queue": {"count": "28704", "ns": "6271088640769"}, "compute_input": {"count": "28704", "ns": "276346825467"}, "compute_infer": {"count": "28704", "ns": "6115508166389"}, "compute_output": {"count": "28704", "ns": "33403323033"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "1053", "ns": "1223951952"}, "compute_infer": {"count": "1053", "ns": "162917923529"}, "compute_output": {"count": "1053", "ns": "204180646"}}, {"batch_size": "2", "compute_input": {"count": "268", "ns": "594165327"}, "compute_infer": {"count": "268", "ns": "37830017783"}, "compute_output": {"count": "268", "ns": "77732946"}}, {"batch_size": "3", "compute_input": {"count": "165", "ns": "350846464"}, "compute_infer": {"count": "165", "ns": "22051656719"}, "compute_output": {"count": "165", "ns": "61782956"}}, {"batch_size": "4", "compute_input": {"count": "90", "ns": "304892802"}, "compute_infer": {"count": "90", "ns": "14285148352"}, "compute_output": {"count": "90", "ns": "40955022"}}, {"batch_size": "5", "compute_input": {"count": "46", "ns": "210965932"}, "compute_infer": {"count": "46", "ns": "8410972623"}, "compute_output": {"count": "46", "ns": "26265756"}}, {"batch_size": "6", "compute_input": {"count": "58", "ns": "237110254"}, "compute_infer": {"count": "58", "ns": "10124259120"}, "compute_output": {"count": "58", "ns": "35930386"}}, {"batch_size": "7", "compute_input": {"count": "69", "ns": "383580407"}, "compute_infer": {"count": "69", "ns": "13298499991"}, "compute_output": {"count": "69", "ns": "48177012"}}, {"batch_size": "8", "compute_input": {"count": "107", "ns": "521730795"}, "compute_infer": {"count": "107", "ns": "20964820711"}, "compute_output": {"count": "107", "ns": "78696086"}}, {"batch_size": "9", "compute_input": {"count": "46", "ns": "268524476"}, "compute_infer": {"count": "46", "ns": "9203850921"}, "compute_output": {"count": "46", "ns": "38408948"}}, {"batch_size": "10", "compute_input": {"count": "39", "ns": "249283425"}, "compute_infer": {"count": "39", "ns": "7728109932"}, "compute_output": {"count": "39", "ns": "36800133"}}, {"batch_size": "11", "compute_input": {"count": "51", "ns": "340092423"}, "compute_infer": {"count": "51", "ns": "10085139639"}, "compute_output": {"count": "51", "ns": "51854313"}}, {"batch_size": "12", "compute_input": {"count": "104", "ns": "1007251092"}, "compute_infer": {"count": "104", "ns": "24873559542"}, "compute_output": {"count": "104", "ns": "108542173"}}, {"batch_size": "13", "compute_input": {"count": "52", "ns": "517311197"}, "compute_infer": {"count": "52", "ns": "10408290355"}, "compute_output": {"count": "52", "ns": "62028886"}}, {"batch_size": "14", "compute_input": {"count": "44", "ns": "447955690"}, "compute_infer": {"count": "44", "ns": "9370716606"}, "compute_output": {"count": "44", "ns": "50999023"}}, {"batch_size": "15", "compute_input": {"count": "74", "ns": "754615820"}, "compute_infer": {"count": "74", "ns": "14340957612"}, "compute_output": {"count": "74", "ns": "92314758"}}, {"batch_size": "16", "compute_input": {"count": "1208", "ns": "13579549066"}, "compute_infer": {"count": "1208", "ns": "271179936303"}, "compute_output": {"count": "1208", "ns": "1618070005"}}]}]}

perf_log/model_repo_tlg_mbr/stats_summary-20.txt

@@ -0,0 +1,52 @@
+model name is attention_rescoring 
+queue 0.00  s, infer 142.53 s, input 5.14  s, output 1.24  s 
+Batch_size 1 , 7176  times, infer 142531.98 ms, avg 19.86 ms, 19.86 ms input 5135.27   ms, avg 0.72 ms, output 1241.08   ms, avg 0.17 ms 
+model name is encoder 
+queue 7.89  s, infer 25.38 s, input 0.17  s, output 0.42  s 
+Batch_size 1 , 3029  times, infer 4449.86   ms, avg 1.47 ms, 1.47 ms input 89.04     ms, avg 0.03 ms, output 101.25    ms, avg 0.03 ms 
+Batch_size 2 , 217   times, infer 566.00    ms, avg 2.61 ms, 1.30 ms input 3.48      ms, avg 0.02 ms, output 11.97     ms, avg 0.06 ms 
+Batch_size 3 , 168   times, infer 461.07    ms, avg 2.74 ms, 0.91 ms input 2.58      ms, avg 0.02 ms, output 8.70      ms, avg 0.05 ms 
+Batch_size 4 , 173   times, infer 529.39    ms, avg 3.06 ms, 0.77 ms input 3.60      ms, avg 0.02 ms, output 11.97     ms, avg 0.07 ms 
+Batch_size 5 , 124   times, infer 407.61    ms, avg 3.29 ms, 0.66 ms input 1.68      ms, avg 0.01 ms, output 7.44      ms, avg 0.06 ms 
+Batch_size 6 , 78    times, infer 321.27    ms, avg 4.12 ms, 0.69 ms input 1.54      ms, avg 0.02 ms, output 7.02      ms, avg 0.09 ms 
+Batch_size 7 , 51    times, infer 247.45    ms, avg 4.85 ms, 0.69 ms input 0.92      ms, avg 0.02 ms, output 3.80      ms, avg 0.07 ms 
+Batch_size 8 , 43    times, infer 238.49    ms, avg 5.55 ms, 0.69 ms input 0.99      ms, avg 0.02 ms, output 3.93      ms, avg 0.09 ms 
+Batch_size 9 , 26    times, infer 191.69    ms, avg 7.37 ms, 0.82 ms input 0.63      ms, avg 0.02 ms, output 2.25      ms, avg 0.09 ms 
+Batch_size 10, 26    times, infer 217.75    ms, avg 8.38 ms, 0.84 ms input 0.99      ms, avg 0.04 ms, output 4.61      ms, avg 0.18 ms 
+Batch_size 11, 11    times, infer 158.18    ms, avg 14.38 ms, 1.31 ms input 0.27      ms, avg 0.02 ms, output 0.73      ms, avg 0.07 ms 
+Batch_size 12, 7     times, infer 85.23     ms, avg 12.18 ms, 1.01 ms input 0.31      ms, avg 0.04 ms, output 0.48      ms, avg 0.07 ms 
+Batch_size 13, 1     times, infer 66.08     ms, avg 66.08 ms, 5.08 ms input 0.03      ms, avg 0.03 ms, output 0.06      ms, avg 0.06 ms 
+Batch_size 16, 1     times, infer 72.78     ms, avg 72.78 ms, 4.55 ms input 0.03      ms, avg 0.03 ms, output 0.06      ms, avg 0.06 ms 
+model name is feature_extractor 
+queue 2.37  s, infer 5.50  s, input 0.26  s, output 0.14  s 
+Batch_size 1 , 3752  times, infer 1378.94   ms, avg 0.37 ms, 0.37 ms input 57.00     ms, avg 0.02 ms, output 38.52     ms, avg 0.01 ms 
+Batch_size 2 , 260   times, infer 178.01    ms, avg 0.68 ms, 0.34 ms input 7.00      ms, avg 0.03 ms, output 4.07      ms, avg 0.02 ms 
+Batch_size 3 , 305   times, infer 295.20    ms, avg 0.97 ms, 0.32 ms input 12.47     ms, avg 0.04 ms, output 7.13      ms, avg 0.02 ms 
+Batch_size 4 , 244   times, infer 294.52    ms, avg 1.21 ms, 0.30 ms input 13.47     ms, avg 0.06 ms, output 7.38      ms, avg 0.03 ms 
+Batch_size 5 , 101   times, infer 147.26    ms, avg 1.46 ms, 0.29 ms input 7.12      ms, avg 0.07 ms, output 3.84      ms, avg 0.04 ms 
+Batch_size 6 , 43    times, infer 74.69     ms, avg 1.74 ms, 0.29 ms input 3.56      ms, avg 0.08 ms, output 1.84      ms, avg 0.04 ms 
+Batch_size 7 , 19    times, infer 36.24     ms, avg 1.91 ms, 0.27 ms input 1.96      ms, avg 0.10 ms, output 0.97      ms, avg 0.05 ms 
+Batch_size 8 , 5     times, infer 11.46     ms, avg 2.29 ms, 0.29 ms input 0.57      ms, avg 0.11 ms, output 0.32      ms, avg 0.06 ms 
+Batch_size 9 , 3     times, infer 6.36      ms, avg 2.12 ms, 0.24 ms input 0.42      ms, avg 0.14 ms, output 0.21      ms, avg 0.07 ms 
+Batch_size 10, 1     times, infer 2.87      ms, avg 2.87 ms, 0.29 ms input 0.11      ms, avg 0.11 ms, output 0.07      ms, avg 0.07 ms 
+Batch_size 11, 1     times, infer 1.90      ms, avg 1.90 ms, 0.17 ms input 0.22      ms, avg 0.22 ms, output 0.07      ms, avg 0.07 ms 
+Batch_size 13, 1     times, infer 2.07      ms, avg 2.07 ms, 0.16 ms input 0.21      ms, avg 0.21 ms, output 0.09      ms, avg 0.09 ms 
+Batch_size 16, 1     times, infer 2.29      ms, avg 2.29 ms, 0.14 ms input 0.95      ms, avg 0.95 ms, output 0.13      ms, avg 0.13 ms 
+model name is scoring 
+queue 53.41 s, infer 111.66 s, input 4.70  s, output 0.68  s 
+Batch_size 1 , 704   times, infer 9195.11   ms, avg 13.06 ms, 13.06 ms input 77.50     ms, avg 0.11 ms, output 13.57     ms, avg 0.02 ms 
+Batch_size 2 , 210   times, infer 2740.42   ms, avg 13.05 ms, 6.52 ms input 47.14     ms, avg 0.22 ms, output 6.11      ms, avg 0.03 ms 
+Batch_size 3 , 137   times, infer 1689.54   ms, avg 12.33 ms, 4.11 ms input 28.67     ms, avg 0.21 ms, output 5.15      ms, avg 0.04 ms 
+Batch_size 4 , 49    times, infer 657.27    ms, avg 13.41 ms, 3.35 ms input 15.91     ms, avg 0.32 ms, output 2.24      ms, avg 0.05 ms 
+Batch_size 5 , 25    times, infer 411.44    ms, avg 16.46 ms, 3.29 ms input 10.82     ms, avg 0.43 ms, output 1.50      ms, avg 0.06 ms 
+Batch_size 6 , 39    times, infer 609.39    ms, avg 15.63 ms, 2.60 ms input 15.68     ms, avg 0.40 ms, output 2.42      ms, avg 0.06 ms 
+Batch_size 7 , 31    times, infer 523.55    ms, avg 16.89 ms, 2.41 ms input 14.63     ms, avg 0.47 ms, output 2.16      ms, avg 0.07 ms 
+Batch_size 8 , 29    times, infer 462.90    ms, avg 15.96 ms, 2.00 ms input 16.66     ms, avg 0.57 ms, output 2.22      ms, avg 0.08 ms 
+Batch_size 9 , 20    times, infer 345.06    ms, avg 17.25 ms, 1.92 ms input 12.82     ms, avg 0.64 ms, output 1.57      ms, avg 0.08 ms 
+Batch_size 10, 17    times, infer 281.49    ms, avg 16.56 ms, 1.66 ms input 10.34     ms, avg 0.61 ms, output 1.64      ms, avg 0.10 ms 
+Batch_size 11, 31    times, infer 550.80    ms, avg 17.77 ms, 1.62 ms input 20.12     ms, avg 0.65 ms, output 3.21      ms, avg 0.10 ms 
+Batch_size 12, 25    times, infer 429.06    ms, avg 17.16 ms, 1.43 ms input 19.56     ms, avg 0.78 ms, output 2.44      ms, avg 0.10 ms 
+Batch_size 13, 23    times, infer 389.35    ms, avg 16.93 ms, 1.30 ms input 16.67     ms, avg 0.72 ms, output 2.64      ms, avg 0.11 ms 
+Batch_size 14, 14    times, infer 250.97    ms, avg 17.93 ms, 1.28 ms input 10.54     ms, avg 0.75 ms, output 1.58      ms, avg 0.11 ms 
+Batch_size 15, 33    times, infer 554.58    ms, avg 16.81 ms, 1.12 ms input 27.39     ms, avg 0.83 ms, output 4.25      ms, avg 0.13 ms 
+Batch_size 16, 166   times, infer 2636.15   ms, avg 15.88 ms, 0.99 ms input 159.00    ms, avg 0.96 ms, output 22.40     ms, avg 0.13 ms 

perf_log/model_repo_tlg_mbr/stats_summary-40.txt

@@ -0,0 +1,54 @@
+model name is attention_rescoring 
+queue 0.00  s, infer 317.17 s, input 11.93 s, output 2.69  s 
+Batch_size 1 , 14352 times, infer 317171.58 ms, avg 22.10 ms, 22.10 ms input 11926.57  ms, avg 0.83 ms, output 2687.87   ms, avg 0.19 ms 
+model name is encoder 
+queue 12.21 s, infer 44.34 s, input 0.32  s, output 0.86  s 
+Batch_size 1 , 5136  times, infer 7498.55   ms, avg 1.46 ms, 1.46 ms input 150.94    ms, avg 0.03 ms, output 174.47    ms, avg 0.03 ms 
+Batch_size 2 , 409   times, infer 926.70    ms, avg 2.27 ms, 1.13 ms input 6.36      ms, avg 0.02 ms, output 22.55     ms, avg 0.06 ms 
+Batch_size 3 , 320   times, infer 777.72    ms, avg 2.43 ms, 0.81 ms input 4.09      ms, avg 0.01 ms, output 16.78     ms, avg 0.05 ms 
+Batch_size 4 , 386   times, infer 1045.43   ms, avg 2.71 ms, 0.68 ms input 5.87      ms, avg 0.02 ms, output 25.44     ms, avg 0.07 ms 
+Batch_size 5 , 291   times, infer 816.68    ms, avg 2.81 ms, 0.56 ms input 4.11      ms, avg 0.01 ms, output 19.70     ms, avg 0.07 ms 
+Batch_size 6 , 171   times, infer 555.71    ms, avg 3.25 ms, 0.54 ms input 3.10      ms, avg 0.02 ms, output 15.10     ms, avg 0.09 ms 
+Batch_size 7 , 145   times, infer 499.66    ms, avg 3.45 ms, 0.49 ms input 2.76      ms, avg 0.02 ms, output 9.52      ms, avg 0.07 ms 
+Batch_size 8 , 92    times, infer 379.19    ms, avg 4.12 ms, 0.52 ms input 1.87      ms, avg 0.02 ms, output 7.48      ms, avg 0.08 ms 
+Batch_size 9 , 77    times, infer 351.04    ms, avg 4.56 ms, 0.51 ms input 1.65      ms, avg 0.02 ms, output 7.54      ms, avg 0.10 ms 
+Batch_size 10, 46    times, infer 280.63    ms, avg 6.10 ms, 0.61 ms input 1.41      ms, avg 0.03 ms, output 5.82      ms, avg 0.13 ms 
+Batch_size 11, 24    times, infer 205.91    ms, avg 8.58 ms, 0.78 ms input 0.57      ms, avg 0.02 ms, output 1.64      ms, avg 0.07 ms 
+Batch_size 12, 12    times, infer 160.14    ms, avg 13.35 ms, 1.11 ms input 0.44      ms, avg 0.04 ms, output 0.81      ms, avg 0.07 ms 
+Batch_size 13, 3     times, infer 73.77     ms, avg 24.59 ms, 1.89 ms input 0.08      ms, avg 0.03 ms, output 0.76      ms, avg 0.25 ms 
+Batch_size 14, 1     times, infer 76.58     ms, avg 76.58 ms, 5.47 ms input 0.03      ms, avg 0.03 ms, output 0.07      ms, avg 0.07 ms 
+Batch_size 16, 3     times, infer 146.61    ms, avg 48.87 ms, 3.05 ms input 0.10      ms, avg 0.03 ms, output 0.21      ms, avg 0.07 ms 
+model name is feature_extractor 
+queue 3.63  s, infer 12.26 s, input 0.56  s, output 0.31  s 
+Batch_size 1 , 6487  times, infer 2161.39   ms, avg 0.33 ms, 0.33 ms input 97.49     ms, avg 0.02 ms, output 66.09     ms, avg 0.01 ms 
+Batch_size 2 , 531   times, infer 360.40    ms, avg 0.68 ms, 0.34 ms input 13.94     ms, avg 0.03 ms, output 8.23      ms, avg 0.02 ms 
+Batch_size 3 , 684   times, infer 673.07    ms, avg 0.98 ms, 0.33 ms input 27.15     ms, avg 0.04 ms, output 15.59     ms, avg 0.02 ms 
+Batch_size 4 , 510   times, infer 631.51    ms, avg 1.24 ms, 0.31 ms input 27.68     ms, avg 0.05 ms, output 15.11     ms, avg 0.03 ms 
+Batch_size 5 , 239   times, infer 362.02    ms, avg 1.51 ms, 0.30 ms input 16.27     ms, avg 0.07 ms, output 8.65      ms, avg 0.04 ms 
+Batch_size 6 , 138   times, infer 238.05    ms, avg 1.73 ms, 0.29 ms input 11.05     ms, avg 0.08 ms, output 5.79      ms, avg 0.04 ms 
+Batch_size 7 , 43    times, infer 87.19     ms, avg 2.03 ms, 0.29 ms input 4.19      ms, avg 0.10 ms, output 2.09      ms, avg 0.05 ms 
+Batch_size 8 , 20    times, infer 44.32     ms, avg 2.22 ms, 0.28 ms input 2.20      ms, avg 0.11 ms, output 1.15      ms, avg 0.06 ms 
+Batch_size 9 , 10    times, infer 22.10     ms, avg 2.21 ms, 0.25 ms input 1.29      ms, avg 0.13 ms, output 0.63      ms, avg 0.06 ms 
+Batch_size 10, 3     times, infer 8.49      ms, avg 2.83 ms, 0.28 ms input 0.38      ms, avg 0.13 ms, output 0.19      ms, avg 0.06 ms 
+Batch_size 11, 2     times, infer 4.95      ms, avg 2.48 ms, 0.23 ms input 0.40      ms, avg 0.20 ms, output 0.16      ms, avg 0.08 ms 
+Batch_size 13, 3     times, infer 8.83      ms, avg 2.94 ms, 0.23 ms input 0.55      ms, avg 0.18 ms, output 0.26      ms, avg 0.09 ms 
+Batch_size 14, 1     times, infer 5.34      ms, avg 5.34 ms, 0.38 ms input 0.47      ms, avg 0.47 ms, output 0.12      ms, avg 0.12 ms 
+Batch_size 16, 2     times, infer 6.50      ms, avg 3.25 ms, 0.20 ms input 1.20      ms, avg 0.60 ms, output 0.25      ms, avg 0.13 ms 
+model name is scoring 
+queue 158.48 s, infer 260.56 s, input 11.05 s, output 1.52  s 
+Batch_size 1 , 890   times, infer 12663.84  ms, avg 14.23 ms, 14.23 ms input 103.59    ms, avg 0.12 ms, output 17.24     ms, avg 0.02 ms 
+Batch_size 2 , 245   times, infer 3390.81   ms, avg 13.84 ms, 6.92 ms input 55.89     ms, avg 0.23 ms, output 7.09      ms, avg 0.03 ms 
+Batch_size 3 , 151   times, infer 1924.99   ms, avg 12.75 ms, 4.25 ms input 32.51     ms, avg 0.22 ms, output 5.67      ms, avg 0.04 ms 
+Batch_size 4 , 73    times, infer 1066.16   ms, avg 14.60 ms, 3.65 ms input 25.26     ms, avg 0.35 ms, output 3.31      ms, avg 0.05 ms 
+Batch_size 5 , 36    times, infer 633.74    ms, avg 17.60 ms, 3.52 ms input 16.87     ms, avg 0.47 ms, output 2.16      ms, avg 0.06 ms 
+Batch_size 6 , 50    times, infer 825.00    ms, avg 16.50 ms, 2.75 ms input 20.95     ms, avg 0.42 ms, output 3.07      ms, avg 0.06 ms 
+Batch_size 7 , 62    times, infer 1173.91   ms, avg 18.93 ms, 2.70 ms input 34.20     ms, avg 0.55 ms, output 4.35      ms, avg 0.07 ms 
+Batch_size 8 , 99    times, infer 1925.33   ms, avg 19.45 ms, 2.43 ms input 47.93     ms, avg 0.48 ms, output 7.32      ms, avg 0.07 ms 
+Batch_size 9 , 41    times, infer 808.34    ms, avg 19.72 ms, 2.19 ms input 24.35     ms, avg 0.59 ms, output 3.45      ms, avg 0.08 ms 
+Batch_size 10, 34    times, infer 659.28    ms, avg 19.39 ms, 1.94 ms input 20.88     ms, avg 0.61 ms, output 3.20      ms, avg 0.09 ms 
+Batch_size 11, 45    times, infer 879.30    ms, avg 19.54 ms, 1.78 ms input 29.65     ms, avg 0.66 ms, output 4.66      ms, avg 0.10 ms 
+Batch_size 12, 39    times, infer 755.62    ms, avg 19.37 ms, 1.61 ms input 30.13     ms, avg 0.77 ms, output 3.89      ms, avg 0.10 ms 
+Batch_size 13, 38    times, infer 706.12    ms, avg 18.58 ms, 1.43 ms input 35.34     ms, avg 0.93 ms, output 4.53      ms, avg 0.12 ms 
+Batch_size 14, 29    times, infer 562.52    ms, avg 19.40 ms, 1.39 ms input 29.36     ms, avg 1.01 ms, output 3.40      ms, avg 0.12 ms 
+Batch_size 15, 51    times, infer 922.34    ms, avg 18.09 ms, 1.21 ms input 48.16     ms, avg 0.94 ms, output 6.44      ms, avg 0.13 ms 
+Batch_size 16, 449   times, infer 8490.18   ms, avg 18.91 ms, 1.18 ms input 443.12    ms, avg 0.99 ms, output 61.16     ms, avg 0.14 ms 

perf_log/model_repo_tlg_mbr/stats_summary-60.txt

@@ -0,0 +1,55 @@
+model name is attention_rescoring 
+queue 0.00  s, infer 519.35 s, input 20.27 s, output 4.26  s 
+Batch_size 1 , 21528 times, infer 519353.70 ms, avg 24.12 ms, 24.12 ms input 20274.32  ms, avg 0.94 ms, output 4256.24   ms, avg 0.20 ms 
+model name is encoder 
+queue 17.51 s, infer 64.52 s, input 0.46  s, output 1.34  s 
+Batch_size 1 , 6814  times, infer 10167.17  ms, avg 1.49 ms, 1.49 ms input 197.25    ms, avg 0.03 ms, output 238.04    ms, avg 0.03 ms 
+Batch_size 2 , 583   times, infer 1266.96   ms, avg 2.17 ms, 1.09 ms input 8.44      ms, avg 0.01 ms, output 32.71     ms, avg 0.06 ms 
+Batch_size 3 , 477   times, infer 1138.17   ms, avg 2.39 ms, 0.80 ms input 5.57      ms, avg 0.01 ms, output 25.23     ms, avg 0.05 ms 
+Batch_size 4 , 581   times, infer 1546.98   ms, avg 2.66 ms, 0.67 ms input 8.30      ms, avg 0.01 ms, output 36.70     ms, avg 0.06 ms 
+Batch_size 5 , 438   times, infer 1202.09   ms, avg 2.74 ms, 0.55 ms input 6.29      ms, avg 0.01 ms, output 28.22     ms, avg 0.06 ms 
+Batch_size 6 , 274   times, infer 839.44    ms, avg 3.06 ms, 0.51 ms input 4.56      ms, avg 0.02 ms, output 22.95     ms, avg 0.08 ms 
+Batch_size 7 , 222   times, infer 719.37    ms, avg 3.24 ms, 0.46 ms input 4.17      ms, avg 0.02 ms, output 14.34     ms, avg 0.06 ms 
+Batch_size 8 , 163   times, infer 589.66    ms, avg 3.62 ms, 0.45 ms input 3.16      ms, avg 0.02 ms, output 12.46     ms, avg 0.08 ms 
+Batch_size 9 , 124   times, infer 499.35    ms, avg 4.03 ms, 0.45 ms input 2.58      ms, avg 0.02 ms, output 10.78     ms, avg 0.09 ms 
+Batch_size 10, 89    times, infer 418.80    ms, avg 4.71 ms, 0.47 ms input 2.38      ms, avg 0.03 ms, output 10.45     ms, avg 0.12 ms 
+Batch_size 11, 41    times, infer 266.86    ms, avg 6.51 ms, 0.59 ms input 1.00      ms, avg 0.02 ms, output 2.99      ms, avg 0.07 ms 
+Batch_size 12, 24    times, infer 204.57    ms, avg 8.52 ms, 0.71 ms input 0.75      ms, avg 0.03 ms, output 2.05      ms, avg 0.09 ms 
+Batch_size 13, 12    times, infer 183.03    ms, avg 15.25 ms, 1.17 ms input 0.34      ms, avg 0.03 ms, output 1.49      ms, avg 0.12 ms 
+Batch_size 14, 4     times, infer 152.74    ms, avg 38.19 ms, 2.73 ms input 0.12      ms, avg 0.03 ms, output 0.27      ms, avg 0.07 ms 
+Batch_size 16, 9     times, infer 176.48    ms, avg 19.61 ms, 1.23 ms input 0.31      ms, avg 0.03 ms, output 3.32      ms, avg 0.37 ms 
+model name is feature_extractor 
+queue 5.12  s, infer 19.57 s, input 0.89  s, output 0.49  s 
+Batch_size 1 , 8833  times, infer 2814.62   ms, avg 0.32 ms, 0.32 ms input 131.08    ms, avg 0.01 ms, output 89.24     ms, avg 0.01 ms 
+Batch_size 2 , 805   times, infer 522.28    ms, avg 0.65 ms, 0.32 ms input 20.89     ms, avg 0.03 ms, output 12.69     ms, avg 0.02 ms 
+Batch_size 3 , 1091  times, infer 1065.70   ms, avg 0.98 ms, 0.33 ms input 42.55     ms, avg 0.04 ms, output 24.62     ms, avg 0.02 ms 
+Batch_size 4 , 777   times, infer 971.92    ms, avg 1.25 ms, 0.31 ms input 41.92     ms, avg 0.05 ms, output 22.93     ms, avg 0.03 ms 
+Batch_size 5 , 397   times, infer 606.57    ms, avg 1.53 ms, 0.31 ms input 26.58     ms, avg 0.07 ms, output 14.09     ms, avg 0.04 ms 
+Batch_size 6 , 223   times, infer 388.66    ms, avg 1.74 ms, 0.29 ms input 17.62     ms, avg 0.08 ms, output 9.22      ms, avg 0.04 ms 
+Batch_size 7 , 85    times, infer 170.88    ms, avg 2.01 ms, 0.29 ms input 8.20      ms, avg 0.10 ms, output 4.06      ms, avg 0.05 ms 
+Batch_size 8 , 35    times, infer 81.79     ms, avg 2.34 ms, 0.29 ms input 3.78      ms, avg 0.11 ms, output 1.97      ms, avg 0.06 ms 
+Batch_size 9 , 28    times, infer 66.52     ms, avg 2.38 ms, 0.26 ms input 3.49      ms, avg 0.12 ms, output 1.86      ms, avg 0.07 ms 
+Batch_size 10, 7     times, infer 19.03     ms, avg 2.72 ms, 0.27 ms input 0.93      ms, avg 0.13 ms, output 0.44      ms, avg 0.06 ms 
+Batch_size 11, 5     times, infer 13.47     ms, avg 2.69 ms, 0.24 ms input 0.86      ms, avg 0.17 ms, output 0.40      ms, avg 0.08 ms 
+Batch_size 13, 3     times, infer 8.83      ms, avg 2.94 ms, 0.23 ms input 0.55      ms, avg 0.18 ms, output 0.26      ms, avg 0.09 ms 
+Batch_size 14, 2     times, infer 8.38      ms, avg 4.19 ms, 0.30 ms input 0.68      ms, avg 0.34 ms, output 0.22      ms, avg 0.11 ms 
+Batch_size 15, 2     times, infer 9.07      ms, avg 4.53 ms, 0.30 ms input 0.40      ms, avg 0.20 ms, output 0.20      ms, avg 0.10 ms 
+Batch_size 16, 2     times, infer 6.50      ms, avg 3.25 ms, 0.20 ms input 1.20      ms, avg 0.60 ms, output 0.25      ms, avg 0.13 ms 
+model name is scoring 
+queue 332.92 s, infer 435.27 s, input 18.93 s, output 2.43  s 
+Batch_size 1 , 974   times, infer 14521.62  ms, avg 14.91 ms, 14.91 ms input 113.87    ms, avg 0.12 ms, output 18.87     ms, avg 0.02 ms 
+Batch_size 2 , 257   times, infer 3572.40   ms, avg 13.90 ms, 6.95 ms input 57.90     ms, avg 0.23 ms, output 7.48      ms, avg 0.03 ms 
+Batch_size 3 , 157   times, infer 2051.46   ms, avg 13.07 ms, 4.36 ms input 33.90     ms, avg 0.22 ms, output 5.91      ms, avg 0.04 ms 
+Batch_size 4 , 83    times, infer 1262.85   ms, avg 15.22 ms, 3.80 ms input 28.47     ms, avg 0.34 ms, output 3.79      ms, avg 0.05 ms 
+Batch_size 5 , 39    times, infer 703.17    ms, avg 18.03 ms, 3.61 ms input 17.77     ms, avg 0.46 ms, output 2.33      ms, avg 0.06 ms 
+Batch_size 6 , 54    times, infer 920.91    ms, avg 17.05 ms, 2.84 ms input 22.27     ms, avg 0.41 ms, output 3.33      ms, avg 0.06 ms 
+Batch_size 7 , 67    times, infer 1284.96   ms, avg 19.18 ms, 2.74 ms input 37.08     ms, avg 0.55 ms, output 4.67      ms, avg 0.07 ms 
+Batch_size 8 , 103   times, infer 2005.15   ms, avg 19.47 ms, 2.43 ms input 49.88     ms, avg 0.48 ms, output 7.57      ms, avg 0.07 ms 
+Batch_size 9 , 46    times, infer 920.39    ms, avg 20.01 ms, 2.22 ms input 26.85     ms, avg 0.58 ms, output 3.84      ms, avg 0.08 ms 
+Batch_size 10, 37    times, infer 721.87    ms, avg 19.51 ms, 1.95 ms input 22.61     ms, avg 0.61 ms, output 3.48      ms, avg 0.09 ms 
+Batch_size 11, 48    times, infer 935.39    ms, avg 19.49 ms, 1.77 ms input 31.47     ms, avg 0.66 ms, output 4.94      ms, avg 0.10 ms 
+Batch_size 12, 103   times, infer 2451.15   ms, avg 23.80 ms, 1.98 ms input 100.28    ms, avg 0.97 ms, output 10.75     ms, avg 0.10 ms 
+Batch_size 13, 47    times, infer 931.63    ms, avg 19.82 ms, 1.52 ms input 45.77     ms, avg 0.97 ms, output 5.55      ms, avg 0.12 ms 
+Batch_size 14, 42    times, infer 891.73    ms, avg 21.23 ms, 1.52 ms input 42.43     ms, avg 1.01 ms, output 4.86      ms, avg 0.12 ms 
+Batch_size 15, 66    times, infer 1271.93   ms, avg 19.27 ms, 1.28 ms input 67.79     ms, avg 1.03 ms, output 8.31      ms, avg 0.13 ms 
+Batch_size 16, 793   times, infer 16839.93  ms, avg 21.24 ms, 1.33 ms input 836.44    ms, avg 1.05 ms, output 107.36    ms, avg 0.14 ms 

perf_log/model_repo_tlg_mbr/stats_summary-80.txt

@@ -0,0 +1,57 @@
+model name is attention_rescoring 
+queue 0.00  s, infer 724.44 s, input 29.47 s, output 5.84  s 
+Batch_size 1 , 28704 times, infer 724443.44 ms, avg 25.24 ms, 25.24 ms input 29468.48  ms, avg 1.03 ms, output 5843.61   ms, avg 0.20 ms 
+model name is encoder 
+queue 23.04 s, infer 85.81 s, input 0.61  s, output 1.83  s 
+Batch_size 1 , 8356  times, infer 12748.50  ms, avg 1.53 ms, 1.53 ms input 240.88    ms, avg 0.03 ms, output 293.62    ms, avg 0.04 ms 
+Batch_size 2 , 758   times, infer 1629.29   ms, avg 2.15 ms, 1.07 ms input 10.88     ms, avg 0.01 ms, output 39.21     ms, avg 0.05 ms 
+Batch_size 3 , 593   times, infer 1405.82   ms, avg 2.37 ms, 0.79 ms input 6.83      ms, avg 0.01 ms, output 32.60     ms, avg 0.05 ms 
+Batch_size 4 , 798   times, infer 2112.92   ms, avg 2.65 ms, 0.66 ms input 11.10     ms, avg 0.01 ms, output 50.08     ms, avg 0.06 ms 
+Batch_size 5 , 591   times, infer 1598.66   ms, avg 2.71 ms, 0.54 ms input 8.47      ms, avg 0.01 ms, output 37.88     ms, avg 0.06 ms 
+Batch_size 6 , 390   times, infer 1162.79   ms, avg 2.98 ms, 0.50 ms input 6.44      ms, avg 0.02 ms, output 33.26     ms, avg 0.09 ms 
+Batch_size 7 , 304   times, infer 953.90    ms, avg 3.14 ms, 0.45 ms input 5.51      ms, avg 0.02 ms, output 19.96     ms, avg 0.07 ms 
+Batch_size 8 , 213   times, infer 748.57    ms, avg 3.51 ms, 0.44 ms input 4.33      ms, avg 0.02 ms, output 17.43     ms, avg 0.08 ms 
+Batch_size 9 , 180   times, infer 675.46    ms, avg 3.75 ms, 0.42 ms input 3.73      ms, avg 0.02 ms, output 16.41     ms, avg 0.09 ms 
+Batch_size 10, 111   times, infer 493.12    ms, avg 4.44 ms, 0.44 ms input 2.88      ms, avg 0.03 ms, output 12.00     ms, avg 0.11 ms 
+Batch_size 11, 67    times, infer 361.09    ms, avg 5.39 ms, 0.49 ms input 1.74      ms, avg 0.03 ms, output 5.56      ms, avg 0.08 ms 
+Batch_size 12, 44    times, infer 274.84    ms, avg 6.25 ms, 0.52 ms input 1.31      ms, avg 0.03 ms, output 3.79      ms, avg 0.09 ms 
+Batch_size 13, 24    times, infer 229.82    ms, avg 9.58 ms, 0.74 ms input 0.84      ms, avg 0.04 ms, output 2.48      ms, avg 0.10 ms 
+Batch_size 14, 9     times, infer 171.05    ms, avg 19.01 ms, 1.36 ms input 0.28      ms, avg 0.03 ms, output 0.72      ms, avg 0.08 ms 
+Batch_size 15, 3     times, infer 162.09    ms, avg 54.03 ms, 3.60 ms input 0.10      ms, avg 0.03 ms, output 0.20      ms, avg 0.07 ms 
+Batch_size 16, 16    times, infer 212.79    ms, avg 13.30 ms, 0.83 ms input 0.66      ms, avg 0.04 ms, output 4.26      ms, avg 0.27 ms 
+model name is feature_extractor 
+queue 6.70  s, infer 27.08 s, input 1.23  s, output 0.68  s 
+Batch_size 1 , 11131 times, infer 3447.81   ms, avg 0.31 ms, 0.31 ms input 163.97    ms, avg 0.01 ms, output 112.15    ms, avg 0.01 ms 
+Batch_size 2 , 1076  times, infer 681.45    ms, avg 0.63 ms, 0.32 ms input 27.79     ms, avg 0.03 ms, output 16.92     ms, avg 0.02 ms 
+Batch_size 3 , 1451  times, infer 1391.86   ms, avg 0.96 ms, 0.32 ms input 56.63     ms, avg 0.04 ms, output 32.83     ms, avg 0.02 ms 
+Batch_size 4 , 1058  times, infer 1317.04   ms, avg 1.24 ms, 0.31 ms input 56.68     ms, avg 0.05 ms, output 31.33     ms, avg 0.03 ms 
+Batch_size 5 , 559   times, infer 850.63    ms, avg 1.52 ms, 0.30 ms input 37.08     ms, avg 0.07 ms, output 19.67     ms, avg 0.04 ms 
+Batch_size 6 , 317   times, infer 553.09    ms, avg 1.74 ms, 0.29 ms input 25.04     ms, avg 0.08 ms, output 13.13     ms, avg 0.04 ms 
+Batch_size 7 , 134   times, infer 272.14    ms, avg 2.03 ms, 0.29 ms input 12.69     ms, avg 0.09 ms, output 6.40      ms, avg 0.05 ms 
+Batch_size 8 , 45    times, infer 105.82    ms, avg 2.35 ms, 0.29 ms input 4.93      ms, avg 0.11 ms, output 2.54      ms, avg 0.06 ms 
+Batch_size 9 , 45    times, infer 106.75    ms, avg 2.37 ms, 0.26 ms input 5.61      ms, avg 0.12 ms, output 2.92      ms, avg 0.06 ms 
+Batch_size 10, 11    times, infer 30.38     ms, avg 2.76 ms, 0.28 ms input 1.48      ms, avg 0.13 ms, output 0.73      ms, avg 0.07 ms 
+Batch_size 11, 7     times, infer 22.22     ms, avg 3.17 ms, 0.29 ms input 1.21      ms, avg 0.17 ms, output 0.55      ms, avg 0.08 ms 
+Batch_size 12, 2     times, infer 7.01      ms, avg 3.51 ms, 0.29 ms input 0.37      ms, avg 0.18 ms, output 0.17      ms, avg 0.08 ms 
+Batch_size 13, 3     times, infer 8.83      ms, avg 2.94 ms, 0.23 ms input 0.55      ms, avg 0.18 ms, output 0.26      ms, avg 0.09 ms 
+Batch_size 14, 2     times, infer 8.38      ms, avg 4.19 ms, 0.30 ms input 0.68      ms, avg 0.34 ms, output 0.22      ms, avg 0.11 ms 
+Batch_size 15, 2     times, infer 9.07      ms, avg 4.53 ms, 0.30 ms input 0.40      ms, avg 0.20 ms, output 0.20      ms, avg 0.10 ms 
+Batch_size 16, 8     times, infer 33.93     ms, avg 4.24 ms, 0.27 ms input 2.64      ms, avg 0.33 ms, output 0.88      ms, avg 0.11 ms 
+model name is scoring 
+queue 627.11 s, infer 611.55 s, input 27.63 s, output 3.34  s 
+Batch_size 1 , 1053  times, infer 16291.79  ms, avg 15.47 ms, 15.47 ms input 122.40    ms, avg 0.12 ms, output 20.42     ms, avg 0.02 ms 
+Batch_size 2 , 268   times, infer 3783.00   ms, avg 14.12 ms, 7.06 ms input 59.42     ms, avg 0.22 ms, output 7.77      ms, avg 0.03 ms 
+Batch_size 3 , 165   times, infer 2205.17   ms, avg 13.36 ms, 4.45 ms input 35.08     ms, avg 0.21 ms, output 6.18      ms, avg 0.04 ms 
+Batch_size 4 , 90    times, infer 1428.51   ms, avg 15.87 ms, 3.97 ms input 30.49     ms, avg 0.34 ms, output 4.10      ms, avg 0.05 ms 
+Batch_size 5 , 46    times, infer 841.10    ms, avg 18.28 ms, 3.66 ms input 21.10     ms, avg 0.46 ms, output 2.63      ms, avg 0.06 ms 
+Batch_size 6 , 58    times, infer 1012.43   ms, avg 17.46 ms, 2.91 ms input 23.71     ms, avg 0.41 ms, output 3.59      ms, avg 0.06 ms 
+Batch_size 7 , 69    times, infer 1329.85   ms, avg 19.27 ms, 2.75 ms input 38.36     ms, avg 0.56 ms, output 4.82      ms, avg 0.07 ms 
+Batch_size 8 , 107   times, infer 2096.48   ms, avg 19.59 ms, 2.45 ms input 52.17     ms, avg 0.49 ms, output 7.87      ms, avg 0.07 ms 
+Batch_size 9 , 46    times, infer 920.39    ms, avg 20.01 ms, 2.22 ms input 26.85     ms, avg 0.58 ms, output 3.84      ms, avg 0.08 ms 
+Batch_size 10, 39    times, infer 772.81    ms, avg 19.82 ms, 1.98 ms input 24.93     ms, avg 0.64 ms, output 3.68      ms, avg 0.09 ms 
+Batch_size 11, 51    times, infer 1008.51   ms, avg 19.77 ms, 1.80 ms input 34.01     ms, avg 0.67 ms, output 5.19      ms, avg 0.10 ms 
+Batch_size 12, 104   times, infer 2487.36   ms, avg 23.92 ms, 1.99 ms input 100.73    ms, avg 0.97 ms, output 10.85     ms, avg 0.10 ms 
+Batch_size 13, 52    times, infer 1040.83   ms, avg 20.02 ms, 1.54 ms input 51.73     ms, avg 0.99 ms, output 6.20      ms, avg 0.12 ms 
+Batch_size 14, 44    times, infer 937.07    ms, avg 21.30 ms, 1.52 ms input 44.80     ms, avg 1.02 ms, output 5.10      ms, avg 0.12 ms 
+Batch_size 15, 74    times, infer 1434.10   ms, avg 19.38 ms, 1.29 ms input 75.46     ms, avg 1.02 ms, output 9.23      ms, avg 0.12 ms 
+Batch_size 16, 1208  times, infer 27117.99  ms, avg 22.45 ms, 1.40 ms input 1357.95   ms, avg 1.12 ms, output 161.81    ms, avg 0.13 ms 

perf_log/model_repo_tlg_mbr/stats_summary.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+# Copyright      2023  Nvidia              (authors: Yuekai Zhang)
+#
+# See LICENSE for clarification regarding multiple authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Convert triton staistic json file for better view.
+
+python3 stats_summary.py
+
+"""
+import json
+import argparse
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+
+    parser.add_argument(
+        "--stats_file",
+        type=str,
+        required=False,
+        default="./stats.json",
+        help="output of stats anaylasis",
+    )
+
+    parser.add_argument(
+        "--summary_file",
+        type=str,
+        required=False,
+        default="./stats_summary.txt",
+        help="output of stats summary",
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = get_args()
+
+    with open(args.stats_file) as stats_f, open(
+        args.summary_file, "w"
+    ) as summary_f:
+        stats = json.load(stats_f)
+        model_stats = stats["model_stats"]
+        for model_state in model_stats:
+            if "last_inference" not in model_state:
+                continue
+            summary_f.write(f"model name is {model_state['name']} \n")
+            model_inference_stats = model_state["inference_stats"]
+            total_queue_time_s = (
+                int(model_inference_stats["queue"]["ns"]) / 1e9
+            )
+            total_infer_time_s = (
+                int(model_inference_stats["compute_infer"]["ns"]) / 1e9
+            )
+            total_input_time_s = (
+                int(model_inference_stats["compute_input"]["ns"]) / 1e9
+            )
+            total_output_time_s = (
+                int(model_inference_stats["compute_output"]["ns"]) / 1e9
+            )
+            summary_f.write(
+                f"queue {total_queue_time_s:<5.2f} s, infer {total_infer_time_s:<5.2f} s, input {total_input_time_s:<5.2f} s, output {total_output_time_s:<5.2f} s \n" # noqa
+            )
+            model_batch_stats = model_state["batch_stats"]
+            for batch in model_batch_stats:
+                batch_size = int(batch["batch_size"])
+                compute_input = batch["compute_input"]
+                compute_output = batch["compute_output"]
+                compute_infer = batch["compute_infer"]
+                batch_count = int(compute_infer["count"])
+                assert (
+                    compute_infer["count"]
+                    == compute_output["count"]
+                    == compute_input["count"]
+                )
+                compute_infer_time_ms = int(compute_infer["ns"]) / 1e6
+                compute_input_time_ms = int(compute_input["ns"]) / 1e6
+                compute_output_time_ms = int(compute_output["ns"]) / 1e6
+                summary_f.write(
+                    f"Batch_size {batch_size:<2}, {batch_count:<5} times, infer {compute_infer_time_ms:<9.2f} ms, avg {compute_infer_time_ms/batch_count:.2f} ms, {compute_infer_time_ms/batch_count/batch_size:.2f} ms " # noqa
+                )
+                summary_f.write(
+                    f"input {compute_input_time_ms:<9.2f} ms, avg {compute_input_time_ms/batch_count:.2f} ms, " # noqa
+                )
+                summary_f.write(
+                    f"output {compute_output_time_ms:<9.2f} ms, avg {compute_output_time_ms/batch_count:.2f} ms \n" # noqa
+                )

perf_log/model_repo_tlg_mbr_skip_blank_0.95/rtf-20.txt

@@ -0,0 +1,4 @@
+RTF: 0.0019
+total_duration: 36108.919 seconds
+(10.03 hours)
+processing time: 66.855 seconds (0.02 hours)

perf_log/model_repo_tlg_mbr_skip_blank_0.95/rtf-40.txt

@@ -0,0 +1,4 @@
+RTF: 0.0011
+total_duration: 36108.919 seconds
+(10.03 hours)
+processing time: 41.506 seconds (0.01 hours)

perf_log/model_repo_tlg_mbr_skip_blank_0.95/rtf-60.txt

@@ -0,0 +1,4 @@
+RTF: 0.0010
+total_duration: 36108.919 seconds
+(10.03 hours)
+processing time: 36.758 seconds (0.01 hours)

perf_log/model_repo_tlg_mbr_skip_blank_0.95/rtf-80.txt

@@ -0,0 +1,4 @@
+RTF: 0.0009
+total_duration: 36108.919 seconds
+(10.03 hours)
+processing time: 34.136 seconds (0.01 hours)

perf_log/model_repo_tlg_mbr_skip_blank_0.95/stats-20.json

@@ -0,0 +1 @@
+{"model_stats": [{"name": "attention_rescoring", "version": "1", "last_inference": "1680592206089", "inference_count": "7176", "execution_count": "7176", "inference_stats": {"success": {"count": "7176", "ns": "1089583784371"}, "fail": {}, "queue": {"count": "7176", "ns": "11789685"}, "compute_input": {"count": "7176", "ns": "59495120494"}, "compute_infer": {"count": "7176", "ns": "733016422820"}, "compute_output": {"count": "7176", "ns": "13621638224"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "7176", "ns": "59495120494"}, "compute_infer": {"count": "7176", "ns": "733016422820"}, "compute_output": {"count": "7176", "ns": "13621638224"}}]}, {"name": "decoder", "version": "1", "inference_stats": {"success": {}, "fail": {}, "queue": {}, "compute_input": {}, "compute_infer": {}, "compute_output": {}, "cache_hit": {}, "cache_miss": {}}}, {"name": "encoder", "version": "1", "last_inference": "1680592206074", "inference_count": "7176", "execution_count": "3329", "inference_stats": {"success": {"count": "7176", "ns": "452340570237"}, "fail": {}, "queue": {"count": "7176", "ns": "103300388502"}, "compute_input": {"count": "7176", "ns": "5322224436"}, "compute_infer": {"count": "7176", "ns": "336357976321"}, "compute_output": {"count": "7176", "ns": "6218208869"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "2350", "ns": "806492634"}, "compute_infer": {"count": "2350", "ns": "38584698603"}, "compute_output": {"count": "2350", "ns": "778851147"}}, {"batch_size": "2", "compute_input": {"count": "245", "ns": "71611407"}, "compute_infer": {"count": "245", "ns": "5772205507"}, "compute_output": {"count": "245", "ns": "153305903"}}, {"batch_size": "3", "compute_input": {"count": "157", "ns": "60443844"}, "compute_infer": {"count": "157", "ns": "4364852071"}, "compute_output": {"count": "157", "ns": "113194987"}}, {"batch_size": "4", "compute_input": {"count": "137", "ns": "117494768"}, "compute_infer": {"count": "137", "ns": "4781630049"}, "compute_output": {"count": "137", "ns": "139504203"}}, {"batch_size": "5", "compute_input": {"count": "103", "ns": "93742338"}, "compute_infer": {"count": "103", "ns": "3770514198"}, "compute_output": {"count": "103", "ns": "88128021"}}, {"batch_size": "6", "compute_input": {"count": "70", "ns": "122594034"}, "compute_infer": {"count": "70", "ns": "3038635446"}, "compute_output": {"count": "70", "ns": "111605955"}}, {"batch_size": "7", "compute_input": {"count": "76", "ns": "75944860"}, "compute_infer": {"count": "76", "ns": "3494349050"}, "compute_output": {"count": "76", "ns": "100993421"}}, {"batch_size": "8", "compute_input": {"count": "57", "ns": "66502927"}, "compute_infer": {"count": "57", "ns": "2931361752"}, "compute_output": {"count": "57", "ns": "64832640"}}, {"batch_size": "9", "compute_input": {"count": "52", "ns": "49201040"}, "compute_infer": {"count": "52", "ns": "2937461599"}, "compute_output": {"count": "52", "ns": "55583887"}}, {"batch_size": "10", "compute_input": {"count": "34", "ns": "47107001"}, "compute_infer": {"count": "34", "ns": "2245117373"}, "compute_output": {"count": "34", "ns": "60194313"}}, {"batch_size": "11", "compute_input": {"count": "22", "ns": "16737856"}, "compute_infer": {"count": "22", "ns": "2168514299"}, "compute_output": {"count": "22", "ns": "37260183"}}, {"batch_size": "12", "compute_input": {"count": "5", "ns": "5152237"}, "compute_infer": {"count": "5", "ns": "1472477119"}, "compute_output": {"count": "5", "ns": "5255153"}}, {"batch_size": "13", "compute_input": {"count": "14", "ns": "14716967"}, "compute_infer": {"count": "14", "ns": "1837771212"}, "compute_output": {"count": "14", "ns": "16578198"}}, {"batch_size": "14", "compute_input": {"count": "4", "ns": "5296813"}, "compute_infer": {"count": "4", "ns": "1538201087"}, "compute_output": {"count": "4", "ns": "4913754"}}, {"batch_size": "15", "compute_input": {"count": "2", "ns": "1483673"}, "compute_infer": {"count": "2", "ns": "1440479253"}, "compute_output": {"count": "2", "ns": "2027995"}}, {"batch_size": "16", "compute_input": {"count": "1", "ns": "358211"}, "compute_infer": {"count": "1", "ns": "722958041"}, "compute_output": {"count": "1", "ns": "587639"}}]}, {"name": "feature_extractor", "version": "1", "last_inference": "1680592206061", "inference_count": "7176", "execution_count": "4806", "inference_stats": {"success": {"count": "7176", "ns": "75734626351"}, "fail": {}, "queue": {"count": "7176", "ns": "23485305263"}, "compute_input": {"count": "7176", "ns": "2711281208"}, "compute_infer": {"count": "7176", "ns": "47684512339"}, "compute_output": {"count": "7176", "ns": "1436378611"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "3779", "ns": "591196441"}, "compute_infer": {"count": "3779", "ns": "13936221910"}, "compute_output": {"count": "3779", "ns": "391023129"}}, {"batch_size": "2", "compute_input": {"count": "337", "ns": "94694716"}, "compute_infer": {"count": "337", "ns": "1965997921"}, "compute_output": {"count": "337", "ns": "54975807"}}, {"batch_size": "3", "compute_input": {"count": "328", "ns": "139549292"}, "compute_infer": {"count": "328", "ns": "2898842204"}, "compute_output": {"count": "328", "ns": "78579693"}}, {"batch_size": "4", "compute_input": {"count": "213", "ns": "123174879"}, "compute_infer": {"count": "213", "ns": "2073187802"}, "compute_output": {"count": "213", "ns": "67062421"}}, {"batch_size": "5", "compute_input": {"count": "86", "ns": "62200906"}, "compute_infer": {"count": "86", "ns": "964471614"}, "compute_output": {"count": "86", "ns": "32786953"}}, {"batch_size": "6", "compute_input": {"count": "28", "ns": "25602530"}, "compute_infer": {"count": "28", "ns": "389098473"}, "compute_output": {"count": "28", "ns": "12857769"}}, {"batch_size": "7", "compute_input": {"count": "18", "ns": "19327117"}, "compute_infer": {"count": "18", "ns": "295416108"}, "compute_output": {"count": "18", "ns": "9259345"}}, {"batch_size": "8", "compute_input": {"count": "9", "ns": "10998973"}, "compute_infer": {"count": "9", "ns": "174486588"}, "compute_output": {"count": "9", "ns": "5669847"}}, {"batch_size": "9", "compute_input": {"count": "2", "ns": "2985136"}, "compute_infer": {"count": "2", "ns": "35508795"}, "compute_output": {"count": "2", "ns": "1224698"}}, {"batch_size": "10", "compute_input": {"count": "2", "ns": "2991772"}, "compute_infer": {"count": "2", "ns": "43812835"}, "compute_output": {"count": "2", "ns": "1434887"}}, {"batch_size": "11", "compute_input": {"count": "1", "ns": "1881654"}, "compute_infer": {"count": "1", "ns": "27867814"}, "compute_output": {"count": "1", "ns": "792575"}}, {"batch_size": "12", "compute_input": {"count": "1", "ns": "1860610"}, "compute_infer": {"count": "1", "ns": "30934344"}, "compute_output": {"count": "1", "ns": "826823"}}, {"batch_size": "14", "compute_input": {"count": "1", "ns": "5769686"}, "compute_infer": {"count": "1", "ns": "30586402"}, "compute_output": {"count": "1", "ns": "1243797"}}, {"batch_size": "16", "compute_input": {"count": "1", "ns": "9428848"}, "compute_infer": {"count": "1", "ns": "21411699"}, "compute_output": {"count": "1", "ns": "1170939"}}]}, {"name": "scoring", "version": "1", "last_inference": "1680592206089", "inference_count": "7176", "execution_count": "2118", "inference_stats": {"success": {"count": "7176", "ns": "561871013164"}, "fail": {}, "queue": {"count": "7176", "ns": "154425210512"}, "compute_input": {"count": "7176", "ns": "51461614850"}, "compute_infer": {"count": "7176", "ns": "348973934160"}, "compute_output": {"count": "7176", "ns": "5967050744"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "1133", "ns": "1957243460"}, "compute_infer": {"count": "1133", "ns": "49068145126"}, "compute_output": {"count": "1133", "ns": "193559903"}}, {"batch_size": "2", "compute_input": {"count": "405", "ns": "1096832437"}, "compute_infer": {"count": "405", "ns": "15299884583"}, "compute_output": {"count": "405", "ns": "107522700"}}, {"batch_size": "3", "compute_input": {"count": "128", "ns": "369491374"}, "compute_infer": {"count": "128", "ns": "4890968142"}, "compute_output": {"count": "128", "ns": "44411727"}}, {"batch_size": "4", "compute_input": {"count": "46", "ns": "220333309"}, "compute_infer": {"count": "46", "ns": "2010225823"}, "compute_output": {"count": "46", "ns": "19745689"}}, {"batch_size": "5", "compute_input": {"count": "25", "ns": "126334265"}, "compute_infer": {"count": "25", "ns": "1296761756"}, "compute_output": {"count": "25", "ns": "13156473"}}, {"batch_size": "6", "compute_input": {"count": "41", "ns": "229889782"}, "compute_infer": {"count": "41", "ns": "1981835514"}, "compute_output": {"count": "41", "ns": "24796122"}}, {"batch_size": "7", "compute_input": {"count": "29", "ns": "223294440"}, "compute_infer": {"count": "29", "ns": "1510357164"}, "compute_output": {"count": "29", "ns": "18702805"}}, {"batch_size": "8", "compute_input": {"count": "36", "ns": "330619715"}, "compute_infer": {"count": "36", "ns": "1710735569"}, "compute_output": {"count": "36", "ns": "24942034"}}, {"batch_size": "9", "compute_input": {"count": "28", "ns": "218260103"}, "compute_infer": {"count": "28", "ns": "1587654525"}, "compute_output": {"count": "28", "ns": "22715937"}}, {"batch_size": "10", "compute_input": {"count": "25", "ns": "219049168"}, "compute_infer": {"count": "25", "ns": "1465358807"}, "compute_output": {"count": "25", "ns": "21938049"}}, {"batch_size": "11", "compute_input": {"count": "24", "ns": "269997932"}, "compute_infer": {"count": "24", "ns": "1308288763"}, "compute_output": {"count": "24", "ns": "25678355"}}, {"batch_size": "12", "compute_input": {"count": "13", "ns": "98460219"}, "compute_infer": {"count": "13", "ns": "745335513"}, "compute_output": {"count": "13", "ns": "13878619"}}, {"batch_size": "13", "compute_input": {"count": "13", "ns": "112887370"}, "compute_infer": {"count": "13", "ns": "759780434"}, "compute_output": {"count": "13", "ns": "14474372"}}, {"batch_size": "14", "compute_input": {"count": "14", "ns": "151081832"}, "compute_infer": {"count": "14", "ns": "837733721"}, "compute_output": {"count": "14", "ns": "16672592"}}, {"batch_size": "15", "compute_input": {"count": "12", "ns": "117984164"}, "compute_infer": {"count": "12", "ns": "705827334"}, "compute_output": {"count": "12", "ns": "15693400"}}, {"batch_size": "16", "compute_input": {"count": "146", "ns": "1590193371"}, "compute_infer": {"count": "146", "ns": "7468031031"}, "compute_output": {"count": "146", "ns": "204462445"}}]}]}

perf_log/model_repo_tlg_mbr_skip_blank_0.95/stats-40.json

@@ -0,0 +1 @@
+{"model_stats": [{"name": "attention_rescoring", "version": "1", "last_inference": "1680592252072", "inference_count": "14352", "execution_count": "14352", "inference_stats": {"success": {"count": "14352", "ns": "2354619589376"}, "fail": {}, "queue": {"count": "14352", "ns": "23179848"}, "compute_input": {"count": "14352", "ns": "149359270024"}, "compute_infer": {"count": "14352", "ns": "1532763416160"}, "compute_output": {"count": "14352", "ns": "32651315841"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "14352", "ns": "149359270024"}, "compute_infer": {"count": "14352", "ns": "1532763416160"}, "compute_output": {"count": "14352", "ns": "32651315841"}}]}, {"name": "decoder", "version": "1", "inference_stats": {"success": {}, "fail": {}, "queue": {}, "compute_input": {}, "compute_infer": {}, "compute_output": {}, "cache_hit": {}, "cache_miss": {}}}, {"name": "encoder", "version": "1", "last_inference": "1680592252046", "inference_count": "14352", "execution_count": "5776", "inference_stats": {"success": {"count": "14352", "ns": "756688732190"}, "fail": {}, "queue": {"count": "14352", "ns": "165462141694"}, "compute_input": {"count": "14352", "ns": "9564375892"}, "compute_infer": {"count": "14352", "ns": "563218367954"}, "compute_output": {"count": "14352", "ns": "15827670589"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "3800", "ns": "1240800955"}, "compute_infer": {"count": "3800", "ns": "63031418363"}, "compute_output": {"count": "3800", "ns": "1330720509"}}, {"batch_size": "2", "compute_input": {"count": "430", "ns": "113483855"}, "compute_infer": {"count": "430", "ns": "9433299313"}, "compute_output": {"count": "430", "ns": "265378157"}}, {"batch_size": "3", "compute_input": {"count": "302", "ns": "114126383"}, "compute_infer": {"count": "302", "ns": "7754730877"}, "compute_output": {"count": "302", "ns": "255846792"}}, {"batch_size": "4", "compute_input": {"count": "264", "ns": "183315090"}, "compute_infer": {"count": "264", "ns": "7930138670"}, "compute_output": {"count": "264", "ns": "246349910"}}, {"batch_size": "5", "compute_input": {"count": "231", "ns": "143136078"}, "compute_infer": {"count": "231", "ns": "7225764536"}, "compute_output": {"count": "231", "ns": "279011169"}}, {"batch_size": "6", "compute_input": {"count": "161", "ns": "167218048"}, "compute_infer": {"count": "161", "ns": "5670787968"}, "compute_output": {"count": "161", "ns": "230258218"}}, {"batch_size": "7", "compute_input": {"count": "142", "ns": "111869394"}, "compute_infer": {"count": "142", "ns": "5555555356"}, "compute_output": {"count": "142", "ns": "212420146"}}, {"batch_size": "8", "compute_input": {"count": "108", "ns": "111443994"}, "compute_infer": {"count": "108", "ns": "4571075796"}, "compute_output": {"count": "108", "ns": "155289472"}}, {"batch_size": "9", "compute_input": {"count": "92", "ns": "76259245"}, "compute_infer": {"count": "92", "ns": "4392654100"}, "compute_output": {"count": "92", "ns": "130098349"}}, {"batch_size": "10", "compute_input": {"count": "81", "ns": "105069998"}, "compute_infer": {"count": "81", "ns": "4033259049"}, "compute_output": {"count": "81", "ns": "145348455"}}, {"batch_size": "11", "compute_input": {"count": "54", "ns": "39510031"}, "compute_infer": {"count": "54", "ns": "3420131628"}, "compute_output": {"count": "54", "ns": "93120683"}}, {"batch_size": "12", "compute_input": {"count": "27", "ns": "23434437"}, "compute_infer": {"count": "27", "ns": "2359812723"}, "compute_output": {"count": "27", "ns": "59210813"}}, {"batch_size": "13", "compute_input": {"count": "30", "ns": "27068671"}, "compute_infer": {"count": "30", "ns": "2530022308"}, "compute_output": {"count": "30", "ns": "49029786"}}, {"batch_size": "14", "compute_input": {"count": "24", "ns": "25810601"}, "compute_infer": {"count": "24", "ns": "2381582173"}, "compute_output": {"count": "24", "ns": "45546977"}}, {"batch_size": "15", "compute_input": {"count": "11", "ns": "8910705"}, "compute_infer": {"count": "11", "ns": "1874442438"}, "compute_output": {"count": "11", "ns": "20488793"}}, {"batch_size": "16", "compute_input": {"count": "19", "ns": "20473928"}, "compute_infer": {"count": "19", "ns": "2535612771"}, "compute_output": {"count": "19", "ns": "47861753"}}]}, {"name": "feature_extractor", "version": "1", "last_inference": "1680592252028", "inference_count": "14352", "execution_count": "8714", "inference_stats": {"success": {"count": "14352", "ns": "154770055886"}, "fail": {}, "queue": {"count": "14352", "ns": "36076037685"}, "compute_input": {"count": "14352", "ns": "5958875389"}, "compute_infer": {"count": "14352", "ns": "108627704846"}, "compute_output": {"count": "14352", "ns": "3176913123"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "6424", "ns": "1008993952"}, "compute_infer": {"count": "6424", "ns": "21421255347"}, "compute_output": {"count": "6424", "ns": "669509299"}}, {"batch_size": "2", "compute_input": {"count": "672", "ns": "191016115"}, "compute_infer": {"count": "672", "ns": "3996954180"}, "compute_output": {"count": "672", "ns": "112103004"}}, {"batch_size": "3", "compute_input": {"count": "716", "ns": "305770405"}, "compute_infer": {"count": "716", "ns": "6322934556"}, "compute_output": {"count": "716", "ns": "172863395"}}, {"batch_size": "4", "compute_input": {"count": "476", "ns": "274440279"}, "compute_infer": {"count": "476", "ns": "4958149814"}, "compute_output": {"count": "476", "ns": "148902926"}}, {"batch_size": "5", "compute_input": {"count": "228", "ns": "167947376"}, "compute_infer": {"count": "228", "ns": "2835910235"}, "compute_output": {"count": "228", "ns": "85529881"}}, {"batch_size": "6", "compute_input": {"count": "104", "ns": "90264949"}, "compute_infer": {"count": "104", "ns": "1644046644"}, "compute_output": {"count": "104", "ns": "45101807"}}, {"batch_size": "7", "compute_input": {"count": "46", "ns": "47306325"}, "compute_infer": {"count": "46", "ns": "796981185"}, "compute_output": {"count": "46", "ns": "22885174"}}, {"batch_size": "8", "compute_input": {"count": "25", "ns": "30854479"}, "compute_infer": {"count": "25", "ns": "494515045"}, "compute_output": {"count": "25", "ns": "14550551"}}, {"batch_size": "9", "compute_input": {"count": "9", "ns": "11983361"}, "compute_infer": {"count": "9", "ns": "193986893"}, "compute_output": {"count": "9", "ns": "5632784"}}, {"batch_size": "10", "compute_input": {"count": "6", "ns": "9006656"}, "compute_infer": {"count": "6", "ns": "160085327"}, "compute_output": {"count": "6", "ns": "4571352"}}, {"batch_size": "11", "compute_input": {"count": "2", "ns": "3192270"}, "compute_infer": {"count": "2", "ns": "54806994"}, "compute_output": {"count": "2", "ns": "1578331"}}, {"batch_size": "12", "compute_input": {"count": "2", "ns": "3947968"}, "compute_infer": {"count": "2", "ns": "52738899"}, "compute_output": {"count": "2", "ns": "1780814"}}, {"batch_size": "13", "compute_input": {"count": "1", "ns": "1799132"}, "compute_infer": {"count": "1", "ns": "43054040"}, "compute_output": {"count": "1", "ns": "779875"}}, {"batch_size": "14", "compute_input": {"count": "1", "ns": "5769686"}, "compute_infer": {"count": "1", "ns": "30586402"}, "compute_output": {"count": "1", "ns": "1243797"}}, {"batch_size": "16", "compute_input": {"count": "2", "ns": "13056155"}, "compute_infer": {"count": "2", "ns": "78870459"}, "compute_output": {"count": "2", "ns": "1965221"}}]}, {"name": "scoring", "version": "1", "last_inference": "1680592252072", "inference_count": "14352", "execution_count": "3169", "inference_stats": {"success": {"count": "14352", "ns": "1443765266692"}, "fail": {}, "queue": {"count": "14352", "ns": "432927425915"}, "compute_input": {"count": "14352", "ns": "133836018743"}, "compute_infer": {"count": "14352", "ns": "860917343360"}, "compute_output": {"count": "14352", "ns": "13646732129"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "1485", "ns": "2755835664"}, "compute_infer": {"count": "1485", "ns": "69446525374"}, "compute_output": {"count": "1485", "ns": "255373896"}}, {"batch_size": "2", "compute_input": {"count": "504", "ns": "1453351888"}, "compute_infer": {"count": "504", "ns": "21986980277"}, "compute_output": {"count": "504", "ns": "135270621"}}, {"batch_size": "3", "compute_input": {"count": "168", "ns": "592184996"}, "compute_infer": {"count": "168", "ns": "7605936134"}, "compute_output": {"count": "168", "ns": "59258868"}}, {"batch_size": "4", "compute_input": {"count": "84", "ns": "526825423"}, "compute_infer": {"count": "84", "ns": "5022508354"}, "compute_output": {"count": "84", "ns": "36195879"}}, {"batch_size": "5", "compute_input": {"count": "56", "ns": "388003475"}, "compute_infer": {"count": "56", "ns": "3379198249"}, "compute_output": {"count": "56", "ns": "28262363"}}, {"batch_size": "6", "compute_input": {"count": "66", "ns": "416236557"}, "compute_infer": {"count": "66", "ns": "3733474332"}, "compute_output": {"count": "66", "ns": "39962551"}}, {"batch_size": "7", "compute_input": {"count": "54", "ns": "451698164"}, "compute_infer": {"count": "54", "ns": "3198170534"}, "compute_output": {"count": "54", "ns": "35861801"}}, {"batch_size": "8", "compute_input": {"count": "93", "ns": "697373295"}, "compute_infer": {"count": "93", "ns": "4997845771"}, "compute_output": {"count": "93", "ns": "65775642"}}, {"batch_size": "9", "compute_input": {"count": "57", "ns": "443952738"}, "compute_infer": {"count": "57", "ns": "3823356803"}, "compute_output": {"count": "57", "ns": "46463533"}}, {"batch_size": "10", "compute_input": {"count": "59", "ns": "517446488"}, "compute_infer": {"count": "59", "ns": "3805599953"}, "compute_output": {"count": "59", "ns": "52007153"}}, {"batch_size": "11", "compute_input": {"count": "42", "ns": "436713045"}, "compute_infer": {"count": "42", "ns": "2771304893"}, "compute_output": {"count": "42", "ns": "44666568"}}, {"batch_size": "12", "compute_input": {"count": "24", "ns": "207787436"}, "compute_infer": {"count": "24", "ns": "1597516769"}, "compute_output": {"count": "24", "ns": "25485016"}}, {"batch_size": "13", "compute_input": {"count": "45", "ns": "510373567"}, "compute_infer": {"count": "45", "ns": "2922905116"}, "compute_output": {"count": "45", "ns": "49673041"}}, {"batch_size": "14", "compute_input": {"count": "47", "ns": "634090393"}, "compute_infer": {"count": "47", "ns": "3518185094"}, "compute_output": {"count": "47", "ns": "57317285"}}, {"batch_size": "15", "compute_input": {"count": "35", "ns": "463892188"}, "compute_infer": {"count": "35", "ns": "2352444569"}, "compute_output": {"count": "35", "ns": "46273331"}}, {"batch_size": "16", "compute_input": {"count": "350", "ns": "4510842216"}, "compute_infer": {"count": "350", "ns": "22391401644"}, "compute_output": {"count": "350", "ns": "485140297"}}]}]}

perf_log/model_repo_tlg_mbr_skip_blank_0.95/stats-60.json

@@ -0,0 +1 @@
+{"model_stats": [{"name": "attention_rescoring", "version": "1", "last_inference": "1680592293445", "inference_count": "21528", "execution_count": "21528", "inference_stats": {"success": {"count": "21528", "ns": "4110931366112"}, "fail": {}, "queue": {"count": "21528", "ns": "33941951"}, "compute_input": {"count": "21528", "ns": "258943332232"}, "compute_infer": {"count": "21528", "ns": "2570223737596"}, "compute_output": {"count": "21528", "ns": "54813615477"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "21528", "ns": "258943332232"}, "compute_infer": {"count": "21528", "ns": "2570223737596"}, "compute_output": {"count": "21528", "ns": "54813615477"}}]}, {"name": "decoder", "version": "1", "inference_stats": {"success": {}, "fail": {}, "queue": {}, "compute_input": {}, "compute_infer": {}, "compute_output": {}, "cache_hit": {}, "cache_miss": {}}}, {"name": "encoder", "version": "1", "last_inference": "1680592293430", "inference_count": "21528", "execution_count": "7943", "inference_stats": {"success": {"count": "21528", "ns": "1077632823431"}, "fail": {}, "queue": {"count": "21528", "ns": "239859956623"}, "compute_input": {"count": "21528", "ns": "13319982566"}, "compute_infer": {"count": "21528", "ns": "793035746167"}, "compute_output": {"count": "21528", "ns": "27206988065"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "4953", "ns": "1660729320"}, "compute_infer": {"count": "4953", "ns": "83422973398"}, "compute_output": {"count": "4953", "ns": "1818372503"}}, {"batch_size": "2", "compute_input": {"count": "606", "ns": "169854603"}, "compute_infer": {"count": "606", "ns": "14596713650"}, "compute_output": {"count": "606", "ns": "404047795"}}, {"batch_size": "3", "compute_input": {"count": "438", "ns": "150516818"}, "compute_infer": {"count": "438", "ns": "11055181495"}, "compute_output": {"count": "438", "ns": "386023500"}}, {"batch_size": "4", "compute_input": {"count": "409", "ns": "227156078"}, "compute_infer": {"count": "409", "ns": "11665672208"}, "compute_output": {"count": "409", "ns": "380868954"}}, {"batch_size": "5", "compute_input": {"count": "333", "ns": "184105130"}, "compute_infer": {"count": "333", "ns": "9973342404"}, "compute_output": {"count": "333", "ns": "394001421"}}, {"batch_size": "6", "compute_input": {"count": "262", "ns": "222052546"}, "compute_infer": {"count": "262", "ns": "8570057248"}, "compute_output": {"count": "262", "ns": "346637085"}}, {"batch_size": "7", "compute_input": {"count": "218", "ns": "141293494"}, "compute_infer": {"count": "218", "ns": "7854122131"}, "compute_output": {"count": "218", "ns": "305560302"}}, {"batch_size": "8", "compute_input": {"count": "163", "ns": "138796594"}, "compute_infer": {"count": "163", "ns": "6464991387"}, "compute_output": {"count": "163", "ns": "275704577"}}, {"batch_size": "9", "compute_input": {"count": "153", "ns": "113079814"}, "compute_infer": {"count": "153", "ns": "6471106500"}, "compute_output": {"count": "153", "ns": "251679153"}}, {"batch_size": "10", "compute_input": {"count": "117", "ns": "136084691"}, "compute_infer": {"count": "117", "ns": "5444170839"}, "compute_output": {"count": "117", "ns": "208530952"}}, {"batch_size": "11", "compute_input": {"count": "82", "ns": "53879457"}, "compute_infer": {"count": "82", "ns": "4604904277"}, "compute_output": {"count": "82", "ns": "140498310"}}, {"batch_size": "12", "compute_input": {"count": "57", "ns": "42603365"}, "compute_infer": {"count": "57", "ns": "3632283558"}, "compute_output": {"count": "57", "ns": "135188515"}}, {"batch_size": "13", "compute_input": {"count": "46", "ns": "33518017"}, "compute_infer": {"count": "46", "ns": "3155126158"}, "compute_output": {"count": "46", "ns": "113416460"}}, {"batch_size": "14", "compute_input": {"count": "31", "ns": "30337830"}, "compute_infer": {"count": "31", "ns": "2701461618"}, "compute_output": {"count": "31", "ns": "81593140"}}, {"batch_size": "15", "compute_input": {"count": "19", "ns": "16025930"}, "compute_infer": {"count": "19", "ns": "2271818628"}, "compute_output": {"count": "19", "ns": "40920676"}}, {"batch_size": "16", "compute_input": {"count": "56", "ns": "63985544"}, "compute_infer": {"count": "56", "ns": "4547991992"}, "compute_output": {"count": "56", "ns": "172243234"}}]}, {"name": "feature_extractor", "version": "1", "last_inference": "1680592293416", "inference_count": "21528", "execution_count": "12179", "inference_stats": {"success": {"count": "21528", "ns": "250981436960"}, "fail": {}, "queue": {"count": "21528", "ns": "51599741728"}, "compute_input": {"count": "21528", "ns": "9632380073"}, "compute_infer": {"count": "21528", "ns": "183088886406"}, "compute_output": {"count": "21528", "ns": "5158726054"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "8608", "ns": "1337938720"}, "compute_infer": {"count": "8608", "ns": "27630608699"}, "compute_output": {"count": "8608", "ns": "891078157"}}, {"batch_size": "2", "compute_input": {"count": "966", "ns": "270018649"}, "compute_infer": {"count": "966", "ns": "5678948946"}, "compute_output": {"count": "966", "ns": "158899748"}}, {"batch_size": "3", "compute_input": {"count": "1066", "ns": "445912125"}, "compute_infer": {"count": "1066", "ns": "9490081856"}, "compute_output": {"count": "1066", "ns": "253612936"}}, {"batch_size": "4", "compute_input": {"count": "758", "ns": "428192303"}, "compute_infer": {"count": "758", "ns": "8266376714"}, "compute_output": {"count": "758", "ns": "231985222"}}, {"batch_size": "5", "compute_input": {"count": "391", "ns": "278957228"}, "compute_infer": {"count": "391", "ns": "5202948610"}, "compute_output": {"count": "391", "ns": "146003961"}}, {"batch_size": "6", "compute_input": {"count": "197", "ns": "168104322"}, "compute_infer": {"count": "197", "ns": "3155226580"}, "compute_output": {"count": "197", "ns": "84557919"}}, {"batch_size": "7", "compute_input": {"count": "85", "ns": "85262743"}, "compute_infer": {"count": "85", "ns": "1571347625"}, "compute_output": {"count": "85", "ns": "42092431"}}, {"batch_size": "8", "compute_input": {"count": "50", "ns": "59799237"}, "compute_infer": {"count": "50", "ns": "1000882607"}, "compute_output": {"count": "50", "ns": "30218871"}}, {"batch_size": "9", "compute_input": {"count": "25", "ns": "32549697"}, "compute_infer": {"count": "25", "ns": "573857035"}, "compute_output": {"count": "25", "ns": "15353687"}}, {"batch_size": "10", "compute_input": {"count": "14", "ns": "20764472"}, "compute_infer": {"count": "14", "ns": "390061694"}, "compute_output": {"count": "14", "ns": "9765118"}}, {"batch_size": "11", "compute_input": {"count": "3", "ns": "4504996"}, "compute_infer": {"count": "3", "ns": "84826275"}, "compute_output": {"count": "3", "ns": "2213688"}}, {"batch_size": "12", "compute_input": {"count": "4", "ns": "7555991"}, "compute_infer": {"count": "4", "ns": "116909453"}, "compute_output": {"count": "4", "ns": "3497580"}}, {"batch_size": "13", "compute_input": {"count": "2", "ns": "3209737"}, "compute_infer": {"count": "2", "ns": "69706469"}, "compute_output": {"count": "2", "ns": "1642008"}}, {"batch_size": "14", "compute_input": {"count": "2", "ns": "7740604"}, "compute_infer": {"count": "2", "ns": "68949631"}, "compute_output": {"count": "2", "ns": "2223423"}}, {"batch_size": "15", "compute_input": {"count": "2", "ns": "4551315"}, "compute_infer": {"count": "2", "ns": "78749353"}, "compute_output": {"count": "2", "ns": "2154116"}}, {"batch_size": "16", "compute_input": {"count": "6", "ns": "22879356"}, "compute_infer": {"count": "6", "ns": "259873668"}, "compute_output": {"count": "6", "ns": "6272609"}}]}, {"name": "scoring", "version": "1", "last_inference": "1680592293445", "inference_count": "21528", "execution_count": "3846", "inference_stats": {"success": {"count": "21528", "ns": "2782775768571"}, "fail": {}, "queue": {"count": "21528", "ns": "926250330612"}, "compute_input": {"count": "21528", "ns": "235990969593"}, "compute_infer": {"count": "21528", "ns": "1594099105023"}, "compute_output": {"count": "21528", "ns": "22447901358"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "1608", "ns": "3151809615"}, "compute_infer": {"count": "1608", "ns": "77165551559"}, "compute_output": {"count": "1608", "ns": "277498018"}}, {"batch_size": "2", "compute_input": {"count": "530", "ns": "1584850929"}, "compute_infer": {"count": "530", "ns": "24089217281"}, "compute_output": {"count": "530", "ns": "142433932"}}, {"batch_size": "3", "compute_input": {"count": "183", "ns": "699415010"}, "compute_infer": {"count": "183", "ns": "8828883216"}, "compute_output": {"count": "183", "ns": "64978605"}}, {"batch_size": "4", "compute_input": {"count": "100", "ns": "661114938"}, "compute_infer": {"count": "100", "ns": "6452211834"}, "compute_output": {"count": "100", "ns": "42676325"}}, {"batch_size": "5", "compute_input": {"count": "72", "ns": "531290170"}, "compute_infer": {"count": "72", "ns": "4907868976"}, "compute_output": {"count": "72", "ns": "35879807"}}, {"batch_size": "6", "compute_input": {"count": "81", "ns": "532843683"}, "compute_infer": {"count": "81", "ns": "5230419942"}, "compute_output": {"count": "81", "ns": "48484191"}}, {"batch_size": "7", "compute_input": {"count": "63", "ns": "589270125"}, "compute_infer": {"count": "63", "ns": "4086703795"}, "compute_output": {"count": "63", "ns": "41993079"}}, {"batch_size": "8", "compute_input": {"count": "102", "ns": "816774555"}, "compute_infer": {"count": "102", "ns": "5821722120"}, "compute_output": {"count": "102", "ns": "72602969"}}, {"batch_size": "9", "compute_input": {"count": "75", "ns": "636597156"}, "compute_infer": {"count": "75", "ns": "5571410303"}, "compute_output": {"count": "75", "ns": "61086051"}}, {"batch_size": "10", "compute_input": {"count": "72", "ns": "655393404"}, "compute_infer": {"count": "72", "ns": "5144162840"}, "compute_output": {"count": "72", "ns": "64658862"}}, {"batch_size": "11", "compute_input": {"count": "50", "ns": "534856128"}, "compute_infer": {"count": "50", "ns": "3369976150"}, "compute_output": {"count": "50", "ns": "51443997"}}, {"batch_size": "12", "compute_input": {"count": "85", "ns": "884321590"}, "compute_infer": {"count": "85", "ns": "8183842689"}, "compute_output": {"count": "85", "ns": "91802529"}}, {"batch_size": "13", "compute_input": {"count": "64", "ns": "793020551"}, "compute_infer": {"count": "64", "ns": "5117564317"}, "compute_output": {"count": "64", "ns": "70975910"}}, {"batch_size": "14", "compute_input": {"count": "59", "ns": "807486923"}, "compute_infer": {"count": "59", "ns": "4641745356"}, "compute_output": {"count": "59", "ns": "74742623"}}, {"batch_size": "15", "compute_input": {"count": "47", "ns": "649650246"}, "compute_infer": {"count": "47", "ns": "3752089553"}, "compute_output": {"count": "47", "ns": "62907943"}}, {"batch_size": "16", "compute_input": {"count": "655", "ns": "9267295648"}, "compute_infer": {"count": "655", "ns": "53793923026"}, "compute_output": {"count": "655", "ns": "899889674"}}]}]}

perf_log/model_repo_tlg_mbr_skip_blank_0.95/stats-80.json

@@ -0,0 +1 @@
+{"model_stats": [{"name": "attention_rescoring", "version": "1", "last_inference": "1680592332128", "inference_count": "28704", "execution_count": "28704", "inference_stats": {"success": {"count": "28704", "ns": "6376793122822"}, "fail": {}, "queue": {"count": "28704", "ns": "44589041"}, "compute_input": {"count": "28704", "ns": "391710867083"}, "compute_infer": {"count": "28704", "ns": "3688057032930"}, "compute_output": {"count": "28704", "ns": "84777491512"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "28704", "ns": "391710867083"}, "compute_infer": {"count": "28704", "ns": "3688057032930"}, "compute_output": {"count": "28704", "ns": "84777491512"}}]}, {"name": "decoder", "version": "1", "inference_stats": {"success": {}, "fail": {}, "queue": {}, "compute_input": {}, "compute_infer": {}, "compute_output": {}, "cache_hit": {}, "cache_miss": {}}}, {"name": "encoder", "version": "1", "last_inference": "1680592332113", "inference_count": "28704", "execution_count": "9791", "inference_stats": {"success": {"count": "28704", "ns": "1460263497689"}, "fail": {}, "queue": {"count": "28704", "ns": "327803168447"}, "compute_input": {"count": "28704", "ns": "17392941531"}, "compute_infer": {"count": "28704", "ns": "1063468042315"}, "compute_output": {"count": "28704", "ns": "45550204913"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "5891", "ns": "1985771997"}, "compute_infer": {"count": "5891", "ns": "101455303888"}, "compute_output": {"count": "5891", "ns": "2293382500"}}, {"batch_size": "2", "compute_input": {"count": "728", "ns": "206830147"}, "compute_infer": {"count": "728", "ns": "17577806157"}, "compute_output": {"count": "728", "ns": "542989136"}}, {"batch_size": "3", "compute_input": {"count": "549", "ns": "180429223"}, "compute_infer": {"count": "549", "ns": "13964072841"}, "compute_output": {"count": "549", "ns": "499372384"}}, {"batch_size": "4", "compute_input": {"count": "533", "ns": "271582670"}, "compute_infer": {"count": "533", "ns": "15313172952"}, "compute_output": {"count": "533", "ns": "493000543"}}, {"batch_size": "5", "compute_input": {"count": "415", "ns": "213735830"}, "compute_infer": {"count": "415", "ns": "12199955179"}, "compute_output": {"count": "415", "ns": "479115163"}}, {"batch_size": "6", "compute_input": {"count": "354", "ns": "252299888"}, "compute_infer": {"count": "354", "ns": "11279707171"}, "compute_output": {"count": "354", "ns": "508703949"}}, {"batch_size": "7", "compute_input": {"count": "287", "ns": "180182880"}, "compute_infer": {"count": "287", "ns": "10181532762"}, "compute_output": {"count": "287", "ns": "475356067"}}, {"batch_size": "8", "compute_input": {"count": "197", "ns": "151102295"}, "compute_infer": {"count": "197", "ns": "7615853345"}, "compute_output": {"count": "197", "ns": "334169277"}}, {"batch_size": "9", "compute_input": {"count": "199", "ns": "135472167"}, "compute_infer": {"count": "199", "ns": "8360673509"}, "compute_output": {"count": "199", "ns": "422404597"}}, {"batch_size": "10", "compute_input": {"count": "152", "ns": "158411452"}, "compute_infer": {"count": "152", "ns": "6951526439"}, "compute_output": {"count": "152", "ns": "376649511"}}, {"batch_size": "11", "compute_input": {"count": "114", "ns": "82745925"}, "compute_infer": {"count": "114", "ns": "6122572111"}, "compute_output": {"count": "114", "ns": "193942939"}}, {"batch_size": "12", "compute_input": {"count": "83", "ns": "63263392"}, "compute_infer": {"count": "83", "ns": "4795672495"}, "compute_output": {"count": "83", "ns": "217095142"}}, {"batch_size": "13", "compute_input": {"count": "79", "ns": "58867220"}, "compute_infer": {"count": "79", "ns": "4628513427"}, "compute_output": {"count": "79", "ns": "263091411"}}, {"batch_size": "14", "compute_input": {"count": "57", "ns": "47895186"}, "compute_infer": {"count": "57", "ns": "3932611692"}, "compute_output": {"count": "57", "ns": "160305833"}}, {"batch_size": "15", "compute_input": {"count": "40", "ns": "35987353"}, "compute_infer": {"count": "40", "ns": "3459230092"}, "compute_output": {"count": "40", "ns": "113817066"}}, {"batch_size": "16", "compute_input": {"count": "113", "ns": "116560452"}, "compute_infer": {"count": "113", "ns": "7878953876"}, "compute_output": {"count": "113", "ns": "473356382"}}]}, {"name": "feature_extractor", "version": "1", "last_inference": "1680592332101", "inference_count": "28704", "execution_count": "15456", "inference_stats": {"success": {"count": "28704", "ns": "352846080879"}, "fail": {}, "queue": {"count": "28704", "ns": "68676951968"}, "compute_input": {"count": "28704", "ns": "13489428398"}, "compute_infer": {"count": "28704", "ns": "261410939372"}, "compute_output": {"count": "28704", "ns": "7170977698"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "10597", "ns": "1632952660"}, "compute_infer": {"count": "10597", "ns": "33188652643"}, "compute_output": {"count": "10597", "ns": "1090569224"}}, {"batch_size": "2", "compute_input": {"count": "1201", "ns": "330558496"}, "compute_infer": {"count": "1201", "ns": "6911106965"}, "compute_output": {"count": "1201", "ns": "195536194"}}, {"batch_size": "3", "compute_input": {"count": "1441", "ns": "597110120"}, "compute_infer": {"count": "1441", "ns": "12631682658"}, "compute_output": {"count": "1441", "ns": "340920086"}}, {"batch_size": "4", "compute_input": {"count": "1062", "ns": "593939858"}, "compute_infer": {"count": "1062", "ns": "11779605706"}, "compute_output": {"count": "1062", "ns": "319437154"}}, {"batch_size": "5", "compute_input": {"count": "553", "ns": "390124072"}, "compute_infer": {"count": "553", "ns": "7597282943"}, "compute_output": {"count": "553", "ns": "203298692"}}, {"batch_size": "6", "compute_input": {"count": "296", "ns": "246223627"}, "compute_infer": {"count": "296", "ns": "4773469858"}, "compute_output": {"count": "296", "ns": "124773661"}}, {"batch_size": "7", "compute_input": {"count": "146", "ns": "145446326"}, "compute_infer": {"count": "146", "ns": "2733221875"}, "compute_output": {"count": "146", "ns": "71198769"}}, {"batch_size": "8", "compute_input": {"count": "68", "ns": "81446652"}, "compute_infer": {"count": "68", "ns": "1393913881"}, "compute_output": {"count": "68", "ns": "39804788"}}, {"batch_size": "9", "compute_input": {"count": "34", "ns": "44250565"}, "compute_infer": {"count": "34", "ns": "775981459"}, "compute_output": {"count": "34", "ns": "21074719"}}, {"batch_size": "10", "compute_input": {"count": "22", "ns": "31892434"}, "compute_infer": {"count": "22", "ns": "603878411"}, "compute_output": {"count": "22", "ns": "15147877"}}, {"batch_size": "11", "compute_input": {"count": "8", "ns": "12347500"}, "compute_infer": {"count": "8", "ns": "253269605"}, "compute_output": {"count": "8", "ns": "5961692"}}, {"batch_size": "12", "compute_input": {"count": "5", "ns": "9365072"}, "compute_infer": {"count": "5", "ns": "153878150"}, "compute_output": {"count": "5", "ns": "4212985"}}, {"batch_size": "13", "compute_input": {"count": "2", "ns": "3209737"}, "compute_infer": {"count": "2", "ns": "69706469"}, "compute_output": {"count": "2", "ns": "1642008"}}, {"batch_size": "14", "compute_input": {"count": "2", "ns": "7740604"}, "compute_infer": {"count": "2", "ns": "68949631"}, "compute_output": {"count": "2", "ns": "2223423"}}, {"batch_size": "15", "compute_input": {"count": "5", "ns": "11082588"}, "compute_infer": {"count": "5", "ns": "208950574"}, "compute_output": {"count": "5", "ns": "4830928"}}, {"batch_size": "16", "compute_input": {"count": "14", "ns": "40555793"}, "compute_infer": {"count": "14", "ns": "613415358"}, "compute_output": {"count": "14", "ns": "14039905"}}]}, {"name": "scoring", "version": "1", "last_inference": "1680592332128", "inference_count": "28704", "execution_count": "4432", "inference_stats": {"success": {"count": "28704", "ns": "4563397829420"}, "fail": {}, "queue": {"count": "28704", "ns": "1801684574764"}, "compute_input": {"count": "28704", "ns": "360828497154"}, "compute_infer": {"count": "28704", "ns": "2363178051243"}, "compute_output": {"count": "28704", "ns": "32056308901"}, "cache_hit": {}, "cache_miss": {}}, "batch_stats": [{"batch_size": "1", "compute_input": {"count": "1700", "ns": "3630853847"}, "compute_infer": {"count": "1700", "ns": "82636170464"}, "compute_output": {"count": "1700", "ns": "294124232"}}, {"batch_size": "2", "compute_input": {"count": "548", "ns": "1681145476"}, "compute_infer": {"count": "548", "ns": "25260015558"}, "compute_output": {"count": "548", "ns": "146923000"}}, {"batch_size": "3", "compute_input": {"count": "192", "ns": "763420332"}, "compute_infer": {"count": "192", "ns": "9467638684"}, "compute_output": {"count": "192", "ns": "68565391"}}, {"batch_size": "4", "compute_input": {"count": "112", "ns": "794664395"}, "compute_infer": {"count": "112", "ns": "7776812288"}, "compute_output": {"count": "112", "ns": "47803974"}}, {"batch_size": "5", "compute_input": {"count": "75", "ns": "546213700"}, "compute_infer": {"count": "75", "ns": "5310128526"}, "compute_output": {"count": "75", "ns": "37545527"}}, {"batch_size": "6", "compute_input": {"count": "84", "ns": "554285959"}, "compute_infer": {"count": "84", "ns": "5518876833"}, "compute_output": {"count": "84", "ns": "50505175"}}, {"batch_size": "7", "compute_input": {"count": "68", "ns": "689526124"}, "compute_infer": {"count": "68", "ns": "4555519171"}, "compute_output": {"count": "68", "ns": "45482481"}}, {"batch_size": "8", "compute_input": {"count": "107", "ns": "918791171"}, "compute_infer": {"count": "107", "ns": "6213554146"}, "compute_output": {"count": "107", "ns": "76338129"}}, {"batch_size": "9", "compute_input": {"count": "79", "ns": "692877958"}, "compute_infer": {"count": "79", "ns": "5913057867"}, "compute_output": {"count": "79", "ns": "64635371"}}, {"batch_size": "10", "compute_input": {"count": "81", "ns": "799665758"}, "compute_infer": {"count": "81", "ns": "6109135599"}, "compute_output": {"count": "81", "ns": "72275522"}}, {"batch_size": "11", "compute_input": {"count": "54", "ns": "587436303"}, "compute_infer": {"count": "54", "ns": "3835935339"}, "compute_output": {"count": "54", "ns": "55303744"}}, {"batch_size": "12", "compute_input": {"count": "88", "ns": "922341932"}, "compute_infer": {"count": "88", "ns": "8607996047"}, "compute_output": {"count": "88", "ns": "94690510"}}, {"batch_size": "13", "compute_input": {"count": "71", "ns": "919680264"}, "compute_infer": {"count": "71", "ns": "5796521939"}, "compute_output": {"count": "71", "ns": "78306760"}}, {"batch_size": "14", "compute_input": {"count": "66", "ns": "952771727"}, "compute_infer": {"count": "66", "ns": "5569791489"}, "compute_output": {"count": "66", "ns": "83334963"}}, {"batch_size": "15", "compute_input": {"count": "57", "ns": "828598088"}, "compute_infer": {"count": "57", "ns": "4987132977"}, "compute_output": {"count": "57", "ns": "75463317"}}, {"batch_size": "16", "compute_input": {"count": "1050", "ns": "16290409790"}, "compute_infer": {"count": "1050", "ns": "96331983417"}, "compute_output": {"count": "1050", "ns": "1455366096"}}]}]}

perf_log/model_repo_tlg_mbr_skip_blank_0.95/stats_summary-20.txt

@@ -0,0 +1,55 @@
+model name is attention_rescoring 
+queue 0.01  s, infer 733.02 s, input 59.50 s, output 13.62 s 
+Batch_size 1 , 7176  times, infer 733016.42 ms, avg 102.15 ms, 102.15 ms input 59495.12  ms, avg 8.29 ms, output 13621.64  ms, avg 1.90 ms 
+model name is encoder 
+queue 103.30 s, infer 336.36 s, input 5.32  s, output 6.22  s 
+Batch_size 1 , 2350  times, infer 38584.70  ms, avg 16.42 ms, 16.42 ms input 806.49    ms, avg 0.34 ms, output 778.85    ms, avg 0.33 ms 
+Batch_size 2 , 245   times, infer 5772.21   ms, avg 23.56 ms, 11.78 ms input 71.61     ms, avg 0.29 ms, output 153.31    ms, avg 0.63 ms 
+Batch_size 3 , 157   times, infer 4364.85   ms, avg 27.80 ms, 9.27 ms input 60.44     ms, avg 0.38 ms, output 113.19    ms, avg 0.72 ms 
+Batch_size 4 , 137   times, infer 4781.63   ms, avg 34.90 ms, 8.73 ms input 117.49    ms, avg 0.86 ms, output 139.50    ms, avg 1.02 ms 
+Batch_size 5 , 103   times, infer 3770.51   ms, avg 36.61 ms, 7.32 ms input 93.74     ms, avg 0.91 ms, output 88.13     ms, avg 0.86 ms 
+Batch_size 6 , 70    times, infer 3038.64   ms, avg 43.41 ms, 7.23 ms input 122.59    ms, avg 1.75 ms, output 111.61    ms, avg 1.59 ms 
+Batch_size 7 , 76    times, infer 3494.35   ms, avg 45.98 ms, 6.57 ms input 75.94     ms, avg 1.00 ms, output 100.99    ms, avg 1.33 ms 
+Batch_size 8 , 57    times, infer 2931.36   ms, avg 51.43 ms, 6.43 ms input 66.50     ms, avg 1.17 ms, output 64.83     ms, avg 1.14 ms 
+Batch_size 9 , 52    times, infer 2937.46   ms, avg 56.49 ms, 6.28 ms input 49.20     ms, avg 0.95 ms, output 55.58     ms, avg 1.07 ms 
+Batch_size 10, 34    times, infer 2245.12   ms, avg 66.03 ms, 6.60 ms input 47.11     ms, avg 1.39 ms, output 60.19     ms, avg 1.77 ms 
+Batch_size 11, 22    times, infer 2168.51   ms, avg 98.57 ms, 8.96 ms input 16.74     ms, avg 0.76 ms, output 37.26     ms, avg 1.69 ms 
+Batch_size 12, 5     times, infer 1472.48   ms, avg 294.50 ms, 24.54 ms input 5.15      ms, avg 1.03 ms, output 5.26      ms, avg 1.05 ms 
+Batch_size 13, 14    times, infer 1837.77   ms, avg 131.27 ms, 10.10 ms input 14.72     ms, avg 1.05 ms, output 16.58     ms, avg 1.18 ms 
+Batch_size 14, 4     times, infer 1538.20   ms, avg 384.55 ms, 27.47 ms input 5.30      ms, avg 1.32 ms, output 4.91      ms, avg 1.23 ms 
+Batch_size 15, 2     times, infer 1440.48   ms, avg 720.24 ms, 48.02 ms input 1.48      ms, avg 0.74 ms, output 2.03      ms, avg 1.01 ms 
+Batch_size 16, 1     times, infer 722.96    ms, avg 722.96 ms, 45.18 ms input 0.36      ms, avg 0.36 ms, output 0.59      ms, avg 0.59 ms 
+model name is feature_extractor 
+queue 23.49 s, infer 47.68 s, input 2.71  s, output 1.44  s 
+Batch_size 1 , 3779  times, infer 13936.22  ms, avg 3.69 ms, 3.69 ms input 591.20    ms, avg 0.16 ms, output 391.02    ms, avg 0.10 ms 
+Batch_size 2 , 337   times, infer 1966.00   ms, avg 5.83 ms, 2.92 ms input 94.69     ms, avg 0.28 ms, output 54.98     ms, avg 0.16 ms 
+Batch_size 3 , 328   times, infer 2898.84   ms, avg 8.84 ms, 2.95 ms input 139.55    ms, avg 0.43 ms, output 78.58     ms, avg 0.24 ms 
+Batch_size 4 , 213   times, infer 2073.19   ms, avg 9.73 ms, 2.43 ms input 123.17    ms, avg 0.58 ms, output 67.06     ms, avg 0.31 ms 
+Batch_size 5 , 86    times, infer 964.47    ms, avg 11.21 ms, 2.24 ms input 62.20     ms, avg 0.72 ms, output 32.79     ms, avg 0.38 ms 
+Batch_size 6 , 28    times, infer 389.10    ms, avg 13.90 ms, 2.32 ms input 25.60     ms, avg 0.91 ms, output 12.86     ms, avg 0.46 ms 
+Batch_size 7 , 18    times, infer 295.42    ms, avg 16.41 ms, 2.34 ms input 19.33     ms, avg 1.07 ms, output 9.26      ms, avg 0.51 ms 
+Batch_size 8 , 9     times, infer 174.49    ms, avg 19.39 ms, 2.42 ms input 11.00     ms, avg 1.22 ms, output 5.67      ms, avg 0.63 ms 
+Batch_size 9 , 2     times, infer 35.51     ms, avg 17.75 ms, 1.97 ms input 2.99      ms, avg 1.49 ms, output 1.22      ms, avg 0.61 ms 
+Batch_size 10, 2     times, infer 43.81     ms, avg 21.91 ms, 2.19 ms input 2.99      ms, avg 1.50 ms, output 1.43      ms, avg 0.72 ms 
+Batch_size 11, 1     times, infer 27.87     ms, avg 27.87 ms, 2.53 ms input 1.88      ms, avg 1.88 ms, output 0.79      ms, avg 0.79 ms 
+Batch_size 12, 1     times, infer 30.93     ms, avg 30.93 ms, 2.58 ms input 1.86      ms, avg 1.86 ms, output 0.83      ms, avg 0.83 ms 
+Batch_size 14, 1     times, infer 30.59     ms, avg 30.59 ms, 2.18 ms input 5.77      ms, avg 5.77 ms, output 1.24      ms, avg 1.24 ms 
+Batch_size 16, 1     times, infer 21.41     ms, avg 21.41 ms, 1.34 ms input 9.43      ms, avg 9.43 ms, output 1.17      ms, avg 1.17 ms 
+model name is scoring 
+queue 154.43 s, infer 348.97 s, input 51.46 s, output 5.97  s 
+Batch_size 1 , 1133  times, infer 49068.15  ms, avg 43.31 ms, 43.31 ms input 1957.24   ms, avg 1.73 ms, output 193.56    ms, avg 0.17 ms 
+Batch_size 2 , 405   times, infer 15299.88  ms, avg 37.78 ms, 18.89 ms input 1096.83   ms, avg 2.71 ms, output 107.52    ms, avg 0.27 ms 
+Batch_size 3 , 128   times, infer 4890.97   ms, avg 38.21 ms, 12.74 ms input 369.49    ms, avg 2.89 ms, output 44.41     ms, avg 0.35 ms 
+Batch_size 4 , 46    times, infer 2010.23   ms, avg 43.70 ms, 10.93 ms input 220.33    ms, avg 4.79 ms, output 19.75     ms, avg 0.43 ms 
+Batch_size 5 , 25    times, infer 1296.76   ms, avg 51.87 ms, 10.37 ms input 126.33    ms, avg 5.05 ms, output 13.16     ms, avg 0.53 ms 
+Batch_size 6 , 41    times, infer 1981.84   ms, avg 48.34 ms, 8.06 ms input 229.89    ms, avg 5.61 ms, output 24.80     ms, avg 0.60 ms 
+Batch_size 7 , 29    times, infer 1510.36   ms, avg 52.08 ms, 7.44 ms input 223.29    ms, avg 7.70 ms, output 18.70     ms, avg 0.64 ms 
+Batch_size 8 , 36    times, infer 1710.74   ms, avg 47.52 ms, 5.94 ms input 330.62    ms, avg 9.18 ms, output 24.94     ms, avg 0.69 ms 
+Batch_size 9 , 28    times, infer 1587.65   ms, avg 56.70 ms, 6.30 ms input 218.26    ms, avg 7.80 ms, output 22.72     ms, avg 0.81 ms 
+Batch_size 10, 25    times, infer 1465.36   ms, avg 58.61 ms, 5.86 ms input 219.05    ms, avg 8.76 ms, output 21.94     ms, avg 0.88 ms 
+Batch_size 11, 24    times, infer 1308.29   ms, avg 54.51 ms, 4.96 ms input 270.00    ms, avg 11.25 ms, output 25.68     ms, avg 1.07 ms 
+Batch_size 12, 13    times, infer 745.34    ms, avg 57.33 ms, 4.78 ms input 98.46     ms, avg 7.57 ms, output 13.88     ms, avg 1.07 ms 
+Batch_size 13, 13    times, infer 759.78    ms, avg 58.44 ms, 4.50 ms input 112.89    ms, avg 8.68 ms, output 14.47     ms, avg 1.11 ms 
+Batch_size 14, 14    times, infer 837.73    ms, avg 59.84 ms, 4.27 ms input 151.08    ms, avg 10.79 ms, output 16.67     ms, avg 1.19 ms 
+Batch_size 15, 12    times, infer 705.83    ms, avg 58.82 ms, 3.92 ms input 117.98    ms, avg 9.83 ms, output 15.69     ms, avg 1.31 ms 
+Batch_size 16, 146   times, infer 7468.03   ms, avg 51.15 ms, 3.20 ms input 1590.19   ms, avg 10.89 ms, output 204.46    ms, avg 1.40 ms 

perf_log/model_repo_tlg_mbr_skip_blank_0.95/stats_summary-40.txt

@@ -0,0 +1,56 @@
+model name is attention_rescoring 
+queue 0.02  s, infer 1532.76 s, input 149.36 s, output 32.65 s 
+Batch_size 1 , 14352 times, infer 1532763.42 ms, avg 106.80 ms, 106.80 ms input 149359.27 ms, avg 10.41 ms, output 32651.32  ms, avg 2.28 ms 
+model name is encoder 
+queue 165.46 s, infer 563.22 s, input 9.56  s, output 15.83 s 
+Batch_size 1 , 3800  times, infer 63031.42  ms, avg 16.59 ms, 16.59 ms input 1240.80   ms, avg 0.33 ms, output 1330.72   ms, avg 0.35 ms 
+Batch_size 2 , 430   times, infer 9433.30   ms, avg 21.94 ms, 10.97 ms input 113.48    ms, avg 0.26 ms, output 265.38    ms, avg 0.62 ms 
+Batch_size 3 , 302   times, infer 7754.73   ms, avg 25.68 ms, 8.56 ms input 114.13    ms, avg 0.38 ms, output 255.85    ms, avg 0.85 ms 
+Batch_size 4 , 264   times, infer 7930.14   ms, avg 30.04 ms, 7.51 ms input 183.32    ms, avg 0.69 ms, output 246.35    ms, avg 0.93 ms 
+Batch_size 5 , 231   times, infer 7225.76   ms, avg 31.28 ms, 6.26 ms input 143.14    ms, avg 0.62 ms, output 279.01    ms, avg 1.21 ms 
+Batch_size 6 , 161   times, infer 5670.79   ms, avg 35.22 ms, 5.87 ms input 167.22    ms, avg 1.04 ms, output 230.26    ms, avg 1.43 ms 
+Batch_size 7 , 142   times, infer 5555.56   ms, avg 39.12 ms, 5.59 ms input 111.87    ms, avg 0.79 ms, output 212.42    ms, avg 1.50 ms 
+Batch_size 8 , 108   times, infer 4571.08   ms, avg 42.32 ms, 5.29 ms input 111.44    ms, avg 1.03 ms, output 155.29    ms, avg 1.44 ms 
+Batch_size 9 , 92    times, infer 4392.65   ms, avg 47.75 ms, 5.31 ms input 76.26     ms, avg 0.83 ms, output 130.10    ms, avg 1.41 ms 
+Batch_size 10, 81    times, infer 4033.26   ms, avg 49.79 ms, 4.98 ms input 105.07    ms, avg 1.30 ms, output 145.35    ms, avg 1.79 ms 
+Batch_size 11, 54    times, infer 3420.13   ms, avg 63.34 ms, 5.76 ms input 39.51     ms, avg 0.73 ms, output 93.12     ms, avg 1.72 ms 
+Batch_size 12, 27    times, infer 2359.81   ms, avg 87.40 ms, 7.28 ms input 23.43     ms, avg 0.87 ms, output 59.21     ms, avg 2.19 ms 
+Batch_size 13, 30    times, infer 2530.02   ms, avg 84.33 ms, 6.49 ms input 27.07     ms, avg 0.90 ms, output 49.03     ms, avg 1.63 ms 
+Batch_size 14, 24    times, infer 2381.58   ms, avg 99.23 ms, 7.09 ms input 25.81     ms, avg 1.08 ms, output 45.55     ms, avg 1.90 ms 
+Batch_size 15, 11    times, infer 1874.44   ms, avg 170.40 ms, 11.36 ms input 8.91      ms, avg 0.81 ms, output 20.49     ms, avg 1.86 ms 
+Batch_size 16, 19    times, infer 2535.61   ms, avg 133.45 ms, 8.34 ms input 20.47     ms, avg 1.08 ms, output 47.86     ms, avg 2.52 ms 
+model name is feature_extractor 
+queue 36.08 s, infer 108.63 s, input 5.96  s, output 3.18  s 
+Batch_size 1 , 6424  times, infer 21421.26  ms, avg 3.33 ms, 3.33 ms input 1008.99   ms, avg 0.16 ms, output 669.51    ms, avg 0.10 ms 
+Batch_size 2 , 672   times, infer 3996.95   ms, avg 5.95 ms, 2.97 ms input 191.02    ms, avg 0.28 ms, output 112.10    ms, avg 0.17 ms 
+Batch_size 3 , 716   times, infer 6322.93   ms, avg 8.83 ms, 2.94 ms input 305.77    ms, avg 0.43 ms, output 172.86    ms, avg 0.24 ms 
+Batch_size 4 , 476   times, infer 4958.15   ms, avg 10.42 ms, 2.60 ms input 274.44    ms, avg 0.58 ms, output 148.90    ms, avg 0.31 ms 
+Batch_size 5 , 228   times, infer 2835.91   ms, avg 12.44 ms, 2.49 ms input 167.95    ms, avg 0.74 ms, output 85.53     ms, avg 0.38 ms 
+Batch_size 6 , 104   times, infer 1644.05   ms, avg 15.81 ms, 2.63 ms input 90.26     ms, avg 0.87 ms, output 45.10     ms, avg 0.43 ms 
+Batch_size 7 , 46    times, infer 796.98    ms, avg 17.33 ms, 2.48 ms input 47.31     ms, avg 1.03 ms, output 22.89     ms, avg 0.50 ms 
+Batch_size 8 , 25    times, infer 494.52    ms, avg 19.78 ms, 2.47 ms input 30.85     ms, avg 1.23 ms, output 14.55     ms, avg 0.58 ms 
+Batch_size 9 , 9     times, infer 193.99    ms, avg 21.55 ms, 2.39 ms input 11.98     ms, avg 1.33 ms, output 5.63      ms, avg 0.63 ms 
+Batch_size 10, 6     times, infer 160.09    ms, avg 26.68 ms, 2.67 ms input 9.01      ms, avg 1.50 ms, output 4.57      ms, avg 0.76 ms 
+Batch_size 11, 2     times, infer 54.81     ms, avg 27.40 ms, 2.49 ms input 3.19      ms, avg 1.60 ms, output 1.58      ms, avg 0.79 ms 
+Batch_size 12, 2     times, infer 52.74     ms, avg 26.37 ms, 2.20 ms input 3.95      ms, avg 1.97 ms, output 1.78      ms, avg 0.89 ms 
+Batch_size 13, 1     times, infer 43.05     ms, avg 43.05 ms, 3.31 ms input 1.80      ms, avg 1.80 ms, output 0.78      ms, avg 0.78 ms 
+Batch_size 14, 1     times, infer 30.59     ms, avg 30.59 ms, 2.18 ms input 5.77      ms, avg 5.77 ms, output 1.24      ms, avg 1.24 ms 
+Batch_size 16, 2     times, infer 78.87     ms, avg 39.44 ms, 2.46 ms input 13.06     ms, avg 6.53 ms, output 1.97      ms, avg 0.98 ms 
+model name is scoring 
+queue 432.93 s, infer 860.92 s, input 133.84 s, output 13.65 s 
+Batch_size 1 , 1485  times, infer 69446.53  ms, avg 46.77 ms, 46.77 ms input 2755.84   ms, avg 1.86 ms, output 255.37    ms, avg 0.17 ms 
+Batch_size 2 , 504   times, infer 21986.98  ms, avg 43.62 ms, 21.81 ms input 1453.35   ms, avg 2.88 ms, output 135.27    ms, avg 0.27 ms 
+Batch_size 3 , 168   times, infer 7605.94   ms, avg 45.27 ms, 15.09 ms input 592.18    ms, avg 3.52 ms, output 59.26     ms, avg 0.35 ms 
+Batch_size 4 , 84    times, infer 5022.51   ms, avg 59.79 ms, 14.95 ms input 526.83    ms, avg 6.27 ms, output 36.20     ms, avg 0.43 ms 
+Batch_size 5 , 56    times, infer 3379.20   ms, avg 60.34 ms, 12.07 ms input 388.00    ms, avg 6.93 ms, output 28.26     ms, avg 0.50 ms 
+Batch_size 6 , 66    times, infer 3733.47   ms, avg 56.57 ms, 9.43 ms input 416.24    ms, avg 6.31 ms, output 39.96     ms, avg 0.61 ms 
+Batch_size 7 , 54    times, infer 3198.17   ms, avg 59.23 ms, 8.46 ms input 451.70    ms, avg 8.36 ms, output 35.86     ms, avg 0.66 ms 
+Batch_size 8 , 93    times, infer 4997.85   ms, avg 53.74 ms, 6.72 ms input 697.37    ms, avg 7.50 ms, output 65.78     ms, avg 0.71 ms 
+Batch_size 9 , 57    times, infer 3823.36   ms, avg 67.08 ms, 7.45 ms input 443.95    ms, avg 7.79 ms, output 46.46     ms, avg 0.82 ms 
+Batch_size 10, 59    times, infer 3805.60   ms, avg 64.50 ms, 6.45 ms input 517.45    ms, avg 8.77 ms, output 52.01     ms, avg 0.88 ms 
+Batch_size 11, 42    times, infer 2771.30   ms, avg 65.98 ms, 6.00 ms input 436.71    ms, avg 10.40 ms, output 44.67     ms, avg 1.06 ms 
+Batch_size 12, 24    times, infer 1597.52   ms, avg 66.56 ms, 5.55 ms input 207.79    ms, avg 8.66 ms, output 25.49     ms, avg 1.06 ms 
+Batch_size 13, 45    times, infer 2922.91   ms, avg 64.95 ms, 5.00 ms input 510.37    ms, avg 11.34 ms, output 49.67     ms, avg 1.10 ms 
+Batch_size 14, 47    times, infer 3518.19   ms, avg 74.86 ms, 5.35 ms input 634.09    ms, avg 13.49 ms, output 57.32     ms, avg 1.22 ms 
+Batch_size 15, 35    times, infer 2352.44   ms, avg 67.21 ms, 4.48 ms input 463.89    ms, avg 13.25 ms, output 46.27     ms, avg 1.32 ms 
+Batch_size 16, 350   times, infer 22391.40  ms, avg 63.98 ms, 4.00 ms input 4510.84   ms, avg 12.89 ms, output 485.14    ms, avg 1.39 ms 

perf_log/model_repo_tlg_mbr_skip_blank_0.95/stats_summary-60.txt

@@ -0,0 +1,57 @@
+model name is attention_rescoring 
+queue 0.03  s, infer 2570.22 s, input 258.94 s, output 54.81 s 
+Batch_size 1 , 21528 times, infer 2570223.74 ms, avg 119.39 ms, 119.39 ms input 258943.33 ms, avg 12.03 ms, output 54813.62  ms, avg 2.55 ms 
+model name is encoder 
+queue 239.86 s, infer 793.04 s, input 13.32 s, output 27.21 s 
+Batch_size 1 , 4953  times, infer 83422.97  ms, avg 16.84 ms, 16.84 ms input 1660.73   ms, avg 0.34 ms, output 1818.37   ms, avg 0.37 ms 
+Batch_size 2 , 606   times, infer 14596.71  ms, avg 24.09 ms, 12.04 ms input 169.85    ms, avg 0.28 ms, output 404.05    ms, avg 0.67 ms 
+Batch_size 3 , 438   times, infer 11055.18  ms, avg 25.24 ms, 8.41 ms input 150.52    ms, avg 0.34 ms, output 386.02    ms, avg 0.88 ms 
+Batch_size 4 , 409   times, infer 11665.67  ms, avg 28.52 ms, 7.13 ms input 227.16    ms, avg 0.56 ms, output 380.87    ms, avg 0.93 ms 
+Batch_size 5 , 333   times, infer 9973.34   ms, avg 29.95 ms, 5.99 ms input 184.11    ms, avg 0.55 ms, output 394.00    ms, avg 1.18 ms 
+Batch_size 6 , 262   times, infer 8570.06   ms, avg 32.71 ms, 5.45 ms input 222.05    ms, avg 0.85 ms, output 346.64    ms, avg 1.32 ms 
+Batch_size 7 , 218   times, infer 7854.12   ms, avg 36.03 ms, 5.15 ms input 141.29    ms, avg 0.65 ms, output 305.56    ms, avg 1.40 ms 
+Batch_size 8 , 163   times, infer 6464.99   ms, avg 39.66 ms, 4.96 ms input 138.80    ms, avg 0.85 ms, output 275.70    ms, avg 1.69 ms 
+Batch_size 9 , 153   times, infer 6471.11   ms, avg 42.29 ms, 4.70 ms input 113.08    ms, avg 0.74 ms, output 251.68    ms, avg 1.64 ms 
+Batch_size 10, 117   times, infer 5444.17   ms, avg 46.53 ms, 4.65 ms input 136.08    ms, avg 1.16 ms, output 208.53    ms, avg 1.78 ms 
+Batch_size 11, 82    times, infer 4604.90   ms, avg 56.16 ms, 5.11 ms input 53.88     ms, avg 0.66 ms, output 140.50    ms, avg 1.71 ms 
+Batch_size 12, 57    times, infer 3632.28   ms, avg 63.72 ms, 5.31 ms input 42.60     ms, avg 0.75 ms, output 135.19    ms, avg 2.37 ms 
+Batch_size 13, 46    times, infer 3155.13   ms, avg 68.59 ms, 5.28 ms input 33.52     ms, avg 0.73 ms, output 113.42    ms, avg 2.47 ms 
+Batch_size 14, 31    times, infer 2701.46   ms, avg 87.14 ms, 6.22 ms input 30.34     ms, avg 0.98 ms, output 81.59     ms, avg 2.63 ms 
+Batch_size 15, 19    times, infer 2271.82   ms, avg 119.57 ms, 7.97 ms input 16.03     ms, avg 0.84 ms, output 40.92     ms, avg 2.15 ms 
+Batch_size 16, 56    times, infer 4547.99   ms, avg 81.21 ms, 5.08 ms input 63.99     ms, avg 1.14 ms, output 172.24    ms, avg 3.08 ms 
+model name is feature_extractor 
+queue 51.60 s, infer 183.09 s, input 9.63  s, output 5.16  s 
+Batch_size 1 , 8608  times, infer 27630.61  ms, avg 3.21 ms, 3.21 ms input 1337.94   ms, avg 0.16 ms, output 891.08    ms, avg 0.10 ms 
+Batch_size 2 , 966   times, infer 5678.95   ms, avg 5.88 ms, 2.94 ms input 270.02    ms, avg 0.28 ms, output 158.90    ms, avg 0.16 ms 
+Batch_size 3 , 1066  times, infer 9490.08   ms, avg 8.90 ms, 2.97 ms input 445.91    ms, avg 0.42 ms, output 253.61    ms, avg 0.24 ms 
+Batch_size 4 , 758   times, infer 8266.38   ms, avg 10.91 ms, 2.73 ms input 428.19    ms, avg 0.56 ms, output 231.99    ms, avg 0.31 ms 
+Batch_size 5 , 391   times, infer 5202.95   ms, avg 13.31 ms, 2.66 ms input 278.96    ms, avg 0.71 ms, output 146.00    ms, avg 0.37 ms 
+Batch_size 6 , 197   times, infer 3155.23   ms, avg 16.02 ms, 2.67 ms input 168.10    ms, avg 0.85 ms, output 84.56     ms, avg 0.43 ms 
+Batch_size 7 , 85    times, infer 1571.35   ms, avg 18.49 ms, 2.64 ms input 85.26     ms, avg 1.00 ms, output 42.09     ms, avg 0.50 ms 
+Batch_size 8 , 50    times, infer 1000.88   ms, avg 20.02 ms, 2.50 ms input 59.80     ms, avg 1.20 ms, output 30.22     ms, avg 0.60 ms 
+Batch_size 9 , 25    times, infer 573.86    ms, avg 22.95 ms, 2.55 ms input 32.55     ms, avg 1.30 ms, output 15.35     ms, avg 0.61 ms 
+Batch_size 10, 14    times, infer 390.06    ms, avg 27.86 ms, 2.79 ms input 20.76     ms, avg 1.48 ms, output 9.77      ms, avg 0.70 ms 
+Batch_size 11, 3     times, infer 84.83     ms, avg 28.28 ms, 2.57 ms input 4.50      ms, avg 1.50 ms, output 2.21      ms, avg 0.74 ms 
+Batch_size 12, 4     times, infer 116.91    ms, avg 29.23 ms, 2.44 ms input 7.56      ms, avg 1.89 ms, output 3.50      ms, avg 0.87 ms 
+Batch_size 13, 2     times, infer 69.71     ms, avg 34.85 ms, 2.68 ms input 3.21      ms, avg 1.60 ms, output 1.64      ms, avg 0.82 ms 
+Batch_size 14, 2     times, infer 68.95     ms, avg 34.47 ms, 2.46 ms input 7.74      ms, avg 3.87 ms, output 2.22      ms, avg 1.11 ms 
+Batch_size 15, 2     times, infer 78.75     ms, avg 39.37 ms, 2.62 ms input 4.55      ms, avg 2.28 ms, output 2.15      ms, avg 1.08 ms 
+Batch_size 16, 6     times, infer 259.87    ms, avg 43.31 ms, 2.71 ms input 22.88     ms, avg 3.81 ms, output 6.27      ms, avg 1.05 ms 
+model name is scoring 
+queue 926.25 s, infer 1594.10 s, input 235.99 s, output 22.45 s 
+Batch_size 1 , 1608  times, infer 77165.55  ms, avg 47.99 ms, 47.99 ms input 3151.81   ms, avg 1.96 ms, output 277.50    ms, avg 0.17 ms 
+Batch_size 2 , 530   times, infer 24089.22  ms, avg 45.45 ms, 22.73 ms input 1584.85   ms, avg 2.99 ms, output 142.43    ms, avg 0.27 ms 
+Batch_size 3 , 183   times, infer 8828.88   ms, avg 48.25 ms, 16.08 ms input 699.42    ms, avg 3.82 ms, output 64.98     ms, avg 0.36 ms 
+Batch_size 4 , 100   times, infer 6452.21   ms, avg 64.52 ms, 16.13 ms input 661.11    ms, avg 6.61 ms, output 42.68     ms, avg 0.43 ms 
+Batch_size 5 , 72    times, infer 4907.87   ms, avg 68.16 ms, 13.63 ms input 531.29    ms, avg 7.38 ms, output 35.88     ms, avg 0.50 ms 
+Batch_size 6 , 81    times, infer 5230.42   ms, avg 64.57 ms, 10.76 ms input 532.84    ms, avg 6.58 ms, output 48.48     ms, avg 0.60 ms 
+Batch_size 7 , 63    times, infer 4086.70   ms, avg 64.87 ms, 9.27 ms input 589.27    ms, avg 9.35 ms, output 41.99     ms, avg 0.67 ms 
+Batch_size 8 , 102   times, infer 5821.72   ms, avg 57.08 ms, 7.13 ms input 816.77    ms, avg 8.01 ms, output 72.60     ms, avg 0.71 ms 
+Batch_size 9 , 75    times, infer 5571.41   ms, avg 74.29 ms, 8.25 ms input 636.60    ms, avg 8.49 ms, output 61.09     ms, avg 0.81 ms 
+Batch_size 10, 72    times, infer 5144.16   ms, avg 71.45 ms, 7.14 ms input 655.39    ms, avg 9.10 ms, output 64.66     ms, avg 0.90 ms 
+Batch_size 11, 50    times, infer 3369.98   ms, avg 67.40 ms, 6.13 ms input 534.86    ms, avg 10.70 ms, output 51.44     ms, avg 1.03 ms 
+Batch_size 12, 85    times, infer 8183.84   ms, avg 96.28 ms, 8.02 ms input 884.32    ms, avg 10.40 ms, output 91.80     ms, avg 1.08 ms 
+Batch_size 13, 64    times, infer 5117.56   ms, avg 79.96 ms, 6.15 ms input 793.02    ms, avg 12.39 ms, output 70.98     ms, avg 1.11 ms 
+Batch_size 14, 59    times, infer 4641.75   ms, avg 78.67 ms, 5.62 ms input 807.49    ms, avg 13.69 ms, output 74.74     ms, avg 1.27 ms 
+Batch_size 15, 47    times, infer 3752.09   ms, avg 79.83 ms, 5.32 ms input 649.65    ms, avg 13.82 ms, output 62.91     ms, avg 1.34 ms 
+Batch_size 16, 655   times, infer 53793.92  ms, avg 82.13 ms, 5.13 ms input 9267.30   ms, avg 14.15 ms, output 899.89    ms, avg 1.37 ms 

perf_log/model_repo_tlg_mbr_skip_blank_0.95/stats_summary-80.txt

@@ -0,0 +1,57 @@
+model name is attention_rescoring 
+queue 0.04  s, infer 3688.06 s, input 391.71 s, output 84.78 s 
+Batch_size 1 , 28704 times, infer 3688057.03 ms, avg 128.49 ms, 128.49 ms input 391710.87 ms, avg 13.65 ms, output 84777.49  ms, avg 2.95 ms 
+model name is encoder 
+queue 327.80 s, infer 1063.47 s, input 17.39 s, output 45.55 s 
+Batch_size 1 , 5891  times, infer 101455.30 ms, avg 17.22 ms, 17.22 ms input 1985.77   ms, avg 0.34 ms, output 2293.38   ms, avg 0.39 ms 
+Batch_size 2 , 728   times, infer 17577.81  ms, avg 24.15 ms, 12.07 ms input 206.83    ms, avg 0.28 ms, output 542.99    ms, avg 0.75 ms 
+Batch_size 3 , 549   times, infer 13964.07  ms, avg 25.44 ms, 8.48 ms input 180.43    ms, avg 0.33 ms, output 499.37    ms, avg 0.91 ms 
+Batch_size 4 , 533   times, infer 15313.17  ms, avg 28.73 ms, 7.18 ms input 271.58    ms, avg 0.51 ms, output 493.00    ms, avg 0.92 ms 
+Batch_size 5 , 415   times, infer 12199.96  ms, avg 29.40 ms, 5.88 ms input 213.74    ms, avg 0.52 ms, output 479.12    ms, avg 1.15 ms 
+Batch_size 6 , 354   times, infer 11279.71  ms, avg 31.86 ms, 5.31 ms input 252.30    ms, avg 0.71 ms, output 508.70    ms, avg 1.44 ms 
+Batch_size 7 , 287   times, infer 10181.53  ms, avg 35.48 ms, 5.07 ms input 180.18    ms, avg 0.63 ms, output 475.36    ms, avg 1.66 ms 
+Batch_size 8 , 197   times, infer 7615.85   ms, avg 38.66 ms, 4.83 ms input 151.10    ms, avg 0.77 ms, output 334.17    ms, avg 1.70 ms 
+Batch_size 9 , 199   times, infer 8360.67   ms, avg 42.01 ms, 4.67 ms input 135.47    ms, avg 0.68 ms, output 422.40    ms, avg 2.12 ms 
+Batch_size 10, 152   times, infer 6951.53   ms, avg 45.73 ms, 4.57 ms input 158.41    ms, avg 1.04 ms, output 376.65    ms, avg 2.48 ms 
+Batch_size 11, 114   times, infer 6122.57   ms, avg 53.71 ms, 4.88 ms input 82.75     ms, avg 0.73 ms, output 193.94    ms, avg 1.70 ms 
+Batch_size 12, 83    times, infer 4795.67   ms, avg 57.78 ms, 4.81 ms input 63.26     ms, avg 0.76 ms, output 217.10    ms, avg 2.62 ms 
+Batch_size 13, 79    times, infer 4628.51   ms, avg 58.59 ms, 4.51 ms input 58.87     ms, avg 0.75 ms, output 263.09    ms, avg 3.33 ms 
+Batch_size 14, 57    times, infer 3932.61   ms, avg 68.99 ms, 4.93 ms input 47.90     ms, avg 0.84 ms, output 160.31    ms, avg 2.81 ms 
+Batch_size 15, 40    times, infer 3459.23   ms, avg 86.48 ms, 5.77 ms input 35.99     ms, avg 0.90 ms, output 113.82    ms, avg 2.85 ms 
+Batch_size 16, 113   times, infer 7878.95   ms, avg 69.73 ms, 4.36 ms input 116.56    ms, avg 1.03 ms, output 473.36    ms, avg 4.19 ms 
+model name is feature_extractor 
+queue 68.68 s, infer 261.41 s, input 13.49 s, output 7.17  s 
+Batch_size 1 , 10597 times, infer 33188.65  ms, avg 3.13 ms, 3.13 ms input 1632.95   ms, avg 0.15 ms, output 1090.57   ms, avg 0.10 ms 
+Batch_size 2 , 1201  times, infer 6911.11   ms, avg 5.75 ms, 2.88 ms input 330.56    ms, avg 0.28 ms, output 195.54    ms, avg 0.16 ms 
+Batch_size 3 , 1441  times, infer 12631.68  ms, avg 8.77 ms, 2.92 ms input 597.11    ms, avg 0.41 ms, output 340.92    ms, avg 0.24 ms 
+Batch_size 4 , 1062  times, infer 11779.61  ms, avg 11.09 ms, 2.77 ms input 593.94    ms, avg 0.56 ms, output 319.44    ms, avg 0.30 ms 
+Batch_size 5 , 553   times, infer 7597.28   ms, avg 13.74 ms, 2.75 ms input 390.12    ms, avg 0.71 ms, output 203.30    ms, avg 0.37 ms 
+Batch_size 6 , 296   times, infer 4773.47   ms, avg 16.13 ms, 2.69 ms input 246.22    ms, avg 0.83 ms, output 124.77    ms, avg 0.42 ms 
+Batch_size 7 , 146   times, infer 2733.22   ms, avg 18.72 ms, 2.67 ms input 145.45    ms, avg 1.00 ms, output 71.20     ms, avg 0.49 ms 
+Batch_size 8 , 68    times, infer 1393.91   ms, avg 20.50 ms, 2.56 ms input 81.45     ms, avg 1.20 ms, output 39.80     ms, avg 0.59 ms 
+Batch_size 9 , 34    times, infer 775.98    ms, avg 22.82 ms, 2.54 ms input 44.25     ms, avg 1.30 ms, output 21.07     ms, avg 0.62 ms 
+Batch_size 10, 22    times, infer 603.88    ms, avg 27.45 ms, 2.74 ms input 31.89     ms, avg 1.45 ms, output 15.15     ms, avg 0.69 ms 
+Batch_size 11, 8     times, infer 253.27    ms, avg 31.66 ms, 2.88 ms input 12.35     ms, avg 1.54 ms, output 5.96      ms, avg 0.75 ms 
+Batch_size 12, 5     times, infer 153.88    ms, avg 30.78 ms, 2.56 ms input 9.37      ms, avg 1.87 ms, output 4.21      ms, avg 0.84 ms 
+Batch_size 13, 2     times, infer 69.71     ms, avg 34.85 ms, 2.68 ms input 3.21      ms, avg 1.60 ms, output 1.64      ms, avg 0.82 ms 
+Batch_size 14, 2     times, infer 68.95     ms, avg 34.47 ms, 2.46 ms input 7.74      ms, avg 3.87 ms, output 2.22      ms, avg 1.11 ms 
+Batch_size 15, 5     times, infer 208.95    ms, avg 41.79 ms, 2.79 ms input 11.08     ms, avg 2.22 ms, output 4.83      ms, avg 0.97 ms 
+Batch_size 16, 14    times, infer 613.42    ms, avg 43.82 ms, 2.74 ms input 40.56     ms, avg 2.90 ms, output 14.04     ms, avg 1.00 ms 
+model name is scoring 
+queue 1801.68 s, infer 2363.18 s, input 360.83 s, output 32.06 s 
+Batch_size 1 , 1700  times, infer 82636.17  ms, avg 48.61 ms, 48.61 ms input 3630.85   ms, avg 2.14 ms, output 294.12    ms, avg 0.17 ms 
+Batch_size 2 , 548   times, infer 25260.02  ms, avg 46.09 ms, 23.05 ms input 1681.15   ms, avg 3.07 ms, output 146.92    ms, avg 0.27 ms 
+Batch_size 3 , 192   times, infer 9467.64   ms, avg 49.31 ms, 16.44 ms input 763.42    ms, avg 3.98 ms, output 68.57     ms, avg 0.36 ms 
+Batch_size 4 , 112   times, infer 7776.81   ms, avg 69.44 ms, 17.36 ms input 794.66    ms, avg 7.10 ms, output 47.80     ms, avg 0.43 ms 
+Batch_size 5 , 75    times, infer 5310.13   ms, avg 70.80 ms, 14.16 ms input 546.21    ms, avg 7.28 ms, output 37.55     ms, avg 0.50 ms 
+Batch_size 6 , 84    times, infer 5518.88   ms, avg 65.70 ms, 10.95 ms input 554.29    ms, avg 6.60 ms, output 50.51     ms, avg 0.60 ms 
+Batch_size 7 , 68    times, infer 4555.52   ms, avg 66.99 ms, 9.57 ms input 689.53    ms, avg 10.14 ms, output 45.48     ms, avg 0.67 ms 
+Batch_size 8 , 107   times, infer 6213.55   ms, avg 58.07 ms, 7.26 ms input 918.79    ms, avg 8.59 ms, output 76.34     ms, avg 0.71 ms 
+Batch_size 9 , 79    times, infer 5913.06   ms, avg 74.85 ms, 8.32 ms input 692.88    ms, avg 8.77 ms, output 64.64     ms, avg 0.82 ms 
+Batch_size 10, 81    times, infer 6109.14   ms, avg 75.42 ms, 7.54 ms input 799.67    ms, avg 9.87 ms, output 72.28     ms, avg 0.89 ms 
+Batch_size 11, 54    times, infer 3835.94   ms, avg 71.04 ms, 6.46 ms input 587.44    ms, avg 10.88 ms, output 55.30     ms, avg 1.02 ms 
+Batch_size 12, 88    times, infer 8608.00   ms, avg 97.82 ms, 8.15 ms input 922.34    ms, avg 10.48 ms, output 94.69     ms, avg 1.08 ms 
+Batch_size 13, 71    times, infer 5796.52   ms, avg 81.64 ms, 6.28 ms input 919.68    ms, avg 12.95 ms, output 78.31     ms, avg 1.10 ms 
+Batch_size 14, 66    times, infer 5569.79   ms, avg 84.39 ms, 6.03 ms input 952.77    ms, avg 14.44 ms, output 83.33     ms, avg 1.26 ms 
+Batch_size 15, 57    times, infer 4987.13   ms, avg 87.49 ms, 5.83 ms input 828.60    ms, avg 14.54 ms, output 75.46     ms, avg 1.32 ms 
+Batch_size 16, 1050  times, infer 96331.98  ms, avg 91.74 ms, 5.73 ms input 16290.41  ms, avg 15.51 ms, output 1455.37   ms, avg 1.39 ms